ATLAS-1267: Client for V2 APIs and TypesREST integration tests

client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.sun.jersey.api.client.WebResource;
+
+import org.apache.atlas.model.SearchFilter;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.commons.configuration.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.util.List;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+
+public class AtlasEntitiesClientV2 extends AtlasBaseClient {
+
+    public static final String ENTITY_API = BASE_URI + "v2/entity/";
+    public static final String ENTITIES_API = BASE_URI + "v2/entities/";
+
+    private static final APIInfo GET_ENTITY_BY_GUID = new APIInfo(ENTITY_API + "guid/", HttpMethod.GET, Response.Status.OK);
+    private static final APIInfo CREATE_ENTITY = new APIInfo(ENTITY_API, HttpMethod.POST, Response.Status.OK);
+    private static final APIInfo UPDATE_ENTITY = CREATE_ENTITY;
+    private static final APIInfo UPDATE_ENTITY_BY_GUID = new APIInfo(ENTITY_API + "guid/", HttpMethod.PUT, Response.Status.OK);
+    private static final APIInfo DELETE_ENTITY_BY_GUID = new APIInfo(ENTITY_API + "guid/", HttpMethod.DELETE, Response.Status.OK);
+    private static final APIInfo GET_CLASSIFICATIONS = new APIInfo(ENTITY_API + "guid/%s/classifications", HttpMethod.GET, Response.Status.OK);
+    private static final APIInfo ADD_CLASSIFICATIONS = new APIInfo(ENTITY_API + "guid/%s/classifications", HttpMethod.POST, Response.Status.OK);
+    private static final APIInfo UPDATE_CLASSIFICATIONS = new APIInfo(ENTITY_API + "guid/%s/classifications", HttpMethod.PUT, Response.Status.OK);
+    private static final APIInfo DELETE_CLASSIFICATION = new APIInfo(ENTITY_API + "guid/%s/classification/%s", HttpMethod.DELETE, Response.Status.OK);
+
+    private static final APIInfo GET_ENTITIES = new APIInfo(ENTITIES_API + "guids/", HttpMethod.GET, Response.Status.OK);
+    private static final APIInfo CREATE_ENTITIES = new APIInfo(ENTITIES_API, HttpMethod.POST, Response.Status.OK);
+    private static final APIInfo UPDATE_ENTITIES = CREATE_ENTITIES;
+    private static final APIInfo DELETE_ENTITIES = new APIInfo(ENTITIES_API + "guids/", HttpMethod.GET, Response.Status.OK);
+    private static final APIInfo SEARCH_ENTITIES = new APIInfo(ENTITIES_API, HttpMethod.GET, Response.Status.OK);
+
+    public AtlasEntitiesClientV2(String[] baseUrl, String[] basicAuthUserNamePassword) {
+        super(baseUrl, basicAuthUserNamePassword);
+    }
+
+    public AtlasEntitiesClientV2(String... baseUrls) throws AtlasException {
+        super(baseUrls);
+    }
+
+    public AtlasEntitiesClientV2(UserGroupInformation ugi, String doAsUser, String... baseUrls) {
+        super(ugi, doAsUser, baseUrls);
+    }
+
+    protected AtlasEntitiesClientV2() {
+        super();
+    }
+
+    @VisibleForTesting
+    AtlasEntitiesClientV2(WebResource service, Configuration configuration) {
+        super(service, configuration);
+    }
+
+    public AtlasEntity getEntityByGuid(String guid) throws AtlasServiceException {
+        return callAPI(GET_ENTITY_BY_GUID, null, AtlasEntity.class, guid);
+    }
+
+    public EntityMutationResponse createEntity(AtlasEntity atlasEntity) throws AtlasServiceException {
+        return callAPI(CREATE_ENTITY, atlasEntity, EntityMutationResponse.class);
+    }
+
+    public EntityMutationResponse updateEntity(AtlasEntity atlasEntity) throws AtlasServiceException {
+        return callAPI(UPDATE_ENTITY, atlasEntity, EntityMutationResponse.class);
+    }
+
+    public EntityMutationResponse updateEntity(String guid, AtlasEntity atlasEntity) throws AtlasServiceException {
+        return callAPI(UPDATE_ENTITY, atlasEntity, EntityMutationResponse.class, guid);
+    }
+
+    public AtlasEntity deleteEntityByGuid(String guid) throws AtlasServiceException {
+        return callAPI(DELETE_ENTITY_BY_GUID, null, AtlasEntity.class, guid);
+    }
+
+    public AtlasClassifications getClassifications(String guid) throws AtlasServiceException {
+        return callAPI(formatPath(GET_CLASSIFICATIONS, guid), null, AtlasClassifications.class);
+    }
+
+    public void addClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException {
+        callAPI(formatPath(ADD_CLASSIFICATIONS, guid), classifications, AtlasClassifications.class);
+    }
+
+    public void updateClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException {
+        callAPI(formatPath(UPDATE_CLASSIFICATIONS, guid), classifications, AtlasClassifications.class);
+    }
+
+    public void deleteClassifications(String guid, List<AtlasClassification> classifications) throws AtlasServiceException {
+        callAPI(formatPath(GET_CLASSIFICATIONS, guid), classifications, AtlasClassifications.class);
+    }
+
+    public void deleteClassification(String guid, String classificationName) throws AtlasServiceException {
+        callAPI(formatPath(DELETE_CLASSIFICATION, guid, classificationName), null, AtlasClassifications.class);
+    }
+
+    // Entities operations
+    public List<AtlasEntity> getEntities(List<String> entityIds) {
+        // TODO Map the query params correctly
+        return null;
+    }
+
+    public List<AtlasEntity> createEntities(List<AtlasEntity> atlasEntities) throws AtlasServiceException {
+        return (List<AtlasEntity>)callAPI(CREATE_ENTITIES, atlasEntities, List.class);
+    }
+
+    public List<AtlasEntity> updateEntities(List<AtlasEntity> atlasEntities) throws AtlasServiceException {
+        return (List<AtlasEntity>)callAPI(UPDATE_ENTITIES, atlasEntities, List.class);
+    }
+
+    public AtlasEntity.AtlasEntities searchEntities(SearchFilter searchFilter) throws AtlasServiceException {
+        return callAPI(GET_ENTITIES, null, AtlasEntity.AtlasEntities.class, searchFilter.getParams());
+    }
+}

client/src/main/java/org/apache/atlas/AtlasServiceException.java

@@ -31,21 +31,40 @@ public class AtlasServiceException extends Exception {
         super("Metadata service API " + api + " failed", e);
     }
 
+    public AtlasServiceException(AtlasBaseClient.APIInfo api, Exception e) {
+        super("Metadata service API " + api + " failed", e);
+    }
+
     public AtlasServiceException(AtlasClient.API api, WebApplicationException e) throws JSONException {
         this(api, ClientResponse.Status.fromStatusCode(e.getResponse().getStatus()),
                 ((JSONObject) e.getResponse().getEntity()).getString("stackTrace"));
     }
 
+    public AtlasServiceException(AtlasBaseClient.APIInfo api, WebApplicationException e) throws JSONException {
+        this(api, ClientResponse.Status.fromStatusCode(e.getResponse().getStatus()),
+                ((JSONObject) e.getResponse().getEntity()).getString("stackTrace"));
+    }
+
     private AtlasServiceException(AtlasClient.API api, ClientResponse.Status status, String response) {
         super("Metadata service API " + api + " failed with status " + (status != null ? status.getStatusCode() : -1)
                 + " (" + status + ") Response Body (" + response + ")");
         this.status = status;
     }
 
+    private AtlasServiceException(AtlasBaseClient.APIInfo api, ClientResponse.Status status, String response) {
+        super("Metadata service API " + api + " failed with status " + (status != null ? status.getStatusCode() : -1)
+                + " (" + status + ") Response Body (" + response + ")");
+        this.status = status;
+    }
+
     public AtlasServiceException(AtlasClient.API api, ClientResponse response) {
         this(api, ClientResponse.Status.fromStatusCode(response.getStatus()), response.getEntity(String.class));
     }
 
+    public AtlasServiceException(AtlasBaseClient.APIInfo api, ClientResponse response) {
+        this(api, ClientResponse.Status.fromStatusCode(response.getStatus()), response.getEntity(String.class));
+    }
+
     public AtlasServiceException(Exception e) {
         super(e);
     }

client/src/main/java/org/apache/atlas/AtlasTypedefClientV2.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.sun.jersey.api.client.WebResource;
+
+import org.apache.atlas.model.SearchFilter;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.type.AtlasType;
+import org.apache.commons.configuration.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+
+public class AtlasTypedefClientV2 extends AtlasBaseClient {
+
+    private static final String BASE_URI = "api/atlas/v2/types/";
+    private static final String TYPEDEFS_PATH = BASE_URI + "typedefs/";
+    private static final String GET_BY_NAME_TEMPLATE = BASE_URI + "%s/name/%s";
+    private static final String GET_BY_GUID_TEMPLATE = BASE_URI + "%s/guid/%s";
+
+    private static final APIInfo GET_ALL_TYPE_DEFS = new APIInfo(TYPEDEFS_PATH, HttpMethod.GET, Response.Status.OK);
+    private static final APIInfo CREATE_ALL_TYPE_DEFS = new APIInfo(TYPEDEFS_PATH, HttpMethod.POST, Response.Status.OK);
+    private static final APIInfo UPDATE_ALL_TYPE_DEFS = new APIInfo(TYPEDEFS_PATH, HttpMethod.PUT, Response.Status.OK);
+    private static final APIInfo DELETE_ALL_TYPE_DEFS = new APIInfo(TYPEDEFS_PATH, HttpMethod.DELETE, Response.Status.OK);
+
+    public AtlasTypedefClientV2(String[] baseUrl, String[] basicAuthUserNamePassword) {
+        super(baseUrl, basicAuthUserNamePassword);
+    }
+
+    public AtlasTypedefClientV2(String... baseUrls) throws AtlasException {
+        super(baseUrls);
+    }
+
+    public AtlasTypedefClientV2(UserGroupInformation ugi, String doAsUser, String... baseUrls) {
+        super(ugi, doAsUser, baseUrls);
+    }
+
+    protected AtlasTypedefClientV2() {
+        super();
+    }
+
+    @VisibleForTesting
+    AtlasTypedefClientV2(WebResource service, Configuration configuration) {
+        super(service, configuration);
+    }
+
+    /**
+     * Bulk retrieval API for retrieving all type definitions in Atlas
+     *
+     * @return A composite wrapper object with lists of all type definitions
+     */
+    public AtlasTypesDef getAllTypeDefs(SearchFilter searchFilter) throws AtlasServiceException {
+        return callAPI(GET_ALL_TYPE_DEFS, null, AtlasTypesDef.class, searchFilter.getParams());
+    }
+
+    public AtlasEnumDef getEnumByName(final String name) throws AtlasServiceException {
+        return getTypeDefByName(name, AtlasEnumDef.class);
+    }
+
+    public AtlasEnumDef getEnumByGuid(final String guid) throws AtlasServiceException {
+        return getTypeDefByGuid(guid, AtlasEnumDef.class);
+    }
+
+    public AtlasStructDef getStructByName(final String name) throws AtlasServiceException {
+        return getTypeDefByName(name, AtlasStructDef.class);
+    }
+
+    public AtlasStructDef getStructByGuid(final String guid) throws AtlasServiceException {
+        return getTypeDefByGuid(guid, AtlasStructDef.class);
+    }
+
+    public AtlasClassificationDef getClassificationByName(final String name) throws AtlasServiceException {
+        return getTypeDefByName(name, AtlasClassificationDef.class);
+    }
+
+    public AtlasClassificationDef getClassificationByGuid(final String guid) throws AtlasServiceException {
+        return getTypeDefByGuid(guid, AtlasClassificationDef.class);
+    }
+
+    public AtlasEntityDef getEntityByName(final String name) throws AtlasServiceException {
+        return getTypeDefByName(name, AtlasEntityDef.class);
+    }
+
+    public AtlasEntityDef getEntityByGuid(final String guid) throws AtlasServiceException {
+        return getTypeDefByGuid(guid, AtlasEntityDef.class);
+    }
+
+    /**
+     * Bulk create APIs for all atlas type definitions, only new definitions will be created.
+     * Any changes to the existing definitions will be discarded
+     *
+     * @param typesDef A composite wrapper object with corresponding lists of the type definition
+     * @return A composite wrapper object with lists of type definitions that were successfully
+     * created
+     */
+    public AtlasTypesDef createAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasServiceException {
+        return callAPI(CREATE_ALL_TYPE_DEFS, AtlasType.toJson(typesDef), AtlasTypesDef.class);
+    }
+
+    /**
+     * Bulk update API for all types, changes detected in the type definitions would be persisted
+     *
+     * @param typesDef A composite object that captures all type definition changes
+     * @return A composite object with lists of type definitions that were updated
+     */
+    public AtlasTypesDef updateAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasServiceException {
+        return callAPI(UPDATE_ALL_TYPE_DEFS, AtlasType.toJson(typesDef), AtlasTypesDef.class);
+    }
+
+    /**
+     * Bulk delete API for all types
+     *
+     * @param typesDef A composite object that captures all types to be deleted
+     */
+    public void deleteAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasServiceException {
+        callAPI(DELETE_ALL_TYPE_DEFS, AtlasType.toJson(typesDef), AtlasTypesDef.class);
+    }
+
+    private <T> T getTypeDefByName(final String name, Class<T> typeDefClass) throws AtlasServiceException {
+        String atlasPath = getAtlasPath(typeDefClass);
+        APIInfo apiInfo = new APIInfo(String.format(GET_BY_NAME_TEMPLATE, atlasPath, name), HttpMethod.GET, Response.Status.OK);
+        return callAPI(apiInfo, null, typeDefClass);
+    }
+
+    private <T> T getTypeDefByGuid(final String guid, Class<T> typeDefClass) throws AtlasServiceException {
+        String atlasPath = getAtlasPath(typeDefClass);
+        APIInfo apiInfo = new APIInfo(String.format(GET_BY_GUID_TEMPLATE, atlasPath, guid), HttpMethod.GET, Response.Status.OK);
+        return callAPI(apiInfo, null, typeDefClass);
+    }
+
+    private <T> String getAtlasPath(Class<T> typeDefClass) {
+        if (typeDefClass.isAssignableFrom(AtlasEnumDef.class)) {
+            return "enumdef";
+        } else if (typeDefClass.isAssignableFrom(AtlasEntityDef.class)) {
+            return "entitydef";
+        } else if (typeDefClass.isAssignableFrom(AtlasClassificationDef.class)) {
+            return "classificationdef";
+        } else if (typeDefClass.isAssignableFrom(AtlasStructDef.class)) {
+            return "structdef";
+        }
+        // Code should never reach this pion
+        return "";
+    }
+}

webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java

@@ -19,6 +19,7 @@
 package org.apache.atlas.web.resources;
 
 import com.sun.jersey.api.client.ClientResponse;
+
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.services.MetadataService;

webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java

@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
+
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasServiceException;
 import org.apache.atlas.typesystem.TypesDef;

webapp/src/test/java/org/apache/atlas/web/security/NegativeSSLAndKerberosTest.java

@@ -20,7 +20,6 @@ package org.apache.atlas.web.security;
 
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
 import org.apache.atlas.web.TestUtils;
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.PropertiesConfiguration;
@@ -61,7 +60,7 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
         // client will actually only leverage subset of these properties
         final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
 
-        persistSSLClientConfiguration((org.apache.commons.configuration.Configuration) configuration);
+        persistSSLClientConfiguration(configuration);
 
         TestUtils.writeConfiguration(configuration, persistDir + File.separator +
             ApplicationProperties.APPLICATION_PROPERTIES);
@@ -103,12 +102,7 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
         originalConf = System.getProperty("atlas.conf");
         System.setProperty("atlas.conf", persistDir);
 
-        dgiClient = new AtlasClient(DGI_URL) {
-            @Override
-            protected PropertiesConfiguration getClientProperties() {
-                return configuration;
-            }
-        };
+        dgiClient = new AtlasClient(configuration, DGI_URL);
 
 
         secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {

webapp/src/test/java/org/apache/atlas/web/security/SSLAndKerberosTest.java

@@ -20,8 +20,8 @@ package org.apache.atlas.web.security;
 
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
 import org.apache.atlas.web.TestUtils;
+import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -30,6 +30,12 @@ import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.security.PrivilegedExceptionAction;
+
 import javax.security.auth.Subject;
 import javax.security.auth.callback.Callback;
 import javax.security.auth.callback.CallbackHandler;
@@ -38,11 +44,6 @@ import javax.security.auth.callback.PasswordCallback;
 import javax.security.auth.callback.UnsupportedCallbackException;
 import javax.security.auth.login.LoginContext;
 import javax.security.auth.login.LoginException;
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.security.PrivilegedExceptionAction;
 
 import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
 
@@ -120,12 +121,7 @@ public class SSLAndKerberosTest extends BaseSSLAndKerberosTest {
         dgiCLient = proxyUser.doAs(new PrivilegedExceptionAction<AtlasClient>() {
             @Override
             public AtlasClient run() throws Exception {
-                return new AtlasClient(DGI_URL) {
-                    @Override
-                    protected PropertiesConfiguration getClientProperties() {
-                        return configuration;
-                    }
-                };
+                return new AtlasClient(configuration, DGI_URL);
             }
         });
 

webapp/src/test/java/org/apache/atlas/web/security/SSLTest.java

@@ -74,21 +74,16 @@ public class SSLTest extends BaseSSLAndKerberosTest {
         setupCredentials();
         final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
         String persistDir = writeConfiguration(configuration);
-        persistSSLClientConfiguration((org.apache.commons.configuration.Configuration) configuration);
+        persistSSLClientConfiguration(configuration);
 
         originalConf = System.getProperty("atlas.conf");
         System.setProperty("atlas.conf", persistDir);
 
-        atlasClient = new AtlasClient(new String[]{DGI_URL},new String[]{"admin","admin"}) {
-            @Override
-            protected PropertiesConfiguration getClientProperties() {
-                return configuration;
-            }
-        };
+        atlasClient = new AtlasClient(configuration, new String[]{DGI_URL},new String[]{"admin","admin"});
 
         secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
             @Override
-            public PropertiesConfiguration getConfiguration() {
+            public org.apache.commons.configuration.Configuration getConfiguration() {
                 return configuration;
             }
         };