Fix tests to rid using TypeSystem in client

client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java

@@ -22,6 +22,7 @@ import com.sun.jersey.api.client.Client;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
+import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 
@@ -29,7 +30,12 @@ import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
+import java.util.ArrayList;
+import java.util.List;
 
+/**
+ * Client for metadata.
+ */
 public class MetadataServiceClient {
     public static final String REQUEST_ID = "requestId";
     public static final String RESULTS = "results";
@@ -50,8 +56,8 @@ public class MetadataServiceClient {
         //Type operations
         CREATE_TYPE("api/metadata/types/submit", HttpMethod.POST),
         GET_TYPE("api/metadata/types/definition", HttpMethod.GET),
-        LIST_TYPE("api/metadata/types/list", HttpMethod.GET),
-        LIST_TRAIT_TYPE("api/metadata/types/traits/list", HttpMethod.GET),
+        LIST_TYPES("api/metadata/types/list", HttpMethod.GET),
+        LIST_TRAIT_TYPES("api/metadata/types/traits/list", HttpMethod.GET),
 
         //Entity operations
         CREATE_ENTITY("api/metadata/entities/submit", HttpMethod.POST),
@@ -81,8 +87,28 @@ public class MetadataServiceClient {
         }
     }
 
-    public JSONObject createEntity(String typeName, String entityAsJson) throws MetadataServiceException {
-        return callAPI(API.CREATE_ENTITY, entityAsJson, typeName);
+    public JSONObject createType(String typeName,
+                                 String typeAsJson) throws MetadataServiceException {
+        return callAPI(API.CREATE_TYPE, typeAsJson, typeName);
+    }
+
+    public List<String> listTypes() throws MetadataServiceException {
+        try {
+            final JSONObject jsonObject = callAPI(API.LIST_TYPES, null);
+            final JSONArray list = jsonObject.getJSONArray(MetadataServiceClient.RESULTS);
+            ArrayList<String> types = new ArrayList<>();
+            for (int index = 0; index < list.length(); index++) {
+                types.add(list.getString(index));
+            }
+
+            return types;
+        } catch (JSONException e) {
+            throw new MetadataServiceException(API.LIST_TYPES, e);
+        }
+    }
+
+    public JSONObject createEntity(String entityAsJson) throws MetadataServiceException {
+        return callAPI(API.CREATE_ENTITY, entityAsJson);
     }
 
     public String getRequestId(JSONObject json) throws MetadataServiceException {
@@ -101,7 +127,9 @@ public class MetadataServiceClient {
             }
         }
 
-        ClientResponse clientResponse = resource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON)
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
                 .method(api.getMethod(), ClientResponse.class, requestObject);
 
         if (clientResponse.getStatus() == Response.Status.OK.getStatusCode()) {

repository/src/main/java/org/apache/hadoop/metadata/listener/EntityChangeListener.java

@@ -29,12 +29,10 @@ public interface EntityChangeListener {
     /**
      * This is upon adding a new typed instance to the repository.
      *
-     * @param typeName type name
      * @param typedInstance a typed instance
      * @throws org.apache.hadoop.metadata.MetadataException
      */
-    void onEntityAdded(String typeName,
-                       ITypedReferenceableInstance typedInstance) throws MetadataException;
+    void onEntityAdded(ITypedReferenceableInstance typedInstance) throws MetadataException;
 
     /**
      * This is upon adding a new trait to a typed instance.

repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java

@@ -69,12 +69,10 @@ public interface MetadataRepository {
      * Creates an entity definition (instance) corresponding to a given type.
      *
      * @param entity     entity (typed instance)
-     * @param entityType entity type name
      * @return a globally unique identifier
      * @throws RepositoryException
      */
-    String createEntity(IReferenceableInstance entity,
-                        String entityType) throws RepositoryException;
+    String createEntity(IReferenceableInstance entity) throws RepositoryException;
 
     /**
      * Fetch the complete definition of an entity given its GUID.
@@ -137,17 +135,6 @@ public interface MetadataRepository {
                   ITypedStruct traitInstance) throws RepositoryException;
 
     /**
-     * Adds a list of traits to an existing entity represented by a guid.
-     *
-     * @param guid           globally unique identifier for the entity
-     * @param traitInstances list of trait instances that needs to be added to entity
-     * @return an entity instance with updated traits
-     * @throws RepositoryException
-     */
-    // ITypedReferenceableInstance addTraits(String guid, Map<String, ITypedStruct> traitInstances)
-    //    throws RepositoryException;
-
-    /**
      * Deletes a given trait from an existing entity represented by a guid.
      *
      * @param guid                 globally unique identifier for the entity
@@ -160,8 +147,8 @@ public interface MetadataRepository {
     /**
      * Adds the property to the entity that corresponds to the GUID
      * @param guid entity id
-     * @param property
-     * @param value
+     * @param property property name
+     * @param value    property value
      */
     void updateEntity(String guid, String property, String value) throws RepositoryException;
 }

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java

@@ -116,9 +116,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
     }
 
     @Override
-    public String createEntity(IReferenceableInstance typedInstance,
-                               String typeName) throws RepositoryException {
-        LOG.info("adding entity={} type={}", typedInstance, typeName);
+    public String createEntity(IReferenceableInstance typedInstance) throws RepositoryException {
+        LOG.info("adding entity={}", typedInstance);
 
         try {
             titanGraph.rollback();

repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java

@@ -28,10 +28,16 @@ import org.apache.hadoop.metadata.repository.MetadataRepository;
 import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
 import org.apache.hadoop.metadata.typesystem.ITypedStruct;
+import org.apache.hadoop.metadata.typesystem.Referenceable;
+import org.apache.hadoop.metadata.typesystem.Struct;
 import org.apache.hadoop.metadata.typesystem.TypesDef;
+import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
 import org.apache.hadoop.metadata.typesystem.json.Serialization$;
 import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
+import org.apache.hadoop.metadata.typesystem.types.ClassType;
 import org.apache.hadoop.metadata.typesystem.types.IDataType;
+import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
+import org.apache.hadoop.metadata.typesystem.types.TraitType;
 import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -151,24 +157,39 @@ public class DefaultMetadataService implements MetadataService {
     /**
      * Creates an entity, instance of the type.
      *
-     * @param entityType       type
-     * @param entityDefinition definition
+     * @param entityInstanceDefinition definition
      * @return guid
      */
     @Override
-    public String createEntity(String entityType,
-                               String entityDefinition) throws MetadataException {
-        Preconditions.checkNotNull(entityDefinition, "entity cannot be null");
-        Preconditions.checkNotNull(entityType, "entity type cannot be null");
+    public String createEntity(String entityInstanceDefinition) throws MetadataException {
+        Preconditions.checkNotNull(entityInstanceDefinition,
+                "entity instance definition cannot be null");
+
+        ITypedReferenceableInstance entityTypedInstance =
+                deserializeClassInstance(entityInstanceDefinition);
 
-        ITypedReferenceableInstance entityInstance =
-                Serialization$.MODULE$.fromJson(entityDefinition);
-        final String guid = repository.createEntity(entityInstance, entityType);
+        final String guid = repository.createEntity(entityTypedInstance);
 
-        onEntityAddedToRepo(entityType, entityInstance);
+        onEntityAddedToRepo(entityTypedInstance);
         return guid;
     }
 
+    private ITypedReferenceableInstance deserializeClassInstance(
+            String entityInstanceDefinition) throws MetadataException {
+
+        try {
+            final Referenceable entityInstance = InstanceSerialization.fromJsonReferenceable(
+                    entityInstanceDefinition, true);
+            final String entityTypeName = entityInstance.getTypeName();
+            Preconditions.checkNotNull(entityTypeName, "entity type cannot be null");
+
+            ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName);
+            return entityType.convert(entityInstance, Multiplicity.REQUIRED);
+        } catch (Exception e) {
+            throw new MetadataException("Error deserializing trait instance");
+        }
+    }
+
     /**
      * Return the definition for the given guid.
      *
@@ -256,7 +277,14 @@ public class DefaultMetadataService implements MetadataService {
         throws MetadataException {
 
         try {
-            return (ITypedStruct) Serialization$.MODULE$.traitFromJson(traitInstanceDefinition);
+            Struct traitInstance = InstanceSerialization.fromJsonStruct(
+                    traitInstanceDefinition, true);
+            final String entityTypeName = traitInstance.getTypeName();
+            Preconditions.checkNotNull(entityTypeName, "entity type cannot be null");
+
+            TraitType traitType = typeSystem.getDataType(TraitType.class, entityTypeName);
+            return traitType.convert(
+                    traitInstance, Multiplicity.REQUIRED);
         } catch (Exception e) {
             throw new MetadataException("Error deserializing trait instance");
         }
@@ -301,12 +329,11 @@ public class DefaultMetadataService implements MetadataService {
         typesChangeListeners.remove(listener);
     }
 
-    private void onEntityAddedToRepo(String typeName,
-                                     ITypedReferenceableInstance typedInstance)
+    private void onEntityAddedToRepo(ITypedReferenceableInstance typedInstance)
         throws MetadataException {
 
         for (EntityChangeListener listener : entityChangeListeners) {
-            listener.onEntityAdded(typeName, typedInstance);
+            listener.onEntityAdded(typedInstance);
         }
     }
 

repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java

@@ -64,11 +64,10 @@ public interface MetadataService {
     /**
      * Creates an entity, instance of the type.
      *
-     * @param entityType type
      * @param entityDefinition definition
      * @return guid
      */
-    String createEntity(String entityType, String entityDefinition) throws MetadataException;
+    String createEntity(String entityDefinition) throws MetadataException;
 
     /**
      * Return the definition for the given guid.
@@ -87,10 +86,11 @@ public interface MetadataService {
     List<String> getEntityList(String entityType) throws MetadataException;
 
     /**
-     * Adds the property to the given entity id(guid)
+     * Adds the property to the given entity id(guid).
+     *
      * @param guid entity id
-     * @param property
-     * @param value
+     * @param property property name
+     * @param value    property value
      */
     void updateEntity(String guid, String property, String value) throws MetadataException;
 

repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java

@@ -78,7 +78,7 @@ public class GraphBackedDiscoveryServiceTest {
         ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
         ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
 
-        repositoryService.createEntity(hrDept2, "Department");
+        repositoryService.createEntity(hrDept2);
     }
 
     private void setupSampleData() throws ScriptException {
@@ -173,12 +173,10 @@ public class GraphBackedDiscoveryServiceTest {
                 {"DB, Table"},
                 /*{"DB as db1 Table where db1.name = \"Reporting\""},*/
                 {"DB name = \"Reporting\""},
-/*
-                {"Column where is PII"},
-                {"Table where is Dimension"},
-                {"View where is Dimension"},
-                {"Column where is PII select Column.name"},
-*/
+                {"Column where Column isa PII"},
+                {"Table is Dimension"},
+                {"View is Dimension"},
+                /*{"Column where Column isa PII select Column.name"},*/
                 {"Column select Column.name"},
                 {"from Table select Table.name"},
         };

repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java

@@ -106,7 +106,7 @@ public class GraphBackedMetadataRepositoryTest {
         ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
         ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
 
-        guid = repositoryService.createEntity(hrDept2, ENTITY_TYPE);
+        guid = repositoryService.createEntity(hrDept2);
         Assert.assertNotNull(guid);
     }
 
@@ -154,14 +154,14 @@ public class GraphBackedMetadataRepositoryTest {
         ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
         System.out.println("db = " + db);
 
-        String dbGUID = repositoryService.createEntity(db, DATABASE_TYPE);
+        String dbGUID = repositoryService.createEntity(db);
         System.out.println("added db = " + dbGUID);
 
         Referenceable dbInstance = new Referenceable(
                 dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
 
         ITypedReferenceableInstance table = createHiveTableInstance(dbInstance);
-        String tableGUID = repositoryService.createEntity(table, TABLE_TYPE);
+        String tableGUID = repositoryService.createEntity(table);
         System.out.println("added table = " + tableGUID);
     }
 

repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperScaleTest.java

@@ -89,14 +89,14 @@ public class GraphRepoMapperScaleTest {
         ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
         ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
 
-        dbGUID = repositoryService.createEntity(db, DATABASE_TYPE);
+        dbGUID = repositoryService.createEntity(db);
 
         Referenceable dbInstance = new Referenceable(
                 dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
 
         for (int index = 0; index < 1000; index++) {
             ITypedReferenceableInstance table = createHiveTableInstance(dbInstance, index);
-            repositoryService.createEntity(table, TABLE_TYPE);
+            repositoryService.createEntity(table);
         }
     }
 

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java

@@ -78,31 +78,28 @@ public class EntityResource {
 
     /**
      * Submits an entity definition (instance) corresponding to a given type.
-     *
-     * @param typeName name of a type which is unique.
      */
     @POST
-    @Path("submit/{typeName}")
+    @Path("submit")
     @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
-    public Response submit(@Context HttpServletRequest request,
-                           @PathParam("typeName") final String typeName) {
+    public Response submit(@Context HttpServletRequest request) {
         try {
             final String entity = Servlets.getRequestPayload(request);
             LOG.debug("submitting entity {} ", entity);
 
-            final String guid = metadataService.createEntity(typeName, entity);
+            final String guid = metadataService.createEntity(entity);
             JSONObject response = new JSONObject();
             response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
             response.put(MetadataServiceClient.RESULTS, guid);
 
             return Response.ok(response).build();
         } catch (MetadataException | IOException | IllegalArgumentException e) {
-            LOG.error("Unable to persist instance for type {}", typeName, e);
+            LOG.error("Unable to persist entity instance", e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
         } catch (JSONException e) {
-            LOG.error("Unable to persist instance for type {}", typeName, e);
+            LOG.error("Unable to persist entity instance", e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
         }
@@ -193,12 +190,13 @@ public class EntityResource {
      * @param guid entity id
      * @param property property to add
      * @param value property's value
-     * @return
+     * @return response payload as json
      */
     @PUT
     @Path("update/{guid}")
     @Produces(MediaType.APPLICATION_JSON)
-    public Response update(@PathParam("guid") String guid, @QueryParam("property") String property,
+    public Response update(@PathParam("guid") String guid,
+                           @QueryParam("property") String property,
                            @QueryParam("value") String value) {
         try {
             metadataService.updateEntity(guid, property, value);

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java

@@ -23,27 +23,28 @@ import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
 import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.web.util.Servlets;
+import org.apache.hadoop.metadata.typesystem.Referenceable;
+import org.apache.hadoop.metadata.typesystem.TypesDef;
+import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
+import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.codehaus.jettison.json.JSONObject;
 import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
 
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
 
+/**
+ * Base class for integration tests.
+ * Sets up the web resource and has helper methods to create type and entity.
+ */
 public abstract class BaseResourceIT {
 
-    protected TypeSystem typeSystem;
     protected WebResource service;
     protected MetadataServiceClient serviceClient;
 
     public void setUp() throws Exception {
-        typeSystem = TypeSystem.getInstance();
-        typeSystem.reset();
-
         String baseUrl = "http://localhost:21000/";
 
         DefaultClientConfig config = new DefaultClientConfig();
@@ -54,10 +55,15 @@ public abstract class BaseResourceIT {
         serviceClient = new MetadataServiceClient(baseUrl);
     }
 
-    protected void sumbitType(String typesAsJSON, String type) throws Exception {
+    protected void createType(TypesDef typesDef) throws Exception {
+        String typesAsJSON = TypesSerialization.toJson(typesDef);
+        createType(typesAsJSON, "removeme");
+    }
+
+    protected void createType(String typesAsJSON, String typeName) throws Exception {
         WebResource resource = service
                 .path("api/metadata/types/submit")
-                .path(type);
+                .path(typeName);
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
@@ -69,8 +75,22 @@ public abstract class BaseResourceIT {
         Assert.assertNotNull(responseAsString);
 
         JSONObject response = new JSONObject(responseAsString);
-        Assert.assertEquals(response.get("typeName"), type);
+        Assert.assertEquals(response.get("typeName"), typeName);
         Assert.assertNotNull(response.get("types"));
         Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
     }
+
+    protected Referenceable createInstance(Referenceable referenceable) throws Exception {
+        String typeName = referenceable.getTypeName();
+        System.out.println("creating instance of type " + typeName);
+
+        String entityJSON = InstanceSerialization.toJson(referenceable, true);
+        System.out.println("Submitting new entity= " + entityJSON);
+        JSONObject jsonObject = serviceClient.createEntity(entityJSON);
+        String guid = jsonObject.getString(MetadataServiceClient.RESULTS);
+        System.out.println("created instance for type " + typeName + ", guid: " + guid);
+
+        // return the reference to created instance with guid
+        return new Referenceable(guid, referenceable.getTypeName(), referenceable.getValuesMap());
+    }
 }

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java

@@ -31,7 +31,6 @@ import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
 import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
 import org.apache.hadoop.metadata.typesystem.types.TraitType;
 import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.apache.hadoop.metadata.web.util.Servlets;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
 import org.testng.Assert;
@@ -187,7 +186,7 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
             HierarchicalTypeDefinition<TraitType> traitTypeDef =
                     TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
             String json = TypesSerialization$.MODULE$.toJson(traitTypeDef, true);
-            sumbitType(json, traitName);
+            createType(json, traitName);
         }
 
         return traitNames;