BUG-32815 BUG-32816 BUG-32827 Entity Trait Management Backend API. Contributed…

repository/src/main/java/org/apache/hadoop/metadata/listener/EntityChangeListener.java

@@ -33,7 +33,24 @@ public interface EntityChangeListener {
      * @param typedInstance a typed instance
      * @throws org.apache.hadoop.metadata.MetadataException
      */
-    void onAdd(String typeName,
-               ITypedReferenceableInstance typedInstance) throws MetadataException;
+    void onEntityAdded(String typeName,
+                       ITypedReferenceableInstance typedInstance) throws MetadataException;
 
+    /**
+     * This is upon adding a new trait to a typed instance.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be added to entity
+     * @throws org.apache.hadoop.metadata.MetadataException
+     */
+    void onTraitAdded(String guid, String traitName) throws MetadataException;
+
+    /**
+     * This is upon deleting a trait from a typed instance.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be deleted from entity
+     * @throws org.apache.hadoop.metadata.MetadataException
+     */
+    void onTraitDeleted(String guid, String traitName) throws MetadataException;
 }

repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java

@@ -20,6 +20,9 @@ package org.apache.hadoop.metadata.repository;
 
 import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.typesystem.ITypedStruct;
+import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
+import org.apache.hadoop.metadata.typesystem.types.IDataType;
 
 import java.util.List;
 
@@ -28,12 +31,130 @@ import java.util.List;
  */
 public interface MetadataRepository {
 
+    /**
+     * Returns the property key used to store entity type name.
+     *
+     * @return property key used to store entity type name.
+     */
     String getTypeAttributeName();
 
+    /**
+     * Return the property key used to store a given traitName in the repository.
+     *
+     * @param dataType  data type
+     * @param traitName trait name
+     * @return property key used to store a given traitName
+     */
+    String getTraitLabel(IDataType<?> dataType, String traitName);
+
+    /**
+     * Return the property key used to store a given attribute in the repository.
+     *
+     * @param dataType data type
+     * @param aInfo    attribute info
+     * @return property key used to store a given attribute
+     */
+    String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo);
+
+    /**
+     * Return the edge label for a given attribute in the repository.
+     *
+     * @param dataType  data type
+     * @param aInfo    attribute info
+     * @return edge label for a given attribute
+     */
+    String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo);
+
+    /**
+     * Creates an entity definition (instance) corresponding to a given type.
+     *
+     * @param entity     entity (typed instance)
+     * @param entityType entity type name
+     * @return a globally unique identifier
+     * @throws RepositoryException
+     */
     String createEntity(IReferenceableInstance entity,
                         String entityType) throws RepositoryException;
 
+    /**
+     * Fetch the complete definition of an entity given its GUID.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return entity (typed instance) definition
+     * @throws RepositoryException
+     */
     ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException;
 
+    /**
+     * Gets the list of entities for a given entity type.
+     *
+     * @param entityType name of a type which is unique
+     * @return a list of entity names for the given type
+     * @throws RepositoryException
+     */
     List<String> getEntityList(String entityType) throws RepositoryException;
+
+    /**
+     * Deletes an entity definition (instance) corresponding to a given type.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return true if deleted else false
+     * @throws RepositoryException
+     */
+    // boolean deleteEntity(String guid) throws RepositoryException;
+
+    /**
+     * Updates an entity given its GUID with the attribute name and value.
+     *
+     * @param guid           globally unique identifier for the entity
+     * @param attributeName  name of the attribute
+     * @param attributeValue value of the attribute
+     * @return an entity instance with updated state
+     * @throws RepositoryException
+     */
+    //ITypedReferenceableInstance updateEntity(String guid, String attributeName,
+    //                                         String attributeValue) throws RepositoryException;
+
+
+    // Trait management functions
+    /**
+     * Gets the list of trait names for a given entity represented by a guid.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return a list of trait names for the given entity guid
+     * @throws RepositoryException
+     */
+    List<String> getTraitNames(String guid) throws RepositoryException;
+
+    /**
+     * Adds a new trait to an existing entity represented by a guid.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be added to entity
+     * @param traitInstance trait instance that needs to be added to entity
+     * @throws RepositoryException
+     */
+    void addTrait(String guid, String traitName,
+                  ITypedStruct traitInstance) throws RepositoryException;
+
+    /**
+     * Adds a list of traits to an existing entity represented by a guid.
+     *
+     * @param guid           globally unique identifier for the entity
+     * @param traitInstances list of trait instances that needs to be added to entity
+     * @return an entity instance with updated traits
+     * @throws RepositoryException
+     */
+    // ITypedReferenceableInstance addTraits(String guid, Map<String, ITypedStruct> traitInstances)
+    //    throws RepositoryException;
+
+    /**
+     * Deletes a given trait from an existing entity represented by a guid.
+     *
+     * @param guid                 globally unique identifier for the entity
+     * @param traitNameToBeDeleted name of the trait
+     * @throws RepositoryException
+     */
+    void deleteTrait(String guid,
+                     String traitNameToBeDeleted) throws RepositoryException;
 }

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java

@@ -33,14 +33,10 @@ public final class Constants {
     public static final String ENTITY_TYPE_INDEX = "type_index";
 
     /**
-     * Data type property key.
-     */
-    // public static final String DATA_TYPE_PROPERTY_KEY = "dataType";
-
-    /**
-     * Trait names property key.
+     * Trait names property key and index name.
      */
     public static final String TRAIT_NAMES_PROPERTY_KEY = "traitNames";
+    public static final String TRAIT_NAMES_INDEX = "trait_names_index";
 
     public static final String VERSION_PROPERTY_KEY = "version";
     public static final String TIMESTAMP_PROPERTY_KEY = "timestamp";

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java

@@ -24,7 +24,6 @@ import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.Graph;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
-import org.apache.hadoop.metadata.typesystem.ITypedInstance;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
 import org.apache.hadoop.metadata.typesystem.persistence.Id;
 import org.slf4j.Logger;
@@ -43,32 +42,34 @@ public final class GraphHelper {
     private GraphHelper() {
     }
 
-    public static Vertex createVertex(Graph graph,
-                                      ITypedReferenceableInstance typedInstance) {
-        return createVertex(graph, typedInstance, typedInstance.getId());
-    }
+    public static Vertex createVertexWithIdentity(Graph graph,
+                                                  ITypedReferenceableInstance typedInstance) {
+        final Vertex vertexWithIdentity = createVertexWithoutIdentity(
+                graph, typedInstance.getTypeName(), typedInstance.getId());
+
+        // add identity
+        final String guid = UUID.randomUUID().toString();
+        vertexWithIdentity.setProperty(Constants.GUID_PROPERTY_KEY, guid);
 
-    public static Vertex createVertex(Graph graph,
-                                      ITypedInstance typedInstance,
-                                      Id typedInstanceId) {
-        return createVertex(graph, typedInstance.getTypeName(), typedInstanceId);
+        return vertexWithIdentity;
     }
 
-    public static Vertex createVertex(Graph graph,
-                                      String typeName,
-                                      Id typedInstanceId) {
-        final Vertex instanceVertex = graph.addVertex(null);
-        // type
-        instanceVertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+    public static Vertex createVertexWithoutIdentity(Graph graph,
+                                                     String typeName,
+                                                     Id typedInstanceId) {
+        final Vertex vertexWithoutIdentity = graph.addVertex(null);
 
-        // id
-        final String guid = UUID.randomUUID().toString();
-        instanceVertex.setProperty(Constants.GUID_PROPERTY_KEY, guid);
+        // add type information
+        vertexWithoutIdentity.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+
+        // add version information
+        vertexWithoutIdentity.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
 
-        // version
-        instanceVertex.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
+        // add timestamp information
+        vertexWithoutIdentity.setProperty(
+                Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
 
-        return instanceVertex;
+        return vertexWithoutIdentity;
     }
 
     public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,

repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java

@@ -20,20 +20,21 @@ package org.apache.hadoop.metadata.services;
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
 import org.apache.hadoop.metadata.discovery.SearchIndexer;
-import org.apache.hadoop.metadata.typesystem.json.Serialization$;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.apache.hadoop.metadata.listener.EntityChangeListener;
 import org.apache.hadoop.metadata.listener.TypesChangeListener;
 import org.apache.hadoop.metadata.repository.MetadataRepository;
 import org.apache.hadoop.metadata.repository.RepositoryException;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.typesystem.ITypedStruct;
+import org.apache.hadoop.metadata.typesystem.TypesDef;
+import org.apache.hadoop.metadata.typesystem.json.Serialization$;
+import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.apache.hadoop.metadata.typesystem.types.IDataType;
 import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
-import org.json.simple.parser.ParseException;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -81,7 +82,7 @@ public class DefaultMetadataService implements MetadataService {
             TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
             Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
 
-            onAdd(typesAdded);
+            onTypesAddedToRepo(typesAdded);
 
             JSONObject response = new JSONObject();
             for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
@@ -101,15 +102,9 @@ public class DefaultMetadataService implements MetadataService {
         Preconditions.checkNotNull(typeDefinition, "type definition cannot be null");
 
         // verify if the type already exists
-        IDataType existingTypeDefinition = null;
-        try {
-            existingTypeDefinition = typeSystem.getDataType(IDataType.class, typeName);
-        } catch (MetadataException ignore) {
-            // do nothing
-        }
-
-        if (existingTypeDefinition != null) {
-            throw new RepositoryException("type is already defined for : " + typeName);
+        if (typeSystem.isRegistered(typeName)) {
+            LOG.error("type is already defined for {}", typeName);
+            throw new MetadataException("type is already defined for : " + typeName);
         }
     }
 
@@ -145,28 +140,16 @@ public class DefaultMetadataService implements MetadataService {
     @Override
     public String createEntity(String entityType,
                                String entityDefinition) throws MetadataException {
-        try {
-            validateEntity(entityDefinition, entityType);
+        Preconditions.checkNotNull(entityDefinition, "entity cannot be null");
+        Preconditions.checkNotNull(entityType, "entity type cannot be null");
 
-            ITypedReferenceableInstance entityInstance =
-                    Serialization$.MODULE$.fromJson(entityDefinition);
-            final String guid = repository.createEntity(entityInstance, entityType);
+        ITypedReferenceableInstance entityInstance =
+                Serialization$.MODULE$.fromJson(entityDefinition);
+        final String guid = repository.createEntity(entityInstance, entityType);
 
-            onAdd(entityType, entityInstance);
+        onEntityAddedToRepo(entityType, entityInstance);
 
-            return guid;
-        } catch (ParseException e) {
-            LOG.error("Unable to parse JSON {} for type {}", entityDefinition, entityType, e);
-            throw new MetadataException("validation failed for: " + entityType);
-        }
-    }
-
-    private void validateEntity(String entity, String entityType) throws ParseException {
-        Preconditions.checkNotNull(entity, "entity cannot be null");
-        Preconditions.checkNotNull(entityType, "entity type cannot be null");
-
-        // todo: this is failing for instances but not types
-        // JSONValue.parseWithException(entity);
+        return guid;
     }
 
     /**
@@ -180,9 +163,7 @@ public class DefaultMetadataService implements MetadataService {
         Preconditions.checkNotNull(guid, "guid cannot be null");
 
         final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
-        return instance == null
-                ? null
-                : Serialization$.MODULE$.toJson(instance);
+        return Serialization$.MODULE$.toJson(instance);
     }
 
     /**
@@ -214,7 +195,65 @@ public class DefaultMetadataService implements MetadataService {
         }
     }
 
-    private void onAdd(Map<String, IDataType> typesAdded) throws MetadataException {
+    /**
+     * Gets the list of trait names for a given entity represented by a guid.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return a list of trait names for the given entity guid
+     * @throws MetadataException
+     */
+    @Override
+    public List<String> getTraitNames(String guid) throws MetadataException {
+        Preconditions.checkNotNull(guid, "entity GUID cannot be null");
+        return repository.getTraitNames(guid);
+    }
+
+    /**
+     * Adds a new trait to an existing entity represented by a guid.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be added to entity
+     * @param traitInstance trait instance that needs to be added to entity
+     * @throws MetadataException
+     */
+    @Override
+    public void addTrait(String guid, String traitName,
+                         ITypedStruct traitInstance) throws MetadataException {
+        Preconditions.checkNotNull(guid, "entity GUID cannot be null");
+        Preconditions.checkNotNull(traitName, "Trait name cannot be null");
+        Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
+
+        // ensure trait type is already registered with the TS
+        Preconditions.checkArgument(!typeSystem.isRegistered(traitName),
+                "trait=%s should be defined in type system before it can be added", traitName);
+
+        repository.addTrait(guid, traitName, traitInstance);
+        onTraitAddedToEntity(guid, traitName);
+    }
+
+    /**
+     * Deletes a given trait from an existing entity represented by a guid.
+     *
+     * @param guid                 globally unique identifier for the entity
+     * @param traitNameToBeDeleted name of the trait
+     * @throws MetadataException
+     */
+    @Override
+    public void deleteTrait(String guid,
+                            String traitNameToBeDeleted) throws MetadataException {
+        Preconditions.checkNotNull(guid, "entity GUID cannot be null");
+        Preconditions.checkNotNull(traitNameToBeDeleted, "Trait name cannot be null");
+
+        // ensure trait type is already registered with the TS
+        Preconditions.checkArgument(!typeSystem.isRegistered(traitNameToBeDeleted),
+                "trait=%s should be defined in type system before it can be deleted",
+                traitNameToBeDeleted);
+
+        repository.deleteTrait(guid, traitNameToBeDeleted);
+        onTraitDeletedFromEntity(guid, traitNameToBeDeleted);
+    }
+
+    private void onTypesAddedToRepo(Map<String, IDataType> typesAdded) throws MetadataException {
         for (TypesChangeListener listener : typesChangeListeners) {
             for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
                 listener.onAdd(entry.getKey(), entry.getValue());
@@ -230,10 +269,26 @@ public class DefaultMetadataService implements MetadataService {
         typesChangeListeners.remove(listener);
     }
 
-    private void onAdd(String typeName,
-                       ITypedReferenceableInstance typedInstance) throws MetadataException {
+    private void onEntityAddedToRepo(String typeName,
+                                     ITypedReferenceableInstance typedInstance)
+        throws MetadataException {
+
+        for (EntityChangeListener listener : entityChangeListeners) {
+            listener.onEntityAdded(typeName, typedInstance);
+        }
+    }
+
+    private void onTraitAddedToEntity(String typeName,
+                                      String traitName) throws MetadataException {
+        for (EntityChangeListener listener : entityChangeListeners) {
+            listener.onTraitAdded(typeName, traitName);
+        }
+    }
+
+    private void onTraitDeletedFromEntity(String typeName,
+                                          String traitName) throws MetadataException {
         for (EntityChangeListener listener : entityChangeListeners) {
-            listener.onAdd(typeName, typedInstance);
+            listener.onTraitDeleted(typeName, traitName);
         }
     }
 

repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.metadata.services;
 
 import org.apache.hadoop.metadata.MetadataException;
+import org.apache.hadoop.metadata.typesystem.ITypedStruct;
 import org.codehaus.jettison.json.JSONObject;
 
 import java.util.List;
@@ -78,4 +79,35 @@ public interface MetadataService {
      * @return list of entity names for the given type in the repository
      */
     List<String> getEntityList(String entityType) throws MetadataException;
+
+    // Trait management functions
+    /**
+     * Gets the list of trait names for a given entity represented by a guid.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return a list of trait names for the given entity guid
+     * @throws MetadataException
+     */
+    List<String> getTraitNames(String guid) throws MetadataException;
+
+    /**
+     * Adds a new trait to an existing entity represented by a guid.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be added to entity
+     * @param traitInstance trait instance that needs to be added to entity
+     * @throws MetadataException
+     */
+    void addTrait(String guid, String traitName,
+                  ITypedStruct traitInstance) throws MetadataException;
+
+    /**
+     * Deletes a given trait from an existing entity represented by a guid.
+     *
+     * @param guid                 globally unique identifier for the entity
+     * @param traitNameToBeDeleted name of the trait
+     * @throws MetadataException
+     */
+    void deleteTrait(String guid,
+                     String traitNameToBeDeleted) throws MetadataException;
 }

repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java

@@ -173,10 +173,12 @@ public class GraphBackedDiscoveryServiceTest {
                 {"DB, Table"},
                 /*{"DB as db1 Table where db1.name = \"Reporting\""},*/
                 {"DB name = \"Reporting\""},
-                {"Column as PII"},
-                {"Table as Dimension"},
-                {"View as Dimension"},
-                {"Column as PII select Column.name"},
+/*
+                {"Column where is PII"},
+                {"Table where is Dimension"},
+                {"View where is Dimension"},
+                {"Column where is PII select Column.name"},
+*/
                 {"Column select Column.name"},
                 {"from Table select Table.name"},
         };

typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java

@@ -81,6 +81,10 @@ public class TypeSystem {
         types.put(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE);
     }
 
+    public boolean isRegistered(String typeName) {
+        return types.containsKey(typeName);
+    }
+
     public <T> T getDataType(Class<T> cls, String name) throws MetadataException {
         if (types.containsKey(name)) {
             return cls.cast(types.get(name));

webapp/pom.xml

@@ -52,6 +52,11 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.hadoop.metadata</groupId>
+            <artifactId>metadata-typesystem</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop.metadata</groupId>
             <artifactId>metadata-repository</artifactId>
         </dependency>
 

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java

@@ -123,7 +123,14 @@ public class EntityResource {
 
             return Response.status(status).entity(response).build();
 
-        } catch (Exception e) {
+        } catch (MetadataException e) {
+            LOG.error("An entity with GUID={} does not exist", guid, e);
+            throw new WebApplicationException(e, Response
+                    .status(Response.Status.NOT_FOUND)
+                    .entity(e.getMessage())
+                    .type(MediaType.APPLICATION_JSON)
+                    .build());
+        } catch (JSONException e) {
             LOG.error("Unable to get instance definition for GUID {}", guid, e);
             throw new WebApplicationException(e, Response
                     .status(Response.Status.INTERNAL_SERVER_ERROR)