BUG-32815 BUG-32816 BUG-32827 Entity Trait Management Backend API. Contributed…

repository/src/main/java/org/apache/hadoop/metadata/listener/EntityChangeListener.java

@@ -33,7 +33,24 @@ public interface EntityChangeListener {
      * @param typedInstance a typed instance
      * @throws org.apache.hadoop.metadata.MetadataException
      */
-    void onAdd(String typeName,
-               ITypedReferenceableInstance typedInstance) throws MetadataException;
+    void onEntityAdded(String typeName,
+                       ITypedReferenceableInstance typedInstance) throws MetadataException;
 
+    /**
+     * This is upon adding a new trait to a typed instance.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be added to entity
+     * @throws org.apache.hadoop.metadata.MetadataException
+     */
+    void onTraitAdded(String guid, String traitName) throws MetadataException;
+
+    /**
+     * This is upon deleting a trait from a typed instance.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be deleted from entity
+     * @throws org.apache.hadoop.metadata.MetadataException
+     */
+    void onTraitDeleted(String guid, String traitName) throws MetadataException;
 }

repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java

@@ -20,6 +20,9 @@ package org.apache.hadoop.metadata.repository;
 
 import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.typesystem.ITypedStruct;
+import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
+import org.apache.hadoop.metadata.typesystem.types.IDataType;
 
 import java.util.List;
 
@@ -28,12 +31,130 @@ import java.util.List;
  */
 public interface MetadataRepository {
 
+    /**
+     * Returns the property key used to store entity type name.
+     *
+     * @return property key used to store entity type name.
+     */
     String getTypeAttributeName();
 
+    /**
+     * Return the property key used to store a given traitName in the repository.
+     *
+     * @param dataType  data type
+     * @param traitName trait name
+     * @return property key used to store a given traitName
+     */
+    String getTraitLabel(IDataType<?> dataType, String traitName);
+
+    /**
+     * Return the property key used to store a given attribute in the repository.
+     *
+     * @param dataType data type
+     * @param aInfo    attribute info
+     * @return property key used to store a given attribute
+     */
+    String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo);
+
+    /**
+     * Return the edge label for a given attribute in the repository.
+     *
+     * @param dataType  data type
+     * @param aInfo    attribute info
+     * @return edge label for a given attribute
+     */
+    String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo);
+
+    /**
+     * Creates an entity definition (instance) corresponding to a given type.
+     *
+     * @param entity     entity (typed instance)
+     * @param entityType entity type name
+     * @return a globally unique identifier
+     * @throws RepositoryException
+     */
     String createEntity(IReferenceableInstance entity,
                         String entityType) throws RepositoryException;
 
+    /**
+     * Fetch the complete definition of an entity given its GUID.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return entity (typed instance) definition
+     * @throws RepositoryException
+     */
     ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException;
 
+    /**
+     * Gets the list of entities for a given entity type.
+     *
+     * @param entityType name of a type which is unique
+     * @return a list of entity names for the given type
+     * @throws RepositoryException
+     */
     List<String> getEntityList(String entityType) throws RepositoryException;
+
+    /**
+     * Deletes an entity definition (instance) corresponding to a given type.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return true if deleted else false
+     * @throws RepositoryException
+     */
+    // boolean deleteEntity(String guid) throws RepositoryException;
+
+    /**
+     * Updates an entity given its GUID with the attribute name and value.
+     *
+     * @param guid           globally unique identifier for the entity
+     * @param attributeName  name of the attribute
+     * @param attributeValue value of the attribute
+     * @return an entity instance with updated state
+     * @throws RepositoryException
+     */
+    //ITypedReferenceableInstance updateEntity(String guid, String attributeName,
+    //                                         String attributeValue) throws RepositoryException;
+
+
+    // Trait management functions
+    /**
+     * Gets the list of trait names for a given entity represented by a guid.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return a list of trait names for the given entity guid
+     * @throws RepositoryException
+     */
+    List<String> getTraitNames(String guid) throws RepositoryException;
+
+    /**
+     * Adds a new trait to an existing entity represented by a guid.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be added to entity
+     * @param traitInstance trait instance that needs to be added to entity
+     * @throws RepositoryException
+     */
+    void addTrait(String guid, String traitName,
+                  ITypedStruct traitInstance) throws RepositoryException;
+
+    /**
+     * Adds a list of traits to an existing entity represented by a guid.
+     *
+     * @param guid           globally unique identifier for the entity
+     * @param traitInstances list of trait instances that needs to be added to entity
+     * @return an entity instance with updated traits
+     * @throws RepositoryException
+     */
+    // ITypedReferenceableInstance addTraits(String guid, Map<String, ITypedStruct> traitInstances)
+    //    throws RepositoryException;
+
+    /**
+     * Deletes a given trait from an existing entity represented by a guid.
+     *
+     * @param guid                 globally unique identifier for the entity
+     * @param traitNameToBeDeleted name of the trait
+     * @throws RepositoryException
+     */
+    void deleteTrait(String guid,
+                     String traitNameToBeDeleted) throws RepositoryException;
 }

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java

@@ -33,14 +33,10 @@ public final class Constants {
     public static final String ENTITY_TYPE_INDEX = "type_index";
 
     /**
-     * Data type property key.
-     */
-    // public static final String DATA_TYPE_PROPERTY_KEY = "dataType";
-
-    /**
-     * Trait names property key.
+     * Trait names property key and index name.
      */
     public static final String TRAIT_NAMES_PROPERTY_KEY = "traitNames";
+    public static final String TRAIT_NAMES_INDEX = "trait_names_index";
 
     public static final String VERSION_PROPERTY_KEY = "version";
     public static final String TIMESTAMP_PROPERTY_KEY = "timestamp";

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.metadata.repository.graph;
 
+import com.google.common.base.Preconditions;
 import com.thinkaurelius.titan.core.TitanGraph;
 import com.thinkaurelius.titan.core.TitanProperty;
 import com.thinkaurelius.titan.core.TitanVertex;
@@ -50,7 +51,6 @@ import javax.inject.Inject;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -96,14 +96,17 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         return Constants.ENTITY_TYPE_PROPERTY_KEY;
     }
 
+    @Override
     public String getTraitLabel(IDataType<?> dataType, String traitName) {
         return dataType.getName() + "." + traitName;
     }
 
+    @Override
     public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) {
         return dataType.getName() + "." + aInfo.name;
     }
 
+    @Override
     public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
         return dataType.getName() + "." + aInfo.name;
     }
@@ -131,20 +134,27 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
 
         try {
             titanGraph.rollback();  // clean up before starting a query
-            Vertex instanceVertex = GraphHelper.findVertexByGUID(titanGraph, guid);
-            if (instanceVertex == null) {
-                LOG.debug("Could not find a vertex for guid {}", guid);
-                return null;
-            }
+            Vertex instanceVertex = getVertexForGUID(guid);
 
             LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
             return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
 
-        } catch (Exception e) {
+        } catch (MetadataException e) {
             throw new RepositoryException(e);
         }
     }
 
+    private Vertex getVertexForGUID(String guid) throws RepositoryException {
+        Vertex instanceVertex = GraphHelper.findVertexByGUID(titanGraph, guid);
+        if (instanceVertex == null) {
+            LOG.debug("Could not find a vertex for guid={}", guid);
+            throw new RepositoryException(
+                    "Could not find an entity in the repository for guid: " + guid);
+        }
+
+        return instanceVertex;
+    }
+
     @Override
     public List<String> getEntityList(String entityType) throws RepositoryException {
         LOG.info("Retrieving entity list for type={}", entityType);
@@ -164,18 +174,133 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         return entityList;
     }
 
-    public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
+    /**
+     * Gets the list of trait names for a given entity represented by a guid.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return a list of trait names for the given entity guid
+     * @throws RepositoryException
+     */
+    @Override
+    public List<String> getTraitNames(String guid) throws RepositoryException {
+        LOG.info("Retrieving trait names for entity={}", guid);
+        titanGraph.rollback();  // clean up before starting a query
+        Vertex instanceVertex = getVertexForGUID(guid);
+        return getTraitNames(instanceVertex);
+    }
+
+    public List<String> getTraitNames(Vertex entityVertex) {
+        ArrayList<String> traits = new ArrayList<>();
+        for (TitanProperty property : ((TitanVertex) entityVertex)
+                .getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
+            traits.add((String) property.getValue());
+        }
+
+        return traits;
+    }
+
+    /**
+     * Adds a new trait to an existing entity represented by a guid.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitInstance trait instance that needs to be added to entity
+     * @throws RepositoryException
+     */
+    @Override
+    public void addTrait(String guid, String traitName,
+                         ITypedStruct traitInstance) throws RepositoryException {
+        Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
+        LOG.info("Adding a new trait={} for entity={}", traitName, guid);
+
+        try {
+            titanGraph.rollback();  // clean up before starting a query
+            Vertex instanceVertex = getVertexForGUID(guid);
+
+            // add the trait instance as a new vertex
+            final String typeName = getTypeName(instanceVertex);
+            instanceToGraphMapper.mapTraitInstanceToVertex(
+                    traitName, traitInstance, getIdFromVertex(typeName, instanceVertex),
+                    typeName, instanceVertex, Collections.<Id, Vertex>emptyMap());
+
+            // update the traits in entity once adding trait instance is successful
+            ((TitanVertex) instanceVertex)
+                    .addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
+
+            titanGraph.commit();  // commit if there are no errors
+        } catch (MetadataException e) {
+            titanGraph.rollback();
+            throw new RepositoryException(e);
+        }
+    }
+
+    /**
+     * Deletes a given trait from an existing entity represented by a guid.
+     *
+     * @param guid      globally unique identifier for the entity
+     * @param traitNameToBeDeleted name of the trait
+     * @throws RepositoryException
+     */
+    @Override
+    public void deleteTrait(String guid, String traitNameToBeDeleted)
+            throws RepositoryException {
+        LOG.info("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
+        try {
+            titanGraph.rollback();  // clean up before starting a query
+            Vertex instanceVertex = getVertexForGUID(guid);
+
+            List<String> traitNames = getTraitNames(instanceVertex);
+            if (!traitNames.contains(traitNameToBeDeleted)) {
+                throw new RepositoryException("Could not find trait=" + traitNameToBeDeleted
+                        + " in the repository for entity: " + guid);
+            }
+
+            final String entityTypeName = getTypeName(instanceVertex);
+            String relationshipLabel = entityTypeName + "." + traitNameToBeDeleted;
+            Iterator<Edge> results = instanceVertex.getEdges(
+                    Direction.OUT, relationshipLabel).iterator();
+            if (results.hasNext()) { // there should only be one edge for this label
+                final Edge traitEdge = results.next();
+                final Vertex traitVertex = traitEdge.getVertex(Direction.IN);
+
+                // remove the edge to the trait instance from the repository
+                titanGraph.removeEdge(traitEdge);
+
+                if (traitVertex != null) { // remove the trait instance from the repository
+                    titanGraph.removeVertex(traitVertex);
+
+                    // update the traits in entity once trait removal is successful
+                    traitNames.remove(traitNameToBeDeleted);
+                    updateTraits(instanceVertex, traitNames);
+                }
+
+                titanGraph.commit();  // commit if there are no errors
+            }
+        } catch (Exception e) {
+            titanGraph.rollback();
+            throw new RepositoryException(e);
+        }
+    }
+
+    private void updateTraits(Vertex instanceVertex, List<String> traitNames) {
+        // remove the key
+        instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
+
+        // add it back again
+        for (String traitName : traitNames) {
+            ((TitanVertex) instanceVertex).addProperty(
+                    Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
+        }
+    }
+
+    public Id getIdFromVertex(String dataTypeName, Vertex vertex) {
         return new Id(
                 vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
                 vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
                 dataTypeName);
     }
 
-    public List<String> getTraitNames(TitanVertex vertex) {
-        final String traitNames = vertex.getProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
-        return traitNames == null
-                ? Collections.<String>emptyList()
-                : Arrays.asList(traitNames.split(","));
+    String getTypeName(Vertex instanceVertex) {
+        return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
     }
 
     private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
@@ -230,7 +355,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                     if (id.isAssigned()) {  // has a GUID
                         instanceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
                     } else {
-                        instanceVertex = GraphHelper.createVertex(titanGraph, typedInstance);
+                        instanceVertex =
+                                GraphHelper.createVertexWithIdentity(titanGraph, typedInstance);
                     }
 
                     idToVertexMap.put(id, instanceVertex);
@@ -242,7 +368,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
     private final class TypedInstanceToGraphMapper {
 
         private String mapTypedInstanceToGraph(IReferenceableInstance typedInstance)
-        throws MetadataException {
+            throws MetadataException {
 
             EntityProcessor entityProcessor = new EntityProcessor();
             try {
@@ -258,7 +384,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             return addDiscoveredInstances(typedInstance, entityProcessor, newTypedInstances);
         }
 
-        /*
+        /**
          * Step 2: Traverse oldIdToInstance map create newInstances :
          * List[ITypedReferenceableInstance]
          *  - create a ITypedReferenceableInstance.
@@ -277,9 +403,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                             transientInstance, Multiplicity.REQUIRED);
                     newTypedInstances.add(newInstance);
 
-                /*
-                 * Now replace old references with new Ids
-                 */
+                    // Now replace old references with new Ids
                     MapIds mapIds = new MapIds(entityProcessor.idToNewIdMap);
                     new ObjectGraphWalker(typeSystem, mapIds, newTypedInstances).walk();
 
@@ -488,8 +612,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                 case CLASS:
                     Id referenceId = (Id) value;
                     mapClassReferenceAsEdge(
-                            instanceVertex, idToVertexMap,
-                            propertyName, referenceId);
+                            instanceVertex, idToVertexMap, propertyName, referenceId);
                     break;
 
                 default:
@@ -521,7 +644,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                                                  Map<Id, Vertex> idToVertexMap)
         throws MetadataException {
             // add a new vertex for the struct or trait instance
-            Vertex structInstanceVertex = GraphHelper.createVertex(
+            Vertex structInstanceVertex = GraphHelper.createVertexWithoutIdentity(
                     graphService.getBlueprintsGraph(), structInstance.getTypeName(), id);
             LOG.debug("created vertex {} for struct {}", structInstanceVertex, attributeInfo.name);
 
@@ -536,18 +659,28 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                                               ITypedReferenceableInstance typedInstance,
                                               Vertex parentInstanceVertex,
                                               Map<Id, Vertex> idToVertexMap)
-        throws MetadataException {
+            throws MetadataException {
             // add a new vertex for the struct or trait instance
-            Vertex traitInstanceVertex = GraphHelper.createVertex(
-                    graphService.getBlueprintsGraph(), traitInstance, typedInstance.getId());
+            mapTraitInstanceToVertex(traitName, traitInstance, typedInstance.getId(),
+                    typedInstance.getTypeName(), parentInstanceVertex, idToVertexMap);
+        }
+
+        private void mapTraitInstanceToVertex(String traitName, ITypedStruct traitInstance,
+                                              Id typedInstanceId, String typedInstanceTypeName,
+                                              Vertex parentInstanceVertex,
+                                              Map<Id, Vertex> idToVertexMap)
+            throws MetadataException {
+            // add a new vertex for the struct or trait instance
+            Vertex traitInstanceVertex = GraphHelper.createVertexWithoutIdentity(
+                    graphService.getBlueprintsGraph(), traitInstance.getTypeName(), typedInstanceId);
             LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
 
             // map all the attributes to this newly created vertex
-            mapInstanceToVertex(typedInstance.getId(), traitInstance, traitInstanceVertex,
+            mapInstanceToVertex(typedInstanceId, traitInstance, traitInstanceVertex,
                     traitInstance.fieldMapping().fields, idToVertexMap);
 
             // add an edge to the newly created vertex from the parent
-            String relationshipLabel = typedInstance.getTypeName() + "." + traitName;
+            String relationshipLabel = typedInstanceTypeName + "." + traitName;
             GraphHelper.addEdge(
                     titanGraph, parentInstanceVertex, traitInstanceVertex, relationshipLabel);
         }
@@ -602,16 +735,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
 
         public ITypedReferenceableInstance mapGraphToTypedInstance(String guid,
                                                                    Vertex instanceVertex)
-        throws MetadataException {
+            throws MetadataException {
 
             LOG.debug("Mapping graph root vertex {} to typed instance for guid {}",
                     instanceVertex, guid);
             String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
-            List<String> traits = new ArrayList<>();
-            for (TitanProperty property : ((TitanVertex) instanceVertex)
-                    .getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
-                traits.add((String) property.getValue());
-            }
+            List<String> traits = getTraitNames(instanceVertex);
 
             Id id = new Id(guid,
                     instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java

@@ -24,7 +24,6 @@ import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.Graph;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
-import org.apache.hadoop.metadata.typesystem.ITypedInstance;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
 import org.apache.hadoop.metadata.typesystem.persistence.Id;
 import org.slf4j.Logger;
@@ -43,32 +42,34 @@ public final class GraphHelper {
     private GraphHelper() {
     }
 
-    public static Vertex createVertex(Graph graph,
-                                      ITypedReferenceableInstance typedInstance) {
-        return createVertex(graph, typedInstance, typedInstance.getId());
-    }
+    public static Vertex createVertexWithIdentity(Graph graph,
+                                                  ITypedReferenceableInstance typedInstance) {
+        final Vertex vertexWithIdentity = createVertexWithoutIdentity(
+                graph, typedInstance.getTypeName(), typedInstance.getId());
+
+        // add identity
+        final String guid = UUID.randomUUID().toString();
+        vertexWithIdentity.setProperty(Constants.GUID_PROPERTY_KEY, guid);
 
-    public static Vertex createVertex(Graph graph,
-                                      ITypedInstance typedInstance,
-                                      Id typedInstanceId) {
-        return createVertex(graph, typedInstance.getTypeName(), typedInstanceId);
+        return vertexWithIdentity;
     }
 
-    public static Vertex createVertex(Graph graph,
-                                      String typeName,
-                                      Id typedInstanceId) {
-        final Vertex instanceVertex = graph.addVertex(null);
-        // type
-        instanceVertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+    public static Vertex createVertexWithoutIdentity(Graph graph,
+                                                     String typeName,
+                                                     Id typedInstanceId) {
+        final Vertex vertexWithoutIdentity = graph.addVertex(null);
 
-        // id
-        final String guid = UUID.randomUUID().toString();
-        instanceVertex.setProperty(Constants.GUID_PROPERTY_KEY, guid);
+        // add type information
+        vertexWithoutIdentity.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+
+        // add version information
+        vertexWithoutIdentity.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
 
-        // version
-        instanceVertex.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
+        // add timestamp information
+        vertexWithoutIdentity.setProperty(
+                Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
 
-        return instanceVertex;
+        return vertexWithoutIdentity;
     }
 
     public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,

repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java

@@ -20,20 +20,21 @@ package org.apache.hadoop.metadata.services;
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
 import org.apache.hadoop.metadata.discovery.SearchIndexer;
-import org.apache.hadoop.metadata.typesystem.json.Serialization$;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.apache.hadoop.metadata.listener.EntityChangeListener;
 import org.apache.hadoop.metadata.listener.TypesChangeListener;
 import org.apache.hadoop.metadata.repository.MetadataRepository;
 import org.apache.hadoop.metadata.repository.RepositoryException;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.typesystem.ITypedStruct;
+import org.apache.hadoop.metadata.typesystem.TypesDef;
+import org.apache.hadoop.metadata.typesystem.json.Serialization$;
+import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.apache.hadoop.metadata.typesystem.types.IDataType;
 import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
-import org.json.simple.parser.ParseException;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -81,7 +82,7 @@ public class DefaultMetadataService implements MetadataService {
             TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
             Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
 
-            onAdd(typesAdded);
+            onTypesAddedToRepo(typesAdded);
 
             JSONObject response = new JSONObject();
             for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
@@ -101,15 +102,9 @@ public class DefaultMetadataService implements MetadataService {
         Preconditions.checkNotNull(typeDefinition, "type definition cannot be null");
 
         // verify if the type already exists
-        IDataType existingTypeDefinition = null;
-        try {
-            existingTypeDefinition = typeSystem.getDataType(IDataType.class, typeName);
-        } catch (MetadataException ignore) {
-            // do nothing
-        }
-
-        if (existingTypeDefinition != null) {
-            throw new RepositoryException("type is already defined for : " + typeName);
+        if (typeSystem.isRegistered(typeName)) {
+            LOG.error("type is already defined for {}", typeName);
+            throw new MetadataException("type is already defined for : " + typeName);
         }
     }
 
@@ -145,28 +140,16 @@ public class DefaultMetadataService implements MetadataService {
     @Override
     public String createEntity(String entityType,
                                String entityDefinition) throws MetadataException {
-        try {
-            validateEntity(entityDefinition, entityType);
+        Preconditions.checkNotNull(entityDefinition, "entity cannot be null");
+        Preconditions.checkNotNull(entityType, "entity type cannot be null");
 
-            ITypedReferenceableInstance entityInstance =
-                    Serialization$.MODULE$.fromJson(entityDefinition);
-            final String guid = repository.createEntity(entityInstance, entityType);
+        ITypedReferenceableInstance entityInstance =
+                Serialization$.MODULE$.fromJson(entityDefinition);
+        final String guid = repository.createEntity(entityInstance, entityType);
 
-            onAdd(entityType, entityInstance);
+        onEntityAddedToRepo(entityType, entityInstance);
 
-            return guid;
-        } catch (ParseException e) {
-            LOG.error("Unable to parse JSON {} for type {}", entityDefinition, entityType, e);
-            throw new MetadataException("validation failed for: " + entityType);
-        }
-    }
-
-    private void validateEntity(String entity, String entityType) throws ParseException {
-        Preconditions.checkNotNull(entity, "entity cannot be null");
-        Preconditions.checkNotNull(entityType, "entity type cannot be null");
-
-        // todo: this is failing for instances but not types
-        // JSONValue.parseWithException(entity);
+        return guid;
     }
 
     /**
@@ -180,9 +163,7 @@ public class DefaultMetadataService implements MetadataService {
         Preconditions.checkNotNull(guid, "guid cannot be null");
 
         final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
-        return instance == null
-                ? null
-                : Serialization$.MODULE$.toJson(instance);
+        return Serialization$.MODULE$.toJson(instance);
     }
 
     /**
@@ -214,7 +195,65 @@ public class DefaultMetadataService implements MetadataService {
         }
     }
 
-    private void onAdd(Map<String, IDataType> typesAdded) throws MetadataException {
+    /**
+     * Gets the list of trait names for a given entity represented by a guid.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return a list of trait names for the given entity guid
+     * @throws MetadataException
+     */
+    @Override
+    public List<String> getTraitNames(String guid) throws MetadataException {
+        Preconditions.checkNotNull(guid, "entity GUID cannot be null");
+        return repository.getTraitNames(guid);
+    }
+
+    /**
+     * Adds a new trait to an existing entity represented by a guid.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be added to entity
+     * @param traitInstance trait instance that needs to be added to entity
+     * @throws MetadataException
+     */
+    @Override
+    public void addTrait(String guid, String traitName,
+                         ITypedStruct traitInstance) throws MetadataException {
+        Preconditions.checkNotNull(guid, "entity GUID cannot be null");
+        Preconditions.checkNotNull(traitName, "Trait name cannot be null");
+        Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
+
+        // ensure trait type is already registered with the TS
+        Preconditions.checkArgument(!typeSystem.isRegistered(traitName),
+                "trait=%s should be defined in type system before it can be added", traitName);
+
+        repository.addTrait(guid, traitName, traitInstance);
+        onTraitAddedToEntity(guid, traitName);
+    }
+
+    /**
+     * Deletes a given trait from an existing entity represented by a guid.
+     *
+     * @param guid                 globally unique identifier for the entity
+     * @param traitNameToBeDeleted name of the trait
+     * @throws MetadataException
+     */
+    @Override
+    public void deleteTrait(String guid,
+                            String traitNameToBeDeleted) throws MetadataException {
+        Preconditions.checkNotNull(guid, "entity GUID cannot be null");
+        Preconditions.checkNotNull(traitNameToBeDeleted, "Trait name cannot be null");
+
+        // ensure trait type is already registered with the TS
+        Preconditions.checkArgument(!typeSystem.isRegistered(traitNameToBeDeleted),
+                "trait=%s should be defined in type system before it can be deleted",
+                traitNameToBeDeleted);
+
+        repository.deleteTrait(guid, traitNameToBeDeleted);
+        onTraitDeletedFromEntity(guid, traitNameToBeDeleted);
+    }
+
+    private void onTypesAddedToRepo(Map<String, IDataType> typesAdded) throws MetadataException {
         for (TypesChangeListener listener : typesChangeListeners) {
             for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
                 listener.onAdd(entry.getKey(), entry.getValue());
@@ -230,10 +269,26 @@ public class DefaultMetadataService implements MetadataService {
         typesChangeListeners.remove(listener);
     }
 
-    private void onAdd(String typeName,
-                       ITypedReferenceableInstance typedInstance) throws MetadataException {
+    private void onEntityAddedToRepo(String typeName,
+                                     ITypedReferenceableInstance typedInstance)
+        throws MetadataException {
+
+        for (EntityChangeListener listener : entityChangeListeners) {
+            listener.onEntityAdded(typeName, typedInstance);
+        }
+    }
+
+    private void onTraitAddedToEntity(String typeName,
+                                      String traitName) throws MetadataException {
+        for (EntityChangeListener listener : entityChangeListeners) {
+            listener.onTraitAdded(typeName, traitName);
+        }
+    }
+
+    private void onTraitDeletedFromEntity(String typeName,
+                                          String traitName) throws MetadataException {
         for (EntityChangeListener listener : entityChangeListeners) {
-            listener.onAdd(typeName, typedInstance);
+            listener.onTraitDeleted(typeName, traitName);
         }
     }
 

repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.metadata.services;
 
 import org.apache.hadoop.metadata.MetadataException;
+import org.apache.hadoop.metadata.typesystem.ITypedStruct;
 import org.codehaus.jettison.json.JSONObject;
 
 import java.util.List;
@@ -78,4 +79,35 @@ public interface MetadataService {
      * @return list of entity names for the given type in the repository
      */
     List<String> getEntityList(String entityType) throws MetadataException;
+
+    // Trait management functions
+    /**
+     * Gets the list of trait names for a given entity represented by a guid.
+     *
+     * @param guid globally unique identifier for the entity
+     * @return a list of trait names for the given entity guid
+     * @throws MetadataException
+     */
+    List<String> getTraitNames(String guid) throws MetadataException;
+
+    /**
+     * Adds a new trait to an existing entity represented by a guid.
+     *
+     * @param guid          globally unique identifier for the entity
+     * @param traitName     trait name for the instance that needs to be added to entity
+     * @param traitInstance trait instance that needs to be added to entity
+     * @throws MetadataException
+     */
+    void addTrait(String guid, String traitName,
+                  ITypedStruct traitInstance) throws MetadataException;
+
+    /**
+     * Deletes a given trait from an existing entity represented by a guid.
+     *
+     * @param guid                 globally unique identifier for the entity
+     * @param traitNameToBeDeleted name of the trait
+     * @throws MetadataException
+     */
+    void deleteTrait(String guid,
+                     String traitNameToBeDeleted) throws MetadataException;
 }

repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java

@@ -173,10 +173,12 @@ public class GraphBackedDiscoveryServiceTest {
                 {"DB, Table"},
                 /*{"DB as db1 Table where db1.name = \"Reporting\""},*/
                 {"DB name = \"Reporting\""},
-                {"Column as PII"},
-                {"Table as Dimension"},
-                {"View as Dimension"},
-                {"Column as PII select Column.name"},
+/*
+                {"Column where is PII"},
+                {"Table where is Dimension"},
+                {"View where is Dimension"},
+                {"Column where is PII select Column.name"},
+*/
                 {"Column select Column.name"},
                 {"from Table select Table.name"},
         };

repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java

@@ -18,24 +18,41 @@
 
 package org.apache.hadoop.metadata.repository.graph;
 
+import com.google.common.collect.ImmutableList;
 import com.thinkaurelius.titan.core.TitanGraph;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Compare;
+import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
 import org.apache.hadoop.metadata.RepositoryMetadataModule;
 import org.apache.hadoop.metadata.TestUtils;
+import org.apache.hadoop.metadata.repository.RepositoryException;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.typesystem.ITypedStruct;
 import org.apache.hadoop.metadata.typesystem.Referenceable;
+import org.apache.hadoop.metadata.typesystem.Struct;
+import org.apache.hadoop.metadata.typesystem.persistence.Id;
+import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
 import org.apache.hadoop.metadata.typesystem.types.ClassType;
+import org.apache.hadoop.metadata.typesystem.types.DataTypes;
+import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
+import org.apache.hadoop.metadata.typesystem.types.EnumValue;
+import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
 import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
+import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
+import org.apache.hadoop.metadata.typesystem.types.TraitType;
 import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
+import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
 import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
+import scala.actors.threadpool.Arrays;
 
 import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.UUID;
 
 /**
  * GraphBackedMetadataRepository test
@@ -47,66 +64,388 @@ import java.util.List;
 public class GraphBackedMetadataRepositoryTest {
 
     private static final String ENTITY_TYPE = "Department";
+    private static final String DATABASE_TYPE = "hive_database";
+    private static final String DATABASE_NAME = "foo";
+    private static final String TABLE_TYPE = "hive_table";
+    private static final String TABLE_NAME = "bar";
+    private static final String CLASSIFICATION = "classification";
+    private static final String PII = "PII";
 
     @Inject
     private TitanGraphService titanGraphService;
     @Inject
     private GraphBackedMetadataRepository repositoryService;
 
-    private TypeSystem ts;
+    private TypeSystem typeSystem;
     private String guid;
 
     @BeforeClass
     public void setUp() throws Exception {
+        typeSystem = TypeSystem.getInstance();
+        typeSystem.reset();
+
         // start the injected graph service
         titanGraphService.initialize();
 
         new GraphBackedSearchIndexer(titanGraphService);
 
-        ts = TypeSystem.getInstance();
+        TestUtils.defineDeptEmployeeTypes(typeSystem);
+        createHiveTypes();
+    }
 
-        TestUtils.defineDeptEmployeeTypes(ts);
+/*
+    @AfterMethod
+    public void tearDown() {
+         dumpGraph();
     }
+*/
 
     @Test
     public void testSubmitEntity() throws Exception {
-        Referenceable hrDept = TestUtils.createDeptEg1(ts);
-        ClassType deptType = ts.getDataType(ClassType.class, "Department");
+        Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
+        ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
         ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
 
         guid = repositoryService.createEntity(hrDept2, ENTITY_TYPE);
         Assert.assertNotNull(guid);
-
-        dumpGraph();
-    }
-
-    private void dumpGraph() {
-        TitanGraph graph = titanGraphService.getTitanGraph();
-        for (Vertex v : graph.getVertices()) {
-            System.out.println("****v = " + GraphHelper.vertexString(v));
-            for (Edge e : v.getEdges(Direction.OUT)) {
-                System.out.println("****e = " + GraphHelper.edgeString(e));
-            }
-        }
     }
 
     @Test(dependsOnMethods = "testSubmitEntity")
-    public void testGetEntityDefinition() throws Exception {
+    public void testGetEntityDefinitionForDepartment() throws Exception {
         ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid);
         Assert.assertNotNull(entity);
     }
 
-    @Test
+    @Test (expectedExceptions = RepositoryException.class)
     public void testGetEntityDefinitionNonExistent() throws Exception {
-        ITypedReferenceableInstance entity = repositoryService.getEntityDefinition("blah");
-        Assert.assertNull(entity);
+        repositoryService.getEntityDefinition("blah");
+        Assert.fail();
     }
 
-    @Test(enabled = false)
+    @Test
     public void testGetEntityList() throws Exception {
         List<String> entityList = repositoryService.getEntityList(ENTITY_TYPE);
         System.out.println("entityList = " + entityList);
         Assert.assertNotNull(entityList);
         Assert.assertEquals(entityList.size(), 1); // one department
     }
+
+    @Test
+    public void testGetTypeAttributeName() throws Exception {
+        Assert.assertEquals(
+                repositoryService.getTypeAttributeName(), Constants.ENTITY_TYPE_PROPERTY_KEY);
+    }
+
+    @Test (dependsOnMethods = "testSubmitEntity")
+    public void testGetTraitLabel() throws Exception {
+        Assert.assertEquals(repositoryService.getTraitLabel(
+                    typeSystem.getDataType(ClassType.class, TABLE_TYPE), CLASSIFICATION),
+                TABLE_TYPE + "." + CLASSIFICATION);
+    }
+
+    @Test
+    public void testCreateEntity() throws Exception {
+        Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+        databaseInstance.set("name", DATABASE_NAME);
+        databaseInstance.set("description", "foo database");
+        System.out.println("databaseInstance = " + databaseInstance);
+
+        ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
+        ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
+        System.out.println("db = " + db);
+
+        String dbGUID = repositoryService.createEntity(db, DATABASE_TYPE);
+        System.out.println("added db = " + dbGUID);
+
+        Referenceable dbInstance = new Referenceable(
+                dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
+
+        ITypedReferenceableInstance table = createHiveTableInstance(dbInstance);
+        String tableGUID = repositoryService.createEntity(table, TABLE_TYPE);
+        System.out.println("added table = " + tableGUID);
+    }
+
+    @Test(dependsOnMethods = "testCreateEntity")
+    public void testGetEntityDefinition() throws Exception {
+        String guid = getGUID();
+
+        ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
+        System.out.println("*** table = " + table);
+    }
+
+    private String getGUID() {
+        Vertex tableVertex = getTableEntityVertex();
+
+        String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+        if (guid == null) {
+            Assert.fail();
+        }
+        return guid;
+    }
+
+    private Vertex getTableEntityVertex() {
+        TitanGraph graph = titanGraphService.getTitanGraph();
+        GraphQuery query = graph.query()
+                .has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
+        Iterator<Vertex> results = query.vertices().iterator();
+        // returning one since guid should be unique
+        Vertex tableVertex = results.hasNext() ? results.next() : null;
+        if (tableVertex == null) {
+            Assert.fail();
+        }
+
+        return tableVertex;
+    }
+
+    @Test (dependsOnMethods = "testCreateEntity")
+    public void testGetTraitNames() throws Exception {
+        final List<String> traitNames = repositoryService.getTraitNames(getGUID());
+        Assert.assertEquals(traitNames.size(), 1);
+        Assert.assertEquals(traitNames, Arrays.asList(new String[]{CLASSIFICATION}));
+    }
+
+    @Test
+    public void testGetTraitNamesForEmptyTraits() throws Exception {
+        final List<String> traitNames = repositoryService.getTraitNames(guid);
+        Assert.assertEquals(traitNames.size(), 0);
+    }
+
+    @Test (expectedExceptions = RepositoryException.class)
+    public void testGetTraitNamesForBadEntity() throws Exception {
+        repositoryService.getTraitNames(UUID.randomUUID().toString());
+        Assert.fail();
+    }
+
+    @Test (dependsOnMethods = "testGetTraitNames")
+    public void testAddTrait() throws Exception {
+        final String aGUID = getGUID();
+
+        List<String> traitNames = repositoryService.getTraitNames(aGUID);
+        System.out.println("traitNames = " + traitNames);
+        Assert.assertEquals(traitNames.size(), 1);
+        Assert.assertTrue(traitNames.contains(CLASSIFICATION));
+        Assert.assertFalse(traitNames.contains(PII));
+
+        HierarchicalTypeDefinition<TraitType> piiTrait =
+                TypesUtil.createTraitTypeDef(PII, ImmutableList.<String>of());
+        TraitType traitType = typeSystem.defineTraitType(piiTrait);
+        ITypedStruct traitInstance = traitType.createInstance();
+
+        repositoryService.addTrait(aGUID, PII, traitInstance);
+
+        // refresh trait names
+        traitNames = repositoryService.getTraitNames(aGUID);
+        Assert.assertEquals(traitNames.size(), 2);
+        Assert.assertTrue(traitNames.contains(PII));
+        Assert.assertTrue(traitNames.contains(CLASSIFICATION));
+    }
+
+    @Test (expectedExceptions = NullPointerException.class)
+    public void testAddTraitWithNullInstance() throws Exception {
+        repositoryService.addTrait(getGUID(), PII, null);
+        Assert.fail();
+    }
+
+    @Test (dependsOnMethods = "testAddTrait", expectedExceptions = RepositoryException.class)
+    public void testAddTraitForBadEntity() throws Exception {
+        TraitType traitType = typeSystem.getDataType(TraitType.class, PII);
+        ITypedStruct traitInstance = traitType.createInstance();
+
+        repositoryService.addTrait(UUID.randomUUID().toString(), PII, traitInstance);
+        Assert.fail();
+    }
+
+    @Test (dependsOnMethods = "testAddTrait")
+    public void testDeleteTrait() throws Exception {
+        final String aGUID = getGUID();
+
+        List<String> traitNames = repositoryService.getTraitNames(aGUID);
+        Assert.assertEquals(traitNames.size(), 2);
+        Assert.assertTrue(traitNames.contains(PII));
+        Assert.assertTrue(traitNames.contains(CLASSIFICATION));
+
+        repositoryService.deleteTrait(aGUID, PII);
+
+        // refresh trait names
+        traitNames = repositoryService.getTraitNames(aGUID);
+        Assert.assertEquals(traitNames.size(), 1);
+        Assert.assertTrue(traitNames.contains(CLASSIFICATION));
+        Assert.assertFalse(traitNames.contains(PII));
+    }
+
+    @Test (expectedExceptions = RepositoryException.class)
+    public void testDeleteTraitForNonExistentEntity() throws Exception {
+        repositoryService.deleteTrait(UUID.randomUUID().toString(), PII);
+        Assert.fail();
+    }
+
+    @Test (expectedExceptions = RepositoryException.class)
+    public void testDeleteTraitForNonExistentTrait() throws Exception {
+        final String aGUID = getGUID();
+        repositoryService.deleteTrait(aGUID, "PCI");
+        Assert.fail();
+    }
+
+    @Test (dependsOnMethods = "testCreateEntity")
+    public void testGetIdFromVertex() throws Exception {
+        Vertex tableVertex = getTableEntityVertex();
+
+        String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+        if (guid == null) {
+            Assert.fail();
+        }
+
+        Id expected = new Id(guid,
+                tableVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), TABLE_TYPE);
+        Assert.assertEquals(repositoryService.getIdFromVertex(TABLE_TYPE, tableVertex),
+                expected);
+    }
+
+    @Test (dependsOnMethods = "testCreateEntity")
+    public void testGetTypeName() throws Exception {
+        Vertex tableVertex = getTableEntityVertex();
+        Assert.assertEquals(repositoryService.getTypeName(tableVertex), TABLE_TYPE);
+    }
+
+/*
+    private void dumpGraph() {
+        TitanGraph graph = titanGraphService.getTitanGraph();
+        System.out.println("*******************Graph Dump****************************");
+        System.out.println("Vertices of " + graph);
+        for (Vertex vertex : graph.getVertices()) {
+            System.out.println(GraphHelper.vertexString(vertex));
+        }
+
+        System.out.println("Edges of " + graph);
+        for (Edge edge : graph.getEdges()) {
+            System.out.println(GraphHelper.edgeString(edge));
+        }
+        System.out.println("*******************Graph Dump****************************");
+    }
+*/
+
+    private void createHiveTypes() throws Exception {
+        HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
+                TypesUtil.createClassTypeDef(DATABASE_TYPE,
+                        ImmutableList.<String>of(),
+                        TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+
+        StructTypeDefinition structTypeDefinition =
+                new StructTypeDefinition("serdeType",
+                        new AttributeDefinition[]{
+                                TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                                TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
+                        });
+
+        EnumValue values[] = {
+                new EnumValue("MANAGED", 1),
+                new EnumValue("EXTERNAL", 2),
+        };
+
+        EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
+        typeSystem.defineEnumType(enumTypeDefinition);
+
+        HierarchicalTypeDefinition<ClassType> columnsDefinition =
+                TypesUtil.createClassTypeDef("column_type",
+                        ImmutableList.<String>of(),
+                        TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
+
+        StructTypeDefinition partitionDefinition =
+                new StructTypeDefinition("partition_type",
+                        new AttributeDefinition[]{
+                                TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        });
+
+        HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
+                TypesUtil.createClassTypeDef(TABLE_TYPE,
+                        ImmutableList.<String>of(),
+                        TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
+                        TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
+                        // enum
+                        new AttributeDefinition("tableType", "tableType",
+                                Multiplicity.REQUIRED, false, null),
+                        // array of strings
+                        new AttributeDefinition("columnNames",
+                                String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
+                                Multiplicity.COLLECTION, false, null),
+                        // array of classes
+                        new AttributeDefinition("columns",
+                                String.format("array<%s>", "column_type"),
+                                Multiplicity.COLLECTION, true, null),
+                        // array of structs
+                        new AttributeDefinition("partitions",
+                                String.format("array<%s>", "partition_type"),
+                                Multiplicity.COLLECTION, true, null),
+                        // struct reference
+                        new AttributeDefinition("serde1",
+                                "serdeType", Multiplicity.REQUIRED, false, null),
+                        new AttributeDefinition("serde2",
+                                "serdeType", Multiplicity.REQUIRED, false, null),
+                        // class reference
+                        new AttributeDefinition("database",
+                                DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
+
+        HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
+                TypesUtil.createTraitTypeDef(CLASSIFICATION,
+                        ImmutableList.<String>of(),
+                        TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
+
+        typeSystem.defineTypes(
+                ImmutableList.of(structTypeDefinition, partitionDefinition),
+                ImmutableList.of(classificationTypeDefinition),
+                ImmutableList.of(databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
+    }
+
+    private ITypedReferenceableInstance createHiveTableInstance(
+            Referenceable databaseInstance) throws Exception {
+        Referenceable tableInstance = new Referenceable(TABLE_TYPE, CLASSIFICATION);
+        tableInstance.set("name", TABLE_NAME);
+        tableInstance.set("description", "bar table");
+        tableInstance.set("type", "managed");
+        tableInstance.set("tableType", 1); // enum
+
+        // refer to an existing class
+        tableInstance.set("database", databaseInstance);
+
+        ArrayList<String> columnNames = new ArrayList<>();
+        columnNames.add("first_name");
+        columnNames.add("last_name");
+        tableInstance.set("columnNames", columnNames);
+
+        Struct traitInstance = (Struct) tableInstance.getTrait(CLASSIFICATION);
+        traitInstance.set("tag", "foundation_etl");
+
+        Struct serde1Instance = new Struct("serdeType");
+        serde1Instance.set("name", "serde1");
+        serde1Instance.set("serde", "serde1");
+        tableInstance.set("serde1", serde1Instance);
+
+        Struct serde2Instance = new Struct("serdeType");
+        serde2Instance.set("name", "serde2");
+        serde2Instance.set("serde", "serde2");
+        tableInstance.set("serde2", serde2Instance);
+
+        ArrayList<Referenceable> columns = new ArrayList<>();
+        for (int index = 0; index < 5; index++) {
+            Referenceable columnInstance = new Referenceable("column_type");
+            columnInstance.set("name", "column_" + index);
+            columnInstance.set("type", "string");
+            columns.add(columnInstance);
+        }
+        tableInstance.set("columns", columns);
+
+        ArrayList<Struct> partitions = new ArrayList<>();
+        for (int index = 0; index < 5; index++) {
+            Struct partitionInstance = new Struct("partition_type");
+            partitionInstance.set("name", "partition_" + index);
+            partitions.add(partitionInstance);
+        }
+        tableInstance.set("partitions", partitions);
+
+        ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
+        return tableType.convert(tableInstance, Multiplicity.REQUIRED);
+    }
 }

repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperTest.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.graph;
-
-import com.google.common.collect.ImmutableList;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanVertex;
-import com.tinkerpop.blueprints.Compare;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.GraphQuery;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.hadoop.metadata.RepositoryMetadataModule;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.Iterator;
-
-@Test
-@Guice(modules = RepositoryMetadataModule.class)
-public class GraphRepoMapperTest {
-
-    private static final String DATABASE_TYPE = "hive_database";
-    private static final String DATABASE_NAME = "foo";
-    private static final String TABLE_TYPE = "hive_table";
-    private static final String TABLE_NAME = "bar";
-
-    @Inject
-    private TitanGraphService titanGraphService;
-    @Inject
-    private GraphBackedMetadataRepository repositoryService;
-
-    private TypeSystem typeSystem;
-
-    @BeforeClass
-    public void setUp() throws Exception {
-        // start the injected graph service
-        titanGraphService.initialize();
-
-        new GraphBackedSearchIndexer(titanGraphService);
-
-        typeSystem = TypeSystem.getInstance();
-        typeSystem.reset();
-
-        createHiveTypes();
-    }
-
-    @AfterMethod
-    public void tearDown() {
-        dumpGraph();
-    }
-
-    @Test
-    public void testSubmitEntity() throws Exception {
-        Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
-        databaseInstance.set("name", DATABASE_NAME);
-        databaseInstance.set("description", "foo database");
-        System.out.println("databaseInstance = " + databaseInstance);
-
-        ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
-        ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
-        System.out.println("db = " + db);
-
-        String dbGUID = repositoryService.createEntity(db, DATABASE_TYPE);
-        System.out.println("added db = " + dbGUID);
-
-        Referenceable dbInstance = new Referenceable(
-                dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
-
-        ITypedReferenceableInstance table = createHiveTableInstance(dbInstance);
-        String tableGUID = repositoryService.createEntity(table, TABLE_TYPE);
-        System.out.println("added table = " + tableGUID);
-    }
-
-    @Test(dependsOnMethods = "testSubmitEntity")
-    public void testGetEntityDefinition() throws Exception {
-        TitanGraph graph = titanGraphService.getTitanGraph();
-        GraphQuery query = graph.query()
-                .has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
-        Iterator<Vertex> results = query.vertices().iterator();
-        // returning one since guid should be unique
-        Vertex tableVertex = results.hasNext() ? results.next() : null;
-        if (tableVertex == null) {
-            Assert.fail();
-        }
-
-        String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
-        if (guid == null) {
-            Assert.fail();
-        }
-
-        ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
-        System.out.println("*** table = " + table);
-    }
-
-    private void dumpGraph() {
-        TitanGraph graph = titanGraphService.getTitanGraph();
-        System.out.println("*******************Graph Dump****************************");
-        System.out.println("Vertices of " + graph);
-        for (Vertex vertex : graph.getVertices()) {
-//            System.out.println(GraphHelper.vertexString(vertex));
-            System.out.print("vertex = " + vertex + " [");
-            TitanVertex titanVertex = (TitanVertex) graph.getVertex(vertex.getId());
-            System.out.print("guid= " + titanVertex.getProperty(Constants.GUID_PROPERTY_KEY));
-            System.out
-                    .print("type= " + titanVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
-            System.out.println("]");
-        }
-
-        System.out.println("Edges of " + graph);
-        for (Edge edge : graph.getEdges()) {
-            System.out.println(GraphHelper.edgeString(edge));
-        }
-        System.out.println("*******************Graph Dump****************************");
-    }
-
-    private void createHiveTypes() throws Exception {
-        HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
-                TypesUtil.createClassTypeDef(DATABASE_TYPE,
-                        ImmutableList.<String>of(),
-                        TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
-                        TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
-        StructTypeDefinition structTypeDefinition =
-                new StructTypeDefinition("serdeType",
-                        new AttributeDefinition[]{
-                                TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
-                                TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
-                        });
-
-        EnumValue values[] = {
-                new EnumValue("MANAGED", 1),
-                new EnumValue("EXTERNAL", 2),
-        };
-
-        EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
-        typeSystem.defineEnumType(enumTypeDefinition);
-
-        HierarchicalTypeDefinition<ClassType> columnsDefinition =
-                TypesUtil.createClassTypeDef("column_type",
-                        ImmutableList.<String>of(),
-                        TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
-                        TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
-
-        StructTypeDefinition partitionDefinition =
-                new StructTypeDefinition("partition_type",
-                        new AttributeDefinition[]{
-                                TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
-                        });
-
-        HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
-                TypesUtil.createClassTypeDef(TABLE_TYPE,
-                        ImmutableList.<String>of(),
-                        TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
-                        TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
-                        TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
-                        // enum
-                        new AttributeDefinition("tableType", "tableType",
-                                Multiplicity.REQUIRED, false, null),
-                        // array of strings
-                        new AttributeDefinition("columnNames",
-                                String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
-                                Multiplicity.COLLECTION, false, null),
-                        // array of classes
-                        new AttributeDefinition("columns",
-                                String.format("array<%s>", "column_type"),
-                                Multiplicity.COLLECTION, true, null),
-                        // array of structs
-                        new AttributeDefinition("partitions",
-                                String.format("array<%s>", "partition_type"),
-                                Multiplicity.COLLECTION, true, null),
-                        // struct reference
-                        new AttributeDefinition("serde1",
-                                "serdeType", Multiplicity.REQUIRED, false, null),
-                        new AttributeDefinition("serde2",
-                                "serdeType", Multiplicity.REQUIRED, false, null),
-                        // class reference
-                        new AttributeDefinition("database",
-                                DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
-
-        HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
-                TypesUtil.createTraitTypeDef("classification",
-                        ImmutableList.<String>of(),
-                        TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
-
-        typeSystem.defineTypes(
-                ImmutableList.of(structTypeDefinition, partitionDefinition),
-                ImmutableList.of(classificationTypeDefinition),
-                ImmutableList.of(databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
-    }
-
-    private ITypedReferenceableInstance createHiveTableInstance(
-            Referenceable databaseInstance) throws Exception {
-/*
-        Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
-        databaseInstance.set("name", DATABASE_NAME);
-        databaseInstance.set("description", "foo database");
-*/
-
-        Referenceable tableInstance = new Referenceable(TABLE_TYPE, "classification");
-        tableInstance.set("name", TABLE_NAME);
-        tableInstance.set("description", "bar table");
-        tableInstance.set("type", "managed");
-        tableInstance.set("tableType", 1); // enum
-
-        // refer to an existing class
-        tableInstance.set("database", databaseInstance);
-
-        ArrayList<String> columnNames = new ArrayList<>();
-        columnNames.add("first_name");
-        columnNames.add("last_name");
-        tableInstance.set("columnNames", columnNames);
-
-        Struct traitInstance = (Struct) tableInstance.getTrait("classification");
-        traitInstance.set("tag", "foundation_etl");
-
-        Struct serde1Instance = new Struct("serdeType");
-        serde1Instance.set("name", "serde1");
-        serde1Instance.set("serde", "serde1");
-        tableInstance.set("serde1", serde1Instance);
-
-        Struct serde2Instance = new Struct("serdeType");
-        serde2Instance.set("name", "serde2");
-        serde2Instance.set("serde", "serde2");
-        tableInstance.set("serde2", serde2Instance);
-
-        ArrayList<Referenceable> columns = new ArrayList<>();
-        for (int index = 0; index < 5; index++) {
-            Referenceable columnInstance = new Referenceable("column_type");
-            columnInstance.set("name", "column_" + index);
-            columnInstance.set("type", "string");
-            columns.add(columnInstance);
-        }
-        tableInstance.set("columns", columns);
-
-        ArrayList<Struct> partitions = new ArrayList<>();
-        for (int index = 0; index < 5; index++) {
-            Struct partitionInstance = new Struct("partition_type");
-            partitionInstance.set("name", "partition_" + index);
-            partitions.add(partitionInstance);
-        }
-        tableInstance.set("partitions", partitions);
-
-        ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
-        return tableType.convert(tableInstance, Multiplicity.REQUIRED);
-    }
-}

typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java

@@ -81,6 +81,10 @@ public class TypeSystem {
         types.put(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE);
     }
 
+    public boolean isRegistered(String typeName) {
+        return types.containsKey(typeName);
+    }
+
     public <T> T getDataType(Class<T> cls, String name) throws MetadataException {
         if (types.containsKey(name)) {
             return cls.cast(types.get(name));

webapp/pom.xml

@@ -52,6 +52,11 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.hadoop.metadata</groupId>
+            <artifactId>metadata-typesystem</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop.metadata</groupId>
             <artifactId>metadata-repository</artifactId>
         </dependency>
 

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java

@@ -123,7 +123,14 @@ public class EntityResource {
 
             return Response.status(status).entity(response).build();
 
-        } catch (Exception e) {
+        } catch (MetadataException e) {
+            LOG.error("An entity with GUID={} does not exist", guid, e);
+            throw new WebApplicationException(e, Response
+                    .status(Response.Status.NOT_FOUND)
+                    .entity(e.getMessage())
+                    .type(MediaType.APPLICATION_JSON)
+                    .build());
+        } catch (JSONException e) {
             LOG.error("Unable to get instance definition for GUID {}", guid, e);
             throw new WebApplicationException(e, Response
                     .status(Response.Status.INTERNAL_SERVER_ERROR)