Skip to content

A
atlas
Commit 82c06b74 by Venkatesh Seetharam
Options

BUG-32815 BUG-32816 BUG-32827 Entity Trait Management Backend API. Contributed…

BUG-32815 BUG-32816 BUG-32827 Entity Trait Management Backend API. Contributed by Venkatesh Seetharam
parent 6ea66f7a
Showing with 839 additions and 412 deletions
repository/src/main/java/org/apache/hadoop/metadata/listener/EntityChangeListener.java
......@@ -33,7 +33,24 @@ public interface EntityChangeListener {
* @param typedInstance a typed instance
* @throws org.apache.hadoop.metadata.MetadataException
*/
void onAdd(String typeName,
ITypedReferenceableInstance typedInstance) throws MetadataException;
void onEntityAdded(String typeName,
ITypedReferenceableInstance typedInstance) throws MetadataException;
/**
* This is upon adding a new trait to a typed instance.
*
* @param guid globally unique identifier for the entity
* @param traitName trait name for the instance that needs to be added to entity
* @throws org.apache.hadoop.metadata.MetadataException
*/
void onTraitAdded(String guid, String traitName) throws MetadataException;
/**
* This is upon deleting a trait from a typed instance.
*
* @param guid globally unique identifier for the entity
* @param traitName trait name for the instance that needs to be deleted from entity
* @throws org.apache.hadoop.metadata.MetadataException
*/
void onTraitDeleted(String guid, String traitName) throws MetadataException;
}
repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java
......@@ -20,6 +20,9 @@ package org.apache.hadoop.metadata.repository;
import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedStruct;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
import org.apache.hadoop.metadata.typesystem.types.IDataType;
import java.util.List;
......@@ -28,12 +31,130 @@ import java.util.List;
*/
public interface MetadataRepository {
/**
* Returns the property key used to store entity type name.
*
* @return property key used to store entity type name.
*/
String getTypeAttributeName();
/**
* Return the property key used to store a given traitName in the repository.
*
* @param dataType data type
* @param traitName trait name
* @return property key used to store a given traitName
*/
String getTraitLabel(IDataType<?> dataType, String traitName);
/**
* Return the property key used to store a given attribute in the repository.
*
* @param dataType data type
* @param aInfo attribute info
* @return property key used to store a given attribute
*/
String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo);
/**
* Return the edge label for a given attribute in the repository.
*
* @param dataType data type
* @param aInfo attribute info
* @return edge label for a given attribute
*/
String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo);
/**
* Creates an entity definition (instance) corresponding to a given type.
*
* @param entity entity (typed instance)
* @param entityType entity type name
* @return a globally unique identifier
* @throws RepositoryException
*/
String createEntity(IReferenceableInstance entity,
String entityType) throws RepositoryException;
/**
* Fetch the complete definition of an entity given its GUID.
*
* @param guid globally unique identifier for the entity
* @return entity (typed instance) definition
* @throws RepositoryException
*/
ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException;
/**
* Gets the list of entities for a given entity type.
*
* @param entityType name of a type which is unique
* @return a list of entity names for the given type
* @throws RepositoryException
*/
List<String> getEntityList(String entityType) throws RepositoryException;
/**
* Deletes an entity definition (instance) corresponding to a given type.
*
* @param guid globally unique identifier for the entity
* @return true if deleted else false
* @throws RepositoryException
*/
// boolean deleteEntity(String guid) throws RepositoryException;
/**
* Updates an entity given its GUID with the attribute name and value.
*
* @param guid globally unique identifier for the entity
* @param attributeName name of the attribute
* @param attributeValue value of the attribute
* @return an entity instance with updated state
* @throws RepositoryException
*/
//ITypedReferenceableInstance updateEntity(String guid, String attributeName,
// String attributeValue) throws RepositoryException;
// Trait management functions
/**
* Gets the list of trait names for a given entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @return a list of trait names for the given entity guid
* @throws RepositoryException
*/
List<String> getTraitNames(String guid) throws RepositoryException;
/**
* Adds a new trait to an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitName trait name for the instance that needs to be added to entity
* @param traitInstance trait instance that needs to be added to entity
* @throws RepositoryException
*/
void addTrait(String guid, String traitName,
ITypedStruct traitInstance) throws RepositoryException;
/**
* Adds a list of traits to an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitInstances list of trait instances that needs to be added to entity
* @return an entity instance with updated traits
* @throws RepositoryException
*/
// ITypedReferenceableInstance addTraits(String guid, Map<String, ITypedStruct> traitInstances)
// throws RepositoryException;
/**
* Deletes a given trait from an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitNameToBeDeleted name of the trait
* @throws RepositoryException
*/
void deleteTrait(String guid,
String traitNameToBeDeleted) throws RepositoryException;
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java
......@@ -33,14 +33,10 @@ public final class Constants {
public static final String ENTITY_TYPE_INDEX = "type_index";
/**
* Data type property key.
*/
// public static final String DATA_TYPE_PROPERTY_KEY = "dataType";
/**
* Trait names property key.
* Trait names property key and index name.
*/
public static final String TRAIT_NAMES_PROPERTY_KEY = "traitNames";
public static final String TRAIT_NAMES_INDEX = "trait_names_index";
public static final String VERSION_PROPERTY_KEY = "version";
public static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -18,6 +18,7 @@
package org.apache.hadoop.metadata.repository.graph;
import com.google.common.base.Preconditions;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
......@@ -50,7 +51,6 @@ import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
......@@ -96,14 +96,17 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return Constants.ENTITY_TYPE_PROPERTY_KEY;
}
@Override
public String getTraitLabel(IDataType<?> dataType, String traitName) {
return dataType.getName() + "." + traitName;
}
@Override
public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) {
return dataType.getName() + "." + aInfo.name;
}
@Override
public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
return dataType.getName() + "." + aInfo.name;
}
......@@ -131,20 +134,27 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
try {
titanGraph.rollback(); // clean up before starting a query
Vertex instanceVertex = GraphHelper.findVertexByGUID(titanGraph, guid);
if (instanceVertex == null) {
LOG.debug("Could not find a vertex for guid {}", guid);
return null;
}
Vertex instanceVertex = getVertexForGUID(guid);
LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
} catch (Exception e) {
} catch (MetadataException e) {
throw new RepositoryException(e);
}
}
private Vertex getVertexForGUID(String guid) throws RepositoryException {
Vertex instanceVertex = GraphHelper.findVertexByGUID(titanGraph, guid);
if (instanceVertex == null) {
LOG.debug("Could not find a vertex for guid={}", guid);
throw new RepositoryException(
"Could not find an entity in the repository for guid: " + guid);
}
return instanceVertex;
}
@Override
public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.info("Retrieving entity list for type={}", entityType);
......@@ -164,18 +174,133 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return entityList;
}
public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
/**
* Gets the list of trait names for a given entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @return a list of trait names for the given entity guid
* @throws RepositoryException
*/
@Override
public List<String> getTraitNames(String guid) throws RepositoryException {
LOG.info("Retrieving trait names for entity={}", guid);
titanGraph.rollback(); // clean up before starting a query
Vertex instanceVertex = getVertexForGUID(guid);
return getTraitNames(instanceVertex);
}
public List<String> getTraitNames(Vertex entityVertex) {
ArrayList<String> traits = new ArrayList<>();
for (TitanProperty property : ((TitanVertex) entityVertex)
.getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
traits.add((String) property.getValue());
}
return traits;
}
/**
* Adds a new trait to an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitInstance trait instance that needs to be added to entity
* @throws RepositoryException
*/
@Override
public void addTrait(String guid, String traitName,
ITypedStruct traitInstance) throws RepositoryException {
Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
LOG.info("Adding a new trait={} for entity={}", traitName, guid);
try {
titanGraph.rollback(); // clean up before starting a query
Vertex instanceVertex = getVertexForGUID(guid);
// add the trait instance as a new vertex
final String typeName = getTypeName(instanceVertex);
instanceToGraphMapper.mapTraitInstanceToVertex(
traitName, traitInstance, getIdFromVertex(typeName, instanceVertex),
typeName, instanceVertex, Collections.<Id, Vertex>emptyMap());
// update the traits in entity once adding trait instance is successful
((TitanVertex) instanceVertex)
.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
titanGraph.commit(); // commit if there are no errors
} catch (MetadataException e) {
titanGraph.rollback();
throw new RepositoryException(e);
}
}
/**
* Deletes a given trait from an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitNameToBeDeleted name of the trait
* @throws RepositoryException
*/
@Override
public void deleteTrait(String guid, String traitNameToBeDeleted)
throws RepositoryException {
LOG.info("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
try {
titanGraph.rollback(); // clean up before starting a query
Vertex instanceVertex = getVertexForGUID(guid);
List<String> traitNames = getTraitNames(instanceVertex);
if (!traitNames.contains(traitNameToBeDeleted)) {
throw new RepositoryException("Could not find trait=" + traitNameToBeDeleted
+ " in the repository for entity: " + guid);
}
final String entityTypeName = getTypeName(instanceVertex);
String relationshipLabel = entityTypeName + "." + traitNameToBeDeleted;
Iterator<Edge> results = instanceVertex.getEdges(
Direction.OUT, relationshipLabel).iterator();
if (results.hasNext()) { // there should only be one edge for this label
final Edge traitEdge = results.next();
final Vertex traitVertex = traitEdge.getVertex(Direction.IN);
// remove the edge to the trait instance from the repository
titanGraph.removeEdge(traitEdge);
if (traitVertex != null) { // remove the trait instance from the repository
titanGraph.removeVertex(traitVertex);
// update the traits in entity once trait removal is successful
traitNames.remove(traitNameToBeDeleted);
updateTraits(instanceVertex, traitNames);
}
titanGraph.commit(); // commit if there are no errors
}
} catch (Exception e) {
titanGraph.rollback();
throw new RepositoryException(e);
}
}
private void updateTraits(Vertex instanceVertex, List<String> traitNames) {
// remove the key
instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
// add it back again
for (String traitName : traitNames) {
((TitanVertex) instanceVertex).addProperty(
Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
}
}
public Id getIdFromVertex(String dataTypeName, Vertex vertex) {
return new Id(
vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
dataTypeName);
}
public List<String> getTraitNames(TitanVertex vertex) {
final String traitNames = vertex.getProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
return traitNames == null
? Collections.<String>emptyList()
: Arrays.asList(traitNames.split(","));
String getTypeName(Vertex instanceVertex) {
return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
}
private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
......@@ -230,7 +355,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
if (id.isAssigned()) { // has a GUID
instanceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
} else {
instanceVertex = GraphHelper.createVertex(titanGraph, typedInstance);
instanceVertex =
GraphHelper.createVertexWithIdentity(titanGraph, typedInstance);
}
idToVertexMap.put(id, instanceVertex);
......@@ -242,7 +368,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private final class TypedInstanceToGraphMapper {
private String mapTypedInstanceToGraph(IReferenceableInstance typedInstance)
throws MetadataException {
throws MetadataException {
EntityProcessor entityProcessor = new EntityProcessor();
try {
......@@ -258,7 +384,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return addDiscoveredInstances(typedInstance, entityProcessor, newTypedInstances);
}
/*
/**
* Step 2: Traverse oldIdToInstance map create newInstances :
* List[ITypedReferenceableInstance]
* - create a ITypedReferenceableInstance.
......@@ -277,9 +403,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
transientInstance, Multiplicity.REQUIRED);
newTypedInstances.add(newInstance);
/*
* Now replace old references with new Ids
*/
// Now replace old references with new Ids
MapIds mapIds = new MapIds(entityProcessor.idToNewIdMap);
new ObjectGraphWalker(typeSystem, mapIds, newTypedInstances).walk();
......@@ -488,8 +612,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
case CLASS:
Id referenceId = (Id) value;
mapClassReferenceAsEdge(
instanceVertex, idToVertexMap,
propertyName, referenceId);
instanceVertex, idToVertexMap, propertyName, referenceId);
break;
default:
......@@ -521,7 +644,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Map<Id, Vertex> idToVertexMap)
throws MetadataException {
// add a new vertex for the struct or trait instance
Vertex structInstanceVertex = GraphHelper.createVertex(
Vertex structInstanceVertex = GraphHelper.createVertexWithoutIdentity(
graphService.getBlueprintsGraph(), structInstance.getTypeName(), id);
LOG.debug("created vertex {} for struct {}", structInstanceVertex, attributeInfo.name);
......@@ -536,18 +659,28 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
ITypedReferenceableInstance typedInstance,
Vertex parentInstanceVertex,
Map<Id, Vertex> idToVertexMap)
throws MetadataException {
throws MetadataException {
// add a new vertex for the struct or trait instance
Vertex traitInstanceVertex = GraphHelper.createVertex(
graphService.getBlueprintsGraph(), traitInstance, typedInstance.getId());
mapTraitInstanceToVertex(traitName, traitInstance, typedInstance.getId(),
typedInstance.getTypeName(), parentInstanceVertex, idToVertexMap);
}
private void mapTraitInstanceToVertex(String traitName, ITypedStruct traitInstance,
Id typedInstanceId, String typedInstanceTypeName,
Vertex parentInstanceVertex,
Map<Id, Vertex> idToVertexMap)
throws MetadataException {
// add a new vertex for the struct or trait instance
Vertex traitInstanceVertex = GraphHelper.createVertexWithoutIdentity(
graphService.getBlueprintsGraph(), traitInstance.getTypeName(), typedInstanceId);
LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
// map all the attributes to this newly created vertex
mapInstanceToVertex(typedInstance.getId(), traitInstance, traitInstanceVertex,
mapInstanceToVertex(typedInstanceId, traitInstance, traitInstanceVertex,
traitInstance.fieldMapping().fields, idToVertexMap);
// add an edge to the newly created vertex from the parent
String relationshipLabel = typedInstance.getTypeName() + "." + traitName;
String relationshipLabel = typedInstanceTypeName + "." + traitName;
GraphHelper.addEdge(
titanGraph, parentInstanceVertex, traitInstanceVertex, relationshipLabel);
}
......@@ -602,16 +735,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid,
Vertex instanceVertex)
throws MetadataException {
throws MetadataException {
LOG.debug("Mapping graph root vertex {} to typed instance for guid {}",
instanceVertex, guid);
String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
List<String> traits = new ArrayList<>();
for (TitanProperty property : ((TitanVertex) instanceVertex)
.getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
traits.add((String) property.getValue());
}
List<String> traits = getTraitNames(instanceVertex);
Id id = new Id(guid,
instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
......@@ -24,7 +24,6 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.typesystem.ITypedInstance;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.persistence.Id;
import org.slf4j.Logger;
......@@ -43,32 +42,34 @@ public final class GraphHelper {
private GraphHelper() {
}
public static Vertex createVertex(Graph graph,
ITypedReferenceableInstance typedInstance) {
return createVertex(graph, typedInstance, typedInstance.getId());
}
public static Vertex createVertexWithIdentity(Graph graph,
ITypedReferenceableInstance typedInstance) {
final Vertex vertexWithIdentity = createVertexWithoutIdentity(
graph, typedInstance.getTypeName(), typedInstance.getId());
// add identity
final String guid = UUID.randomUUID().toString();
vertexWithIdentity.setProperty(Constants.GUID_PROPERTY_KEY, guid);
public static Vertex createVertex(Graph graph,
ITypedInstance typedInstance,
Id typedInstanceId) {
return createVertex(graph, typedInstance.getTypeName(), typedInstanceId);
return vertexWithIdentity;
}
public static Vertex createVertex(Graph graph,
String typeName,
Id typedInstanceId) {
final Vertex instanceVertex = graph.addVertex(null);
// type
instanceVertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
public static Vertex createVertexWithoutIdentity(Graph graph,
String typeName,
Id typedInstanceId) {
final Vertex vertexWithoutIdentity = graph.addVertex(null);
// id
final String guid = UUID.randomUUID().toString();
instanceVertex.setProperty(Constants.GUID_PROPERTY_KEY, guid);
// add type information
vertexWithoutIdentity.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
// add version information
vertexWithoutIdentity.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
// version
instanceVertex.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
// add timestamp information
vertexWithoutIdentity.setProperty(
Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
return instanceVertex;
return vertexWithoutIdentity;
}
public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
......@@ -20,20 +20,21 @@ package org.apache.hadoop.metadata.services;
import com.google.common.base.Preconditions;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.typesystem.TypesDef;
import org.apache.hadoop.metadata.discovery.SearchIndexer;
import org.apache.hadoop.metadata.typesystem.json.Serialization$;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.listener.EntityChangeListener;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedStruct;
import org.apache.hadoop.metadata.typesystem.TypesDef;
import org.apache.hadoop.metadata.typesystem.json.Serialization$;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.types.IDataType;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -81,7 +82,7 @@ public class DefaultMetadataService implements MetadataService {
TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
onAdd(typesAdded);
onTypesAddedToRepo(typesAdded);
JSONObject response = new JSONObject();
for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
......@@ -101,15 +102,9 @@ public class DefaultMetadataService implements MetadataService {
Preconditions.checkNotNull(typeDefinition, "type definition cannot be null");
// verify if the type already exists
IDataType existingTypeDefinition = null;
try {
existingTypeDefinition = typeSystem.getDataType(IDataType.class, typeName);
} catch (MetadataException ignore) {
// do nothing
}
if (existingTypeDefinition != null) {
throw new RepositoryException("type is already defined for : " + typeName);
if (typeSystem.isRegistered(typeName)) {
LOG.error("type is already defined for {}", typeName);
throw new MetadataException("type is already defined for : " + typeName);
}
}
......@@ -145,28 +140,16 @@ public class DefaultMetadataService implements MetadataService {
@Override
public String createEntity(String entityType,
String entityDefinition) throws MetadataException {
try {
validateEntity(entityDefinition, entityType);
Preconditions.checkNotNull(entityDefinition, "entity cannot be null");
Preconditions.checkNotNull(entityType, "entity type cannot be null");
ITypedReferenceableInstance entityInstance =
Serialization$.MODULE$.fromJson(entityDefinition);
final String guid = repository.createEntity(entityInstance, entityType);
ITypedReferenceableInstance entityInstance =
Serialization$.MODULE$.fromJson(entityDefinition);
final String guid = repository.createEntity(entityInstance, entityType);
onAdd(entityType, entityInstance);
onEntityAddedToRepo(entityType, entityInstance);
return guid;
} catch (ParseException e) {
LOG.error("Unable to parse JSON {} for type {}", entityDefinition, entityType, e);
throw new MetadataException("validation failed for: " + entityType);
}
}
private void validateEntity(String entity, String entityType) throws ParseException {
Preconditions.checkNotNull(entity, "entity cannot be null");
Preconditions.checkNotNull(entityType, "entity type cannot be null");
// todo: this is failing for instances but not types
// JSONValue.parseWithException(entity);
return guid;
}
/**
......@@ -180,9 +163,7 @@ public class DefaultMetadataService implements MetadataService {
Preconditions.checkNotNull(guid, "guid cannot be null");
final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
return instance == null
? null
: Serialization$.MODULE$.toJson(instance);
return Serialization$.MODULE$.toJson(instance);
}
/**
......@@ -214,7 +195,65 @@ public class DefaultMetadataService implements MetadataService {
}
}
private void onAdd(Map<String, IDataType> typesAdded) throws MetadataException {
/**
* Gets the list of trait names for a given entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @return a list of trait names for the given entity guid
* @throws MetadataException
*/
@Override
public List<String> getTraitNames(String guid) throws MetadataException {
Preconditions.checkNotNull(guid, "entity GUID cannot be null");
return repository.getTraitNames(guid);
}
/**
* Adds a new trait to an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitName trait name for the instance that needs to be added to entity
* @param traitInstance trait instance that needs to be added to entity
* @throws MetadataException
*/
@Override
public void addTrait(String guid, String traitName,
ITypedStruct traitInstance) throws MetadataException {
Preconditions.checkNotNull(guid, "entity GUID cannot be null");
Preconditions.checkNotNull(traitName, "Trait name cannot be null");
Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
// ensure trait type is already registered with the TS
Preconditions.checkArgument(!typeSystem.isRegistered(traitName),
"trait=%s should be defined in type system before it can be added", traitName);
repository.addTrait(guid, traitName, traitInstance);
onTraitAddedToEntity(guid, traitName);
}
/**
* Deletes a given trait from an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitNameToBeDeleted name of the trait
* @throws MetadataException
*/
@Override
public void deleteTrait(String guid,
String traitNameToBeDeleted) throws MetadataException {
Preconditions.checkNotNull(guid, "entity GUID cannot be null");
Preconditions.checkNotNull(traitNameToBeDeleted, "Trait name cannot be null");
// ensure trait type is already registered with the TS
Preconditions.checkArgument(!typeSystem.isRegistered(traitNameToBeDeleted),
"trait=%s should be defined in type system before it can be deleted",
traitNameToBeDeleted);
repository.deleteTrait(guid, traitNameToBeDeleted);
onTraitDeletedFromEntity(guid, traitNameToBeDeleted);
}
private void onTypesAddedToRepo(Map<String, IDataType> typesAdded) throws MetadataException {
for (TypesChangeListener listener : typesChangeListeners) {
for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
listener.onAdd(entry.getKey(), entry.getValue());
......@@ -230,10 +269,26 @@ public class DefaultMetadataService implements MetadataService {
typesChangeListeners.remove(listener);
}
private void onAdd(String typeName,
ITypedReferenceableInstance typedInstance) throws MetadataException {
private void onEntityAddedToRepo(String typeName,
ITypedReferenceableInstance typedInstance)
throws MetadataException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntityAdded(typeName, typedInstance);
}
}
private void onTraitAddedToEntity(String typeName,
String traitName) throws MetadataException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onTraitAdded(typeName, traitName);
}
}
private void onTraitDeletedFromEntity(String typeName,
String traitName) throws MetadataException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onAdd(typeName, typedInstance);
listener.onTraitDeleted(typeName, traitName);
}
}
......
repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java
......@@ -19,6 +19,7 @@
package org.apache.hadoop.metadata.services;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.typesystem.ITypedStruct;
import org.codehaus.jettison.json.JSONObject;
import java.util.List;
......@@ -78,4 +79,35 @@ public interface MetadataService {
* @return list of entity names for the given type in the repository
*/
List<String> getEntityList(String entityType) throws MetadataException;
// Trait management functions
/**
* Gets the list of trait names for a given entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @return a list of trait names for the given entity guid
* @throws MetadataException
*/
List<String> getTraitNames(String guid) throws MetadataException;
/**
* Adds a new trait to an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitName trait name for the instance that needs to be added to entity
* @param traitInstance trait instance that needs to be added to entity
* @throws MetadataException
*/
void addTrait(String guid, String traitName,
ITypedStruct traitInstance) throws MetadataException;
/**
* Deletes a given trait from an existing entity represented by a guid.
*
* @param guid globally unique identifier for the entity
* @param traitNameToBeDeleted name of the trait
* @throws MetadataException
*/
void deleteTrait(String guid,
String traitNameToBeDeleted) throws MetadataException;
}
repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -173,10 +173,12 @@ public class GraphBackedDiscoveryServiceTest {
{"DB, Table"},
/*{"DB as db1 Table where db1.name = \"Reporting\""},*/
{"DB name = \"Reporting\""},
{"Column as PII"},
{"Table as Dimension"},
{"View as Dimension"},
{"Column as PII select Column.name"},
/*
{"Column where is PII"},
{"Table where is Dimension"},
{"View where is Dimension"},
{"Column where is PII select Column.name"},
*/
{"Column select Column.name"},
{"from Table select Table.name"},
};
......
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -18,24 +18,41 @@
package org.apache.hadoop.metadata.repository.graph;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.TestUtils;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedStruct;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.Struct;
import org.apache.hadoop.metadata.typesystem.persistence.Id;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.EnumValue;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
/**
* GraphBackedMetadataRepository test
......@@ -47,66 +64,388 @@ import java.util.List;
public class GraphBackedMetadataRepositoryTest {
private static final String ENTITY_TYPE = "Department";
private static final String DATABASE_TYPE = "hive_database";
private static final String DATABASE_NAME = "foo";
private static final String TABLE_TYPE = "hive_table";
private static final String TABLE_NAME = "bar";
private static final String CLASSIFICATION = "classification";
private static final String PII = "PII";
@Inject
private TitanGraphService titanGraphService;
@Inject
private GraphBackedMetadataRepository repositoryService;
private TypeSystem ts;
private TypeSystem typeSystem;
private String guid;
@BeforeClass
public void setUp() throws Exception {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
// start the injected graph service
titanGraphService.initialize();
new GraphBackedSearchIndexer(titanGraphService);
ts = TypeSystem.getInstance();
TestUtils.defineDeptEmployeeTypes(typeSystem);
createHiveTypes();
}
TestUtils.defineDeptEmployeeTypes(ts);
/*
@AfterMethod
public void tearDown() {
dumpGraph();
}
*/
@Test
public void testSubmitEntity() throws Exception {
Referenceable hrDept = TestUtils.createDeptEg1(ts);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
guid = repositoryService.createEntity(hrDept2, ENTITY_TYPE);
Assert.assertNotNull(guid);
dumpGraph();
}
private void dumpGraph() {
TitanGraph graph = titanGraphService.getTitanGraph();
for (Vertex v : graph.getVertices()) {
System.out.println("****v = " + GraphHelper.vertexString(v));
for (Edge e : v.getEdges(Direction.OUT)) {
System.out.println("****e = " + GraphHelper.edgeString(e));
}
}
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetEntityDefinition() throws Exception {
public void testGetEntityDefinitionForDepartment() throws Exception {
ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid);
Assert.assertNotNull(entity);
}
@Test
@Test (expectedExceptions = RepositoryException.class)
public void testGetEntityDefinitionNonExistent() throws Exception {
ITypedReferenceableInstance entity = repositoryService.getEntityDefinition("blah");
Assert.assertNull(entity);
repositoryService.getEntityDefinition("blah");
Assert.fail();
}
@Test(enabled = false)
@Test
public void testGetEntityList() throws Exception {
List<String> entityList = repositoryService.getEntityList(ENTITY_TYPE);
System.out.println("entityList = " + entityList);
Assert.assertNotNull(entityList);
Assert.assertEquals(entityList.size(), 1); // one department
}
@Test
public void testGetTypeAttributeName() throws Exception {
Assert.assertEquals(
repositoryService.getTypeAttributeName(), Constants.ENTITY_TYPE_PROPERTY_KEY);
}
@Test (dependsOnMethods = "testSubmitEntity")
public void testGetTraitLabel() throws Exception {
Assert.assertEquals(repositoryService.getTraitLabel(
typeSystem.getDataType(ClassType.class, TABLE_TYPE), CLASSIFICATION),
TABLE_TYPE + "." + CLASSIFICATION);
}
@Test
public void testCreateEntity() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database");
System.out.println("databaseInstance = " + databaseInstance);
ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
System.out.println("db = " + db);
String dbGUID = repositoryService.createEntity(db, DATABASE_TYPE);
System.out.println("added db = " + dbGUID);
Referenceable dbInstance = new Referenceable(
dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
ITypedReferenceableInstance table = createHiveTableInstance(dbInstance);
String tableGUID = repositoryService.createEntity(table, TABLE_TYPE);
System.out.println("added table = " + tableGUID);
}
@Test(dependsOnMethods = "testCreateEntity")
public void testGetEntityDefinition() throws Exception {
String guid = getGUID();
ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
System.out.println("*** table = " + table);
}
private String getGUID() {
Vertex tableVertex = getTableEntityVertex();
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
if (guid == null) {
Assert.fail();
}
return guid;
}
private Vertex getTableEntityVertex() {
TitanGraph graph = titanGraphService.getTitanGraph();
GraphQuery query = graph.query()
.has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
Vertex tableVertex = results.hasNext() ? results.next() : null;
if (tableVertex == null) {
Assert.fail();
}
return tableVertex;
}
@Test (dependsOnMethods = "testCreateEntity")
public void testGetTraitNames() throws Exception {
final List<String> traitNames = repositoryService.getTraitNames(getGUID());
Assert.assertEquals(traitNames.size(), 1);
Assert.assertEquals(traitNames, Arrays.asList(new String[]{CLASSIFICATION}));
}
@Test
public void testGetTraitNamesForEmptyTraits() throws Exception {
final List<String> traitNames = repositoryService.getTraitNames(guid);
Assert.assertEquals(traitNames.size(), 0);
}
@Test (expectedExceptions = RepositoryException.class)
public void testGetTraitNamesForBadEntity() throws Exception {
repositoryService.getTraitNames(UUID.randomUUID().toString());
Assert.fail();
}
@Test (dependsOnMethods = "testGetTraitNames")
public void testAddTrait() throws Exception {
final String aGUID = getGUID();
List<String> traitNames = repositoryService.getTraitNames(aGUID);
System.out.println("traitNames = " + traitNames);
Assert.assertEquals(traitNames.size(), 1);
Assert.assertTrue(traitNames.contains(CLASSIFICATION));
Assert.assertFalse(traitNames.contains(PII));
HierarchicalTypeDefinition<TraitType> piiTrait =
TypesUtil.createTraitTypeDef(PII, ImmutableList.<String>of());
TraitType traitType = typeSystem.defineTraitType(piiTrait);
ITypedStruct traitInstance = traitType.createInstance();
repositoryService.addTrait(aGUID, PII, traitInstance);
// refresh trait names
traitNames = repositoryService.getTraitNames(aGUID);
Assert.assertEquals(traitNames.size(), 2);
Assert.assertTrue(traitNames.contains(PII));
Assert.assertTrue(traitNames.contains(CLASSIFICATION));
}
@Test (expectedExceptions = NullPointerException.class)
public void testAddTraitWithNullInstance() throws Exception {
repositoryService.addTrait(getGUID(), PII, null);
Assert.fail();
}
@Test (dependsOnMethods = "testAddTrait", expectedExceptions = RepositoryException.class)
public void testAddTraitForBadEntity() throws Exception {
TraitType traitType = typeSystem.getDataType(TraitType.class, PII);
ITypedStruct traitInstance = traitType.createInstance();
repositoryService.addTrait(UUID.randomUUID().toString(), PII, traitInstance);
Assert.fail();
}
@Test (dependsOnMethods = "testAddTrait")
public void testDeleteTrait() throws Exception {
final String aGUID = getGUID();
List<String> traitNames = repositoryService.getTraitNames(aGUID);
Assert.assertEquals(traitNames.size(), 2);
Assert.assertTrue(traitNames.contains(PII));
Assert.assertTrue(traitNames.contains(CLASSIFICATION));
repositoryService.deleteTrait(aGUID, PII);
// refresh trait names
traitNames = repositoryService.getTraitNames(aGUID);
Assert.assertEquals(traitNames.size(), 1);
Assert.assertTrue(traitNames.contains(CLASSIFICATION));
Assert.assertFalse(traitNames.contains(PII));
}
@Test (expectedExceptions = RepositoryException.class)
public void testDeleteTraitForNonExistentEntity() throws Exception {
repositoryService.deleteTrait(UUID.randomUUID().toString(), PII);
Assert.fail();
}
@Test (expectedExceptions = RepositoryException.class)
public void testDeleteTraitForNonExistentTrait() throws Exception {
final String aGUID = getGUID();
repositoryService.deleteTrait(aGUID, "PCI");
Assert.fail();
}
@Test (dependsOnMethods = "testCreateEntity")
public void testGetIdFromVertex() throws Exception {
Vertex tableVertex = getTableEntityVertex();
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
if (guid == null) {
Assert.fail();
}
Id expected = new Id(guid,
tableVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), TABLE_TYPE);
Assert.assertEquals(repositoryService.getIdFromVertex(TABLE_TYPE, tableVertex),
expected);
}
@Test (dependsOnMethods = "testCreateEntity")
public void testGetTypeName() throws Exception {
Vertex tableVertex = getTableEntityVertex();
Assert.assertEquals(repositoryService.getTypeName(tableVertex), TABLE_TYPE);
}
/*
private void dumpGraph() {
TitanGraph graph = titanGraphService.getTitanGraph();
System.out.println("*******************Graph Dump****************************");
System.out.println("Vertices of " + graph);
for (Vertex vertex : graph.getVertices()) {
System.out.println(GraphHelper.vertexString(vertex));
}
System.out.println("Edges of " + graph);
for (Edge edge : graph.getEdges()) {
System.out.println(GraphHelper.edgeString(edge));
}
System.out.println("*******************Graph Dump****************************");
}
*/
private void createHiveTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
TypesUtil.createClassTypeDef(DATABASE_TYPE,
ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition =
new StructTypeDefinition("serdeType",
new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
});
EnumValue values[] = {
new EnumValue("MANAGED", 1),
new EnumValue("EXTERNAL", 2),
};
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
typeSystem.defineEnumType(enumTypeDefinition);
HierarchicalTypeDefinition<ClassType> columnsDefinition =
TypesUtil.createClassTypeDef("column_type",
ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
StructTypeDefinition partitionDefinition =
new StructTypeDefinition("partition_type",
new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
TypesUtil.createClassTypeDef(TABLE_TYPE,
ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
// enum
new AttributeDefinition("tableType", "tableType",
Multiplicity.REQUIRED, false, null),
// array of strings
new AttributeDefinition("columnNames",
String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
Multiplicity.COLLECTION, false, null),
// array of classes
new AttributeDefinition("columns",
String.format("array<%s>", "column_type"),
Multiplicity.COLLECTION, true, null),
// array of structs
new AttributeDefinition("partitions",
String.format("array<%s>", "partition_type"),
Multiplicity.COLLECTION, true, null),
// struct reference
new AttributeDefinition("serde1",
"serdeType", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("serde2",
"serdeType", Multiplicity.REQUIRED, false, null),
// class reference
new AttributeDefinition("database",
DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
TypesUtil.createTraitTypeDef(CLASSIFICATION,
ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
typeSystem.defineTypes(
ImmutableList.of(structTypeDefinition, partitionDefinition),
ImmutableList.of(classificationTypeDefinition),
ImmutableList.of(databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
}
private ITypedReferenceableInstance createHiveTableInstance(
Referenceable databaseInstance) throws Exception {
Referenceable tableInstance = new Referenceable(TABLE_TYPE, CLASSIFICATION);
tableInstance.set("name", TABLE_NAME);
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
tableInstance.set("tableType", 1); // enum
// refer to an existing class
tableInstance.set("database", databaseInstance);
ArrayList<String> columnNames = new ArrayList<>();
columnNames.add("first_name");
columnNames.add("last_name");
tableInstance.set("columnNames", columnNames);
Struct traitInstance = (Struct) tableInstance.getTrait(CLASSIFICATION);
traitInstance.set("tag", "foundation_etl");
Struct serde1Instance = new Struct("serdeType");
serde1Instance.set("name", "serde1");
serde1Instance.set("serde", "serde1");
tableInstance.set("serde1", serde1Instance);
Struct serde2Instance = new Struct("serdeType");
serde2Instance.set("name", "serde2");
serde2Instance.set("serde", "serde2");
tableInstance.set("serde2", serde2Instance);
ArrayList<Referenceable> columns = new ArrayList<>();
for (int index = 0; index < 5; index++) {
Referenceable columnInstance = new Referenceable("column_type");
columnInstance.set("name", "column_" + index);
columnInstance.set("type", "string");
columns.add(columnInstance);
}
tableInstance.set("columns", columns);
ArrayList<Struct> partitions = new ArrayList<>();
for (int index = 0; index < 5; index++) {
Struct partitionInstance = new Struct("partition_type");
partitionInstance.set("name", "partition_" + index);
partitions.add(partitionInstance);
}
tableInstance.set("partitions", partitions);
ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
}
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.Struct;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.EnumValue;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Iterator;
@Test
@Guice(modules = RepositoryMetadataModule.class)
public class GraphRepoMapperTest {
private static final String DATABASE_TYPE = "hive_database";
private static final String DATABASE_NAME = "foo";
private static final String TABLE_TYPE = "hive_table";
private static final String TABLE_NAME = "bar";
@Inject
private TitanGraphService titanGraphService;
@Inject
private GraphBackedMetadataRepository repositoryService;
private TypeSystem typeSystem;
@BeforeClass
public void setUp() throws Exception {
// start the injected graph service
titanGraphService.initialize();
new GraphBackedSearchIndexer(titanGraphService);
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
createHiveTypes();
}
@AfterMethod
public void tearDown() {
dumpGraph();
}
@Test
public void testSubmitEntity() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database");
System.out.println("databaseInstance = " + databaseInstance);
ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
System.out.println("db = " + db);
String dbGUID = repositoryService.createEntity(db, DATABASE_TYPE);
System.out.println("added db = " + dbGUID);
Referenceable dbInstance = new Referenceable(
dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
ITypedReferenceableInstance table = createHiveTableInstance(dbInstance);
String tableGUID = repositoryService.createEntity(table, TABLE_TYPE);
System.out.println("added table = " + tableGUID);
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetEntityDefinition() throws Exception {
TitanGraph graph = titanGraphService.getTitanGraph();
GraphQuery query = graph.query()
.has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
Vertex tableVertex = results.hasNext() ? results.next() : null;
if (tableVertex == null) {
Assert.fail();
}
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
if (guid == null) {
Assert.fail();
}
ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
System.out.println("*** table = " + table);
}
private void dumpGraph() {
TitanGraph graph = titanGraphService.getTitanGraph();
System.out.println("*******************Graph Dump****************************");
System.out.println("Vertices of " + graph);
for (Vertex vertex : graph.getVertices()) {
// System.out.println(GraphHelper.vertexString(vertex));
System.out.print("vertex = " + vertex + " [");
TitanVertex titanVertex = (TitanVertex) graph.getVertex(vertex.getId());
System.out.print("guid= " + titanVertex.getProperty(Constants.GUID_PROPERTY_KEY));
System.out
.print("type= " + titanVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
System.out.println("]");
}
System.out.println("Edges of " + graph);
for (Edge edge : graph.getEdges()) {
System.out.println(GraphHelper.edgeString(edge));
}
System.out.println("*******************Graph Dump****************************");
}
private void createHiveTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
TypesUtil.createClassTypeDef(DATABASE_TYPE,
ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition =
new StructTypeDefinition("serdeType",
new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
});
EnumValue values[] = {
new EnumValue("MANAGED", 1),
new EnumValue("EXTERNAL", 2),
};
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
typeSystem.defineEnumType(enumTypeDefinition);
HierarchicalTypeDefinition<ClassType> columnsDefinition =
TypesUtil.createClassTypeDef("column_type",
ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
StructTypeDefinition partitionDefinition =
new StructTypeDefinition("partition_type",
new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
TypesUtil.createClassTypeDef(TABLE_TYPE,
ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
// enum
new AttributeDefinition("tableType", "tableType",
Multiplicity.REQUIRED, false, null),
// array of strings
new AttributeDefinition("columnNames",
String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
Multiplicity.COLLECTION, false, null),
// array of classes
new AttributeDefinition("columns",
String.format("array<%s>", "column_type"),
Multiplicity.COLLECTION, true, null),
// array of structs
new AttributeDefinition("partitions",
String.format("array<%s>", "partition_type"),
Multiplicity.COLLECTION, true, null),
// struct reference
new AttributeDefinition("serde1",
"serdeType", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("serde2",
"serdeType", Multiplicity.REQUIRED, false, null),
// class reference
new AttributeDefinition("database",
DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
TypesUtil.createTraitTypeDef("classification",
ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
typeSystem.defineTypes(
ImmutableList.of(structTypeDefinition, partitionDefinition),
ImmutableList.of(classificationTypeDefinition),
ImmutableList.of(databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
}
private ITypedReferenceableInstance createHiveTableInstance(
Referenceable databaseInstance) throws Exception {
/*
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database");
*/
Referenceable tableInstance = new Referenceable(TABLE_TYPE, "classification");
tableInstance.set("name", TABLE_NAME);
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
tableInstance.set("tableType", 1); // enum
// refer to an existing class
tableInstance.set("database", databaseInstance);
ArrayList<String> columnNames = new ArrayList<>();
columnNames.add("first_name");
columnNames.add("last_name");
tableInstance.set("columnNames", columnNames);
Struct traitInstance = (Struct) tableInstance.getTrait("classification");
traitInstance.set("tag", "foundation_etl");
Struct serde1Instance = new Struct("serdeType");
serde1Instance.set("name", "serde1");
serde1Instance.set("serde", "serde1");
tableInstance.set("serde1", serde1Instance);
Struct serde2Instance = new Struct("serdeType");
serde2Instance.set("name", "serde2");
serde2Instance.set("serde", "serde2");
tableInstance.set("serde2", serde2Instance);
ArrayList<Referenceable> columns = new ArrayList<>();
for (int index = 0; index < 5; index++) {
Referenceable columnInstance = new Referenceable("column_type");
columnInstance.set("name", "column_" + index);
columnInstance.set("type", "string");
columns.add(columnInstance);
}
tableInstance.set("columns", columns);
ArrayList<Struct> partitions = new ArrayList<>();
for (int index = 0; index < 5; index++) {
Struct partitionInstance = new Struct("partition_type");
partitionInstance.set("name", "partition_" + index);
partitions.add(partitionInstance);
}
tableInstance.set("partitions", partitions);
ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
}
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
......@@ -81,6 +81,10 @@ public class TypeSystem {
types.put(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE);
}
public boolean isRegistered(String typeName) {
return types.containsKey(typeName);
}
public <T> T getDataType(Class<T> cls, String name) throws MetadataException {
if (types.containsKey(name)) {
return cls.cast(types.get(name));
......
webapp/pom.xml
......@@ -52,6 +52,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-repository</artifactId>
</dependency>
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java
......@@ -123,7 +123,14 @@ public class EntityResource {
return Response.status(status).entity(response).build();
} catch (Exception e) {
} catch (MetadataException e) {
LOG.error("An entity with GUID={} does not exist", guid, e);
throw new WebApplicationException(e, Response
.status(Response.Status.NOT_FOUND)
.entity(e.getMessage())
.type(MediaType.APPLICATION_JSON)
.build());
} catch (JSONException e) {
LOG.error("Unable to get instance definition for GUID {}", guid, e);
throw new WebApplicationException(e, Response
.status(Response.Status.INTERNAL_SERVER_ERROR)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment