Skip to content

A
atlas
Commit ea38da10 by Venkatesh Seetharam
Options

ISSUE-38 Map type to graph with type prefixes to enable search. Contributed by Venkatesh Seetharam

parent 386d8d38
Showing with 238 additions and 130 deletions
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
......@@ -29,13 +29,13 @@ import com.google.inject.Scopes;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.hadoop.metadata.services.DefaultMetadataService;
import org.apache.hadoop.metadata.services.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.services.GraphProvider;
import org.apache.hadoop.metadata.services.GraphService;
import org.apache.hadoop.metadata.services.GraphServiceConfigurator;
import org.apache.hadoop.metadata.services.MetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.apache.hadoop.metadata.repository.graph.GraphServiceConfigurator;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.services.MetadataService;
import org.apache.hadoop.metadata.services.TitanGraphProvider;
import org.apache.hadoop.metadata.repository.graph.TitanGraphProvider;
/**
* Guice module for Repository module.
......
repository/src/main/java/org/apache/hadoop/metadata/listener/TypedInstanceChangeListener.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.listener;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.types.IDataType;
/**
* Typed instance change notification listener.
*/
public interface TypedInstanceChangeListener {
/**
* This is upon adding a new typed instance to the repository.
*
* @param typeName type name
* @param typedInstance a typed instance
* @throws org.apache.hadoop.metadata.MetadataException
*/
void onAdd(String typeName,
ITypedReferenceableInstance typedInstance) throws MetadataException;
}
repository/src/main/java/org/apache/hadoop/metadata/services/MetadataRepository.java → repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java
......@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.metadata.services;
package org.apache.hadoop.metadata.repository;
import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
final class Constants {
private Constants() {
}
static final String GUID_PROPERTY_KEY = "GUID";
static final String ENTITY_TYPE_PROPERTY_KEY = "typeName";
static final String VERSION_PROPERTY_KEY = "version";
static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
}
repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepository.java → repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -16,14 +16,13 @@
* limitations under the License.
*/
package org.apache.hadoop.metadata.services;
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.IReferenceableInstance;
......@@ -31,6 +30,7 @@ import org.apache.hadoop.metadata.ITypedInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.ITypedStruct;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.storage.Id;
import org.apache.hadoop.metadata.storage.MapIds;
import org.apache.hadoop.metadata.storage.RepositoryException;
......@@ -43,7 +43,6 @@ import org.apache.hadoop.metadata.types.ObjectGraphWalker;
import org.apache.hadoop.metadata.types.StructType;
import org.apache.hadoop.metadata.types.TraitType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.apache.hadoop.metadata.util.GraphUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -54,7 +53,6 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
......@@ -69,12 +67,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private static final Logger LOG =
LoggerFactory.getLogger(GraphBackedMetadataRepository.class);
private static final String GUID_PROPERTY_KEY = "GUID";
private static final String ENTITY_TYPE_PROPERTY_KEY = "entityType";
private static final String VERSION_PROPERTY_KEY = "version";
private static final String TRAIT_PROPERTY_SUFFIX = "trait.";
private final AtomicInteger ID_SEQ = new AtomicInteger(0);
private final TypedInstanceToGraphMapper instanceToGraphMapper
......@@ -121,9 +113,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
@Override
public String createEntity(IReferenceableInstance entity,
String entityType) throws RepositoryException {
LOG.info("adding entity={} type={}", entity, entityType);
public String createEntity(IReferenceableInstance typedInstance,
String typeName) throws RepositoryException {
LOG.info("adding entity={} type={}", typedInstance, typeName);
final TransactionalGraph transactionalGraph = graphService.getTransactionalGraph();
try {
......@@ -131,7 +123,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
transactionalGraph.rollback();
return instanceToGraphMapper.mapTypedInstanceToGraph(entity, transactionalGraph);
return instanceToGraphMapper.mapTypedInstanceToGraph(typedInstance, transactionalGraph);
} catch (MetadataException e) {
transactionalGraph.rollback();
......@@ -147,10 +139,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
final Graph graph = graphService.getBlueprintsGraph();
try {
GraphQuery query = graph.query().has(GUID_PROPERTY_KEY, guid);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since name/type is unique
Vertex instanceVertex = results.hasNext() ? results.next() : null;
Vertex instanceVertex = GraphUtils.findVertex(graph, Constants.GUID_PROPERTY_KEY, guid);
if (instanceVertex == null) {
return null;
}
......@@ -159,6 +148,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} catch (Exception e) {
throw new RepositoryException(e);
} finally {
GraphUtils.dumpToLog(graph);
}
}
......@@ -216,14 +207,14 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
List<ITypedReferenceableInstance> newInstances) {
for (ITypedReferenceableInstance typedInstance : newInstances) {
final Vertex instanceVertex = transactionalGraph.addVertex(null);
instanceVertex.setProperty(ENTITY_TYPE_PROPERTY_KEY, typedInstance.getTypeName());
instanceVertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typedInstance.getTypeName());
// entityVertex.setProperty("entityName", instance.getString("name"));
final String guid = UUID.randomUUID().toString();
instanceVertex.setProperty(GUID_PROPERTY_KEY, guid);
instanceVertex.setProperty(Constants.GUID_PROPERTY_KEY, guid);
final Id typedInstanceId = typedInstance.getId();
instanceVertex.setProperty(VERSION_PROPERTY_KEY, typedInstanceId.version);
instanceVertex.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
idToVertexMap.put(typedInstanceId, instanceVertex);
}
......@@ -309,14 +300,16 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
// add the attributes for the trait instance
instanceVertex.setProperty(TRAIT_PROPERTY_SUFFIX + traitName, traitName);
final String vertexPropertyName = typedInstance.getTypeName() + "." + traitName;
instanceVertex.setProperty(vertexPropertyName, traitName);
addInstanceToVertex(traitInstance, instanceVertex,
traitInstance.fieldMapping().fields,
entityProcessor.idToVertexMap);
}
if (typedInstance.getId() == entity.getId()) {
guid = instanceVertex.getProperty(GUID_PROPERTY_KEY);
guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
}
}
......@@ -329,7 +322,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
for (AttributeInfo attributeInfo : fields.values()) {
System.out.println("*** attributeInfo = " + attributeInfo);
final IDataType dataType = attributeInfo.dataType();
Object attributeValue = typedInstance.get(attributeInfo.name);
final Object attributeValue = typedInstance.get(attributeInfo.name);
final String vertexPropertyName =
typedInstance.getTypeName() + "." + attributeInfo.name;
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
......@@ -337,7 +332,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break;
case ENUM:
addToVertex(instanceVertex, attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getInt(attributeInfo.name));
break;
......@@ -365,7 +360,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Id id = (Id) typedInstance.get(attributeInfo.name);
if (id != null) {
Vertex referenceVertex = idToVertexMap.get(id);
GraphUtils.addEdge(instanceVertex, referenceVertex, id.id);
GraphUtils.addEdge(instanceVertex, referenceVertex,
Constants.GUID_PROPERTY_KEY, id.className);
}
break;
......@@ -382,42 +378,40 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return;
}
final String vertexPropertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getString(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getShort(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getInt(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getBigInt(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getBoolean(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getByte(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getLong(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getFloat(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getDouble(attributeInfo.name));
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
instanceVertex.setProperty(attributeInfo.name,
instanceVertex.setProperty(vertexPropertyName,
typedInstance.getBigDecimal(attributeInfo.name));
}
}
public void addToVertex(Vertex instanceVertex, String name, int value) {
instanceVertex.setProperty(name, value);
}
}
private final class GraphToTypedInstanceMapper {
......@@ -426,7 +420,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Vertex instanceVertex)
throws MetadataException {
String typeName = instanceVertex.getProperty(ENTITY_TYPE_PROPERTY_KEY);
String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
List<String> traits = new ArrayList<>();
for (TitanProperty property : ((TitanVertex) instanceVertex).getProperties( "traits")) {
traits.add((String) property.getValue());
......@@ -448,6 +442,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
for (AttributeInfo attributeInfo : fields.values()) {
System.out.println("*** attributeInfo = " + attributeInfo);
final IDataType dataType = attributeInfo.dataType();
final String vertexPropertyName =
typedInstance.getTypeName() + "." + attributeInfo.name;
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
......@@ -459,7 +455,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
// EnumType.class, attributeInfo.name);
// todo - is this enough
typedInstance.setInt(attributeInfo.name,
instanceVertex.<Integer>getProperty(attributeInfo.name));
instanceVertex.<Integer>getProperty(vertexPropertyName));
break;
case ARRAY:
......@@ -495,9 +491,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Id referenceId = null;
for (Edge edge : instanceVertex.getEdges(Direction.IN)) {
final Vertex vertex = edge.getVertex(Direction.OUT);
if (vertex.getProperty(ENTITY_TYPE_PROPERTY_KEY).equals(attributeInfo.name)) {
referenceId = new Id(vertex.<String>getProperty(GUID_PROPERTY_KEY),
vertex.<Integer>getProperty("version"),
if (vertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY).equals(attributeInfo.name)) {
referenceId = new Id(
vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
attributeInfo.name);
break;
}
......@@ -517,40 +514,41 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private void mapVertexToInstance(Vertex instanceVertex,
ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException {
if (instanceVertex.getProperty(attributeInfo.name) == null) {
final String vertexPropertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
if (instanceVertex.getProperty(vertexPropertyName) == null) {
return;
}
if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
typedInstance.setString(attributeInfo.name,
instanceVertex.<String>getProperty(attributeInfo.name));
instanceVertex.<String>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
typedInstance.setShort(attributeInfo.name,
instanceVertex.<Short>getProperty(attributeInfo.name));
instanceVertex.<Short>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
typedInstance.setInt(attributeInfo.name,
instanceVertex.<Integer>getProperty(attributeInfo.name));
instanceVertex.<Integer>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
typedInstance.setBigInt(attributeInfo.name,
instanceVertex.<BigInteger>getProperty(attributeInfo.name));
instanceVertex.<BigInteger>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
typedInstance.setBoolean(attributeInfo.name,
instanceVertex.<Boolean>getProperty(attributeInfo.name));
instanceVertex.<Boolean>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
typedInstance.setByte(attributeInfo.name,
instanceVertex.<Byte>getProperty(attributeInfo.name));
instanceVertex.<Byte>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
typedInstance.setLong(attributeInfo.name,
instanceVertex.<Long>getProperty(attributeInfo.name));
instanceVertex.<Long>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
typedInstance.setFloat(attributeInfo.name,
instanceVertex.<Float>getProperty(attributeInfo.name));
instanceVertex.<Float>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
typedInstance.setDouble(attributeInfo.name,
instanceVertex.<Double>getProperty(attributeInfo.name));
instanceVertex.<Double>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
typedInstance.setBigDecimal(attributeInfo.name,
instanceVertex.<BigDecimal>getProperty(attributeInfo.name));
instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
}
}
}
......
repository/src/main/java/org/apache/hadoop/metadata/services/GraphProvider.java → repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java
package org.apache.hadoop.metadata.services;
package org.apache.hadoop.metadata.repository.graph;
import org.apache.commons.configuration.ConfigurationException;
......
repository/src/main/java/org/apache/hadoop/metadata/services/GraphService.java → repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphService.java
......@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.metadata.services;
package org.apache.hadoop.metadata.repository.graph;
import java.util.Set;
......
repository/src/main/java/org/apache/hadoop/metadata/services/GraphServiceConfigurator.java → repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphServiceConfigurator.java
package org.apache.hadoop.metadata.services;
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
......
repository/src/main/java/org/apache/hadoop/metadata/util/GraphUtils.java → repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphUtils.java
package org.apache.hadoop.metadata.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
/**
* Utility class for graph operations.
......@@ -23,37 +35,37 @@ public final class GraphUtils {
private static final Logger LOG = LoggerFactory.getLogger(GraphUtils.class);
private static final String GUID_PROPERTY_KEY = "guid";
private static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
private GraphUtils() {
}
public static Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
return addEdge(fromVertex, toVertex, edgeLabel, null);
public static Edge addEdge(Vertex fromVertex, Vertex toVertex,
String vertexPropertyKey, String edgeLabel) {
return addEdge(fromVertex, toVertex, vertexPropertyKey, edgeLabel, null);
}
public static Edge addEdge(Vertex fromVertex, Vertex toVertex,
String edgeLabel, String timestamp) {
Edge edge = findEdge(fromVertex, toVertex, edgeLabel);
String vertexPropertyKey, String edgeLabel, String timestamp) {
Edge edge = findEdge(fromVertex, toVertex, vertexPropertyKey, edgeLabel);
Edge edgeToVertex = edge != null ? edge : fromVertex.addEdge(edgeLabel, toVertex);
if (timestamp != null) {
edgeToVertex.setProperty(TIMESTAMP_PROPERTY_KEY, timestamp);
edgeToVertex.setProperty(Constants.TIMESTAMP_PROPERTY_KEY, timestamp);
}
return edgeToVertex;
}
public static Edge findEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
return findEdge(fromVertex, toVertex.getProperty(GUID_PROPERTY_KEY), edgeLabel);
public static Edge findEdge(Vertex fromVertex, Vertex toVertex,
String vertexPropertyKey, String edgeLabel) {
return findEdge(fromVertex, toVertex.getProperty(vertexPropertyKey),
vertexPropertyKey, edgeLabel);
}
public static Edge findEdge(Vertex fromVertex, Object toVertexName, String edgeLabel) {
public static Edge findEdge(Vertex fromVertex, Object toVertexName,
String vertexPropertyKey, String edgeLabel) {
Edge edgeToFind = null;
for (Edge edge : fromVertex.getEdges(Direction.OUT, edgeLabel)) {
if (edge.getVertex(Direction.IN).getProperty(
GUID_PROPERTY_KEY).equals(toVertexName)) {
if (edge.getVertex(Direction.IN).getProperty(vertexPropertyKey).equals(toVertexName)) {
edgeToFind = edge;
break;
}
......@@ -63,36 +75,15 @@ public final class GraphUtils {
}
public static Vertex findVertex(Graph blueprintsGraph,
String guid) {
LOG.debug("Finding vertex for: guid={}", guid);
GraphQuery query = blueprintsGraph.query().has("guid", guid);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since name/type is unique
return results.hasNext() ? results.next() : null;
}
public static Vertex findVertex(Graph blueprintsGraph,
String entityName, String entityType) {
LOG.debug("Finding vertex for: name={}, type={}", entityName, entityType);
String key, String value) {
LOG.debug("Finding vertex for key={}, value={}", key, value);
GraphQuery query = blueprintsGraph.query()
.has("entityName", entityName)
.has("entityType", entityType);
GraphQuery query = blueprintsGraph.query().has(key, value);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since name/type is unique
return results.hasNext() ? results.next() : null;
}
public static Map<String, String> extractProperties(Vertex entityVertex) {
Map<String, String> properties = new HashMap<>();
for (String key : entityVertex.getPropertyKeys()) {
properties.put(key, String.valueOf(entityVertex.getProperty(key)));
}
return properties;
}
public static String vertexString(final Vertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
......@@ -104,10 +95,6 @@ public final class GraphUtils {
return "v[" + vertex.getId() + "], Properties[" + properties + "]";
}
public static JSONObject vertexJSON(final Vertex vertex) throws JSONException {
return GraphSONUtility.jsonFromElement(vertex, null, GraphSONMode.NORMAL);
}
public static String edgeString(final Edge edge) {
return "e[" + edge.getLabel() + "], ["
+ edge.getVertex(Direction.OUT).getProperty("name")
......@@ -115,4 +102,18 @@ public final class GraphUtils {
+ edge.getVertex(Direction.IN).getProperty("name")
+ "]";
}
public static void dumpToLog(final Graph graph) {
LOG.debug("Vertices of {}", graph);
for (Vertex vertex : graph.getVertices()) {
LOG.debug(vertexString(vertex));
System.out.println(vertexString(vertex));
}
LOG.debug("Edges of {}", graph);
for (Edge edge : graph.getEdges()) {
LOG.debug(edgeString(edge));
System.out.println(edgeString(edge));
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphProvider.java → repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java
package org.apache.hadoop.metadata.services;
package org.apache.hadoop.metadata.repository.graph;
import javax.inject.Singleton;
......
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java → repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphService.java
......@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.metadata.services;
package org.apache.hadoop.metadata.repository.graph;
import java.io.IOException;
import java.util.ArrayList;
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
......@@ -24,7 +24,9 @@ import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.TypesDef;
import org.apache.hadoop.metadata.json.Serialization$;
import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.listener.TypedInstanceChangeListener;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.codehaus.jettison.json.JSONException;
......@@ -47,6 +49,8 @@ public class DefaultMetadataService implements MetadataService {
LoggerFactory.getLogger(DefaultMetadataService.class);
private final Set<TypesChangeListener> typesChangeListeners = new LinkedHashSet<>();
private final Set<TypedInstanceChangeListener> typedInstanceChangeListeners
= new LinkedHashSet<>();
private final TypeSystem typeSystem;
private final MetadataRepository repository;
......@@ -147,7 +151,11 @@ public class DefaultMetadataService implements MetadataService {
ITypedReferenceableInstance entityInstance =
Serialization$.MODULE$.fromJson(entityDefinition);
return repository.createEntity(entityInstance, entityType);
final String guid = repository.createEntity(entityInstance, entityType);
onAdd(entityType, entityInstance);
return guid;
} catch (ParseException e) {
LOG.error("Unable to parse JSON {} for type {}", entityDefinition, entityType, e);
throw new MetadataException("validation failed for: " + entityType);
......@@ -204,6 +212,21 @@ public class DefaultMetadataService implements MetadataService {
typesChangeListeners.remove(listener);
}
private void onAdd(String typeName,
ITypedReferenceableInstance typedInstance) throws MetadataException {
for (TypedInstanceChangeListener listener : typedInstanceChangeListeners) {
listener.onAdd(typeName, typedInstance);
}
}
public void registerListener(TypedInstanceChangeListener listener) {
typedInstanceChangeListeners.add(listener);
}
public void unregisterListener(TypedInstanceChangeListener listener) {
typedInstanceChangeListeners.remove(listener);
}
/**
* Starts the service. This method blocks until the service has completely started.
*
......
repository/src/test/java/org/apache/hadoop/metadata/RepositoryServiceLoadingTest.java
......@@ -2,7 +2,7 @@ package org.apache.hadoop.metadata;
import junit.framework.Assert;
import org.apache.hadoop.metadata.services.GraphService;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
......
repository/src/test/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryTest.java
......@@ -6,6 +6,8 @@ import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.Referenceable;
import org.apache.hadoop.metadata.RepositoryModuleBaseTest;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.apache.hadoop.metadata.storage.IRepository;
import org.apache.hadoop.metadata.storage.memory.MemRepository;
import org.apache.hadoop.metadata.types.AttributeDefinition;
......@@ -18,7 +20,7 @@ import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.apache.hadoop.metadata.util.GraphUtils;
import org.apache.hadoop.metadata.repository.graph.GraphUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......
repository/src/test/java/org/apache/hadoop/metadata/services/TitanGraphServiceTest.java
package org.apache.hadoop.metadata.services;
import org.apache.hadoop.metadata.RepositoryModuleBaseTest;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......
repository/src/test/resources/application.properties
......@@ -17,7 +17,7 @@
#
# GraphService implementation
metadata.graph.impl.class=org.apache.hadoop.metadata.services.TitanGraphService
metadata.graph.impl.class=org.apache.hadoop.metadata.repository.graph.TitanGraphService
# Graph implementation
......
webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java
......@@ -4,7 +4,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.services.GraphService;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java
......@@ -35,7 +35,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.metadata.services.GraphService;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.apache.hadoop.metadata.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
......
webapp/src/main/resources/application.properties
......@@ -18,7 +18,7 @@
######### Graph Database Configs #########
# Graph implementation
metadata.graph.impl.class=org.apache.hadoop.metadata.services.TitanGraphService
metadata.graph.impl.class=org.apache.hadoop.metadata.repository.graph.TitanGraphService
# Graph Storage
metadata.graph.storage.backend=berkeleyje
......
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
......@@ -56,6 +56,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
private static final String TABLE_NAME = "bar";
private static final String TRAIT_TYPE = "hive_fetl";
private String tableInstanceAsJSON;
private String guid;
@BeforeClass
......@@ -70,7 +71,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
public void testSubmitEntity() throws Exception {
ITypedReferenceableInstance tableInstance = createHiveTableInstance();
String instanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
tableInstanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
WebResource resource = service
.path("api/metadata/entities/submit")
......@@ -79,7 +80,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.POST, ClientResponse.class, instanceAsJSON);
.method(HttpMethod.POST, ClientResponse.class, tableInstanceAsJSON);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
String responseAsString = clientResponse.getEntity(String.class);
......@@ -99,7 +100,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
}
@Test (dependsOnMethods = "testSubmitEntity")
public void testGetEntityDefinition() {
public void testGetEntityDefinition() throws Exception {
WebResource resource = service
.path("api/metadata/entities/definition")
.path(guid);
......@@ -109,8 +110,19 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
String response = clientResponse.getEntity(String.class);
System.out.println("response = " + response);
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertNotNull(response.get("requestId"));
final String definition = response.getString("definition");
Assert.assertNotNull(definition);
System.out.println("definition = " + definition);
System.out.println("tableInstanceAsJSON = " + tableInstanceAsJSON);
// Assert.assertEquals(definition, tableInstanceAsJSON);
}
@Test
......
webapp/src/test/resources/application.properties
......@@ -17,7 +17,7 @@
#
# GraphService implementation
metadata.graph.impl.class=org.apache.hadoop.metadata.services.TitanGraphService
metadata.graph.impl.class=org.apache.hadoop.metadata.repository.graph.TitanGraphService
# Graph Storage
metadata.graph.storage.backend=inmemory
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment