Skip to content

A
atlas
Commit 7f4227e2 by Venkatesh Seetharam
Options

Simply Graph Initialization and minor refactoring

parent cbdbed1d
Showing with 142 additions and 1126 deletions
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hivetypes/HiveGraphRepositoryTest.java
......@@ -19,19 +19,17 @@
package org.apache.hadoop.metadata.hivetypes;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphHelper;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.apache.hadoop.metadata.repository.graph.TitanGraphProvider;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterClass;
......@@ -50,19 +48,19 @@ public class HiveGraphRepositoryTest {
LoggerFactory.getLogger(HiveGraphRepositoryTest.class);
protected HiveTypeSystem hts;
private GraphBackedMetadataRepository repository;
private GraphService gs;
private TitanGraph graph;
@BeforeClass
public void setup() throws ConfigurationException, MetadataException {
gs = new TitanGraphService(new TitanGraphProvider());
repository = new GraphBackedMetadataRepository(gs);
final TitanGraphProvider titanGraphProvider = new TitanGraphProvider();
graph = titanGraphProvider.get();
repository = new GraphBackedMetadataRepository(titanGraphProvider);
hts = HiveTypeSystem.getInstance();
}
@AfterClass
public void tearDown() {
Graph graph = gs.getBlueprintsGraph();
System.out.println("*******************Graph Dump****************************");
System.out.println("Vertices of " + graph);
for (Vertex vertex : graph.getVertices()) {
......
addons/hive-bridge/src/test/resources/application.properties
......@@ -16,20 +16,21 @@
# limitations under the License.
#
# GraphService implementation
metadata.graph.impl.class=org.apache.hadoop.metadata.repository.graph.TitanGraphService
# Graph implementation
#metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
######### Graph Database Configs #########
# Graph Storage
metadata.graph.storage.backend=inmemory
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=./data/berkeley
# Graph Search Index
#metadata.graph.index.search.backend=elasticsearch
#metadata.graph.index.search.directory=target/data/es
#metadata.graph.index.search.elasticsearch.client-only=false
#metadata.graph.index.search.elasticsearch.local-mode=true
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=./data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
######### Security Properties #########
# SSL config
metadata.enableTLS=false
######### Security Properties #########
addons/hive-bridge/src/test/resources/graph.properties deleted 100644 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
storage.backend=berkeleyje
storage.directory=target/data/berkeley
# Graph Search Index
index.search.backend=elasticsearch
index.search.directory=target/data/es
index.search.elasticsearch.client-only=false
index.search.elasticsearch.local-mode=true
\ No newline at end of file
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
......@@ -28,13 +28,11 @@ import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphBackedSearchIndexer;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.apache.hadoop.metadata.repository.graph.GraphServiceConfigurator;
import org.apache.hadoop.metadata.repository.typestore.GraphTypeStore;
import org.apache.hadoop.metadata.repository.graph.TitanGraphProvider;
import org.apache.hadoop.metadata.repository.typestore.GraphTypeStore;
import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
import org.apache.hadoop.metadata.services.DefaultMetadataService;
import org.apache.hadoop.metadata.services.MetadataService;
import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
/**
* Guice module for Repository module.
......@@ -42,7 +40,7 @@ import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// Graph Service implementation class
private Class<? extends GraphService> graphServiceClass;
// private Class<? extends GraphService> graphServiceClass;
// MetadataRepositoryService implementation class
private Class<? extends MetadataRepository> metadataRepoClass;
......@@ -52,10 +50,10 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private Class<? extends SearchIndexer> searchIndexer;
public RepositoryMetadataModule() {
GraphServiceConfigurator gsp = new GraphServiceConfigurator();
// GraphServiceConfigurator gsp = new GraphServiceConfigurator();
// get the impl classes for the repo and the graph service
this.graphServiceClass = gsp.getImplClass();
// this.graphServiceClass = gsp.getImplClass();
this.metadataRepoClass = GraphBackedMetadataRepository.class;
this.typeStore = GraphTypeStore.class;
this.metadataService = DefaultMetadataService.class;
......@@ -79,7 +77,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bind(ITypeStore.class).to(typeStore);
// bind the GraphService interface to an implementation
bind(GraphService.class).to(graphServiceClass);
// bind(GraphService.class).to(graphServiceClass);
// bind the MetadataService interface to an implementation
bind(MetadataService.class).to(metadataService);
......
repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryService.java
......@@ -18,12 +18,8 @@
package org.apache.hadoop.metadata.discovery;
import org.codehaus.jettison.json.JSONObject;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Metadata discovery service.
......@@ -48,25 +44,4 @@ public interface DiscoveryService {
* @throws org.apache.hadoop.metadata.discovery.DiscoveryException
*/
List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException;
/**
* Simple direct graph search and depth traversal.
* @param searchText is plain text
* @param prop is the Vertex property to search.
*/
Map<String, HashMap<String, JSONObject>> textSearch(String searchText,
int depth, String prop);
/**
* Simple graph walker for search interface, which allows following of specific edges only.
* @param edgesToFollow is a comma-separated-list of edges to follow.
*/
Map<String, HashMap<String, JSONObject>> relationshipWalk(String guid,
int depth, String edgesToFollow);
/**
* Return a Set of indexed properties in the graph.
* No parameters.
*/
Set<String> getGraphIndexedFields();
}
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/DefaultGraphPersistenceStrategy.java
......@@ -24,7 +24,7 @@ import org.apache.hadoop.metadata.query.Expressions;
import org.apache.hadoop.metadata.query.GraphPersistenceStrategies;
import org.apache.hadoop.metadata.query.TypeUtils;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.graph.Constants;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedStruct;
......
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
......@@ -21,11 +21,6 @@ package org.apache.hadoop.metadata.discovery.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.collections.iterators.IteratorChain;
import org.apache.hadoop.metadata.discovery.DiscoveryException;
import org.apache.hadoop.metadata.discovery.DiscoveryService;
import org.apache.hadoop.metadata.query.Expressions;
......@@ -36,9 +31,7 @@ import org.apache.hadoop.metadata.query.GremlinTranslator;
import org.apache.hadoop.metadata.query.QueryParser;
import org.apache.hadoop.metadata.query.QueryProcessor;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphHelper;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.codehaus.jettison.json.JSONObject;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.util.Either;
......@@ -51,10 +44,8 @@ import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class GraphBackedDiscoveryService implements DiscoveryService {
......@@ -64,82 +55,12 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
@Inject
GraphBackedDiscoveryService(TitanGraphService graphService,
GraphBackedDiscoveryService(GraphProvider<TitanGraph> graphProvider,
MetadataRepository metadataRepository) throws DiscoveryException {
this.titanGraph = graphService.getTitanGraph();
this.titanGraph = graphProvider.get();
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
}
private static void searchWalker(Vertex vtx, final int max, int counter,
HashMap<String, JSONObject> e,
HashMap<String, JSONObject> v, String edgesToFollow) {
counter++;
if (counter <= max) {
Map<String, String> jsonVertexMap = new HashMap<>();
Iterator<Edge> edgeIterator;
// If we're doing a lineage traversal, only follow the edges specified by the query.
// Otherwise return them all.
if (edgesToFollow != null) {
IteratorChain ic = new IteratorChain();
for (String iterateOn : edgesToFollow.split(",")) {
ic.addIterator(vtx.query().labels(iterateOn).edges().iterator());
}
edgeIterator = ic;
} else {
edgeIterator = vtx.query().edges().iterator();
}
//Iterator<Edge> edgeIterator = vtx.query().labels("Fathered").edges().iterator();
jsonVertexMap.put("HasRelationships", ((Boolean) edgeIterator.hasNext()).toString());
for (String pKey : vtx.getPropertyKeys()) {
jsonVertexMap.put(pKey, vtx.getProperty(pKey).toString());
}
// Add to the Vertex map.
v.put(vtx.getId().toString(), new JSONObject(jsonVertexMap));
// Follow this Vertex's edges if this isn't the last level of depth
if (counter < max) {
while (edgeIterator != null && edgeIterator.hasNext()) {
Edge edge = edgeIterator.next();
String label = edge.getLabel();
Map<String, String> jsonEdgeMap = new HashMap<>();
String tail = edge.getVertex(Direction.OUT).getId().toString();
String head = edge.getVertex(Direction.IN).getId().toString();
jsonEdgeMap.put("tail", tail);
jsonEdgeMap.put("head", head);
jsonEdgeMap.put("label", label);
Direction d;
if (tail.equals(vtx.getId().toString())) {
d = Direction.IN;
} else {
d = Direction.OUT;
}
/* If we want an Edge's property keys, uncomment here. Or we can parameterize
it.
Code is here now for reference/memory-jogging.
for (String pKey: edge.getPropertyKeys()) {
jsonEdgeMap.put(pKey, edge.getProperty(pKey).toString());
}
*/
e.put(edge.getId().toString(), new JSONObject(jsonEdgeMap));
searchWalker(edge.getVertex(d), max, counter, e, v, edgesToFollow);
}
}
}
}
/**
* Search using query DSL.
*
......@@ -233,93 +154,4 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
throw new DiscoveryException(se);
}
}
/*
* Simple direct graph search and depth traversal.
* @param searchText is plain text
* @param prop is the Vertex property to search.
*/
@Override
public Map<String, HashMap<String, JSONObject>> textSearch(String searchText,
int depth, String prop) {
HashMap<String, HashMap<String, JSONObject>> result = new HashMap<>();
// HashMaps, which contain sub JOSN Objects to be relayed back to the parent.
HashMap<String, JSONObject> vertices = new HashMap<>();
HashMap<String, JSONObject> edges = new HashMap<>();
/* todo: Later - when we allow search limitation by "type".
ArrayList<String> typesList = new ArrayList<String>();
for (String s: types.split(",")) {
// Types validity check.
if (typesList.contains(s)) {
LOG.error("Specifyed type is not a member of the Type System= {}", s);
throw new WebApplicationException(
Servlets.getErrorResponse("Invalid type specified in query.", Response
.Status.INTERNAL_SERVER_ERROR));
}
typesList.add(s);
}*/
int resultCount = 0;
//for (Result<Vertex> v: g.indexQuery(Constants.VERTEX_INDEX, "v." + prop + ":(" +
// searchText + ")").vertices()) {
for (Vertex v : ((GraphQuery) titanGraph.query().has(prop, searchText)).vertices()) {
//searchWalker(v.getElement(), depth, 0, edges, vertices, null);
searchWalker(v, depth, 0, edges, vertices, null);
resultCount++;
}
LOG.debug("Search for {} returned {} results.", searchText, resultCount);
result.put("vertices", vertices);
result.put("edges", edges);
return result;
}
/*
* Simple graph walker for search interface, which allows following of specific edges only.
* @param edgesToFollow is a comma-separated-list of edges to follow.
*/
@Override
public Map<String, HashMap<String, JSONObject>> relationshipWalk(String guid, int depth,
String edgesToFollow) {
HashMap<String, HashMap<String, JSONObject>> result = new HashMap<>();
// HashMaps, which contain sub JOSN Objects to be relayed back to the parent.
HashMap<String, JSONObject> vertices = new HashMap<>();
HashMap<String, JSONObject> edges = new HashMap<>();
// Get the Vertex with the specified GUID.
Vertex v = GraphHelper.findVertexByGUID(titanGraph, guid);
if (v != null) {
searchWalker(v, depth, 0, edges, vertices, edgesToFollow);
LOG.debug("Vertex {} found for guid {}", v, guid);
} else {
LOG.debug("Vertex not found for guid {}", guid);
}
result.put("vertices", vertices);
result.put("edges", edges);
return result;
}
/**
* Return a Set of indexed properties in the graph.
* No parameters.
*/
@Override
public Set<String> getGraphIndexedFields() {
return titanGraph.getIndexedKeys(Vertex.class);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java → repository/src/main/java/org/apache/hadoop/metadata/repository/Constants.java
......@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
package org.apache.hadoop.metadata.repository;
public final class Constants {
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -27,6 +27,7 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
......@@ -74,17 +75,14 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private final GraphToTypedInstanceMapper graphToInstanceMapper
= new GraphToTypedInstanceMapper();
private final GraphService graphService;
private final TypeSystem typeSystem;
private final TitanGraph titanGraph;
@Inject
public GraphBackedMetadataRepository(GraphService graphService) throws MetadataException {
this.graphService = graphService;
public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider) throws MetadataException {
this.typeSystem = TypeSystem.getInstance();
this.titanGraph = ((TitanGraphService) graphService).getTitanGraph();
this.titanGraph = graphProvider.get();
}
public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
......@@ -161,7 +159,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.info("Retrieving entity list for type={}", entityType);
GraphQuery query = graphService.getBlueprintsGraph().query()
GraphQuery query = titanGraph.query()
.has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<Vertex> results = query.vertices().iterator();
if (!results.hasNext()) {
......@@ -688,7 +686,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throws MetadataException {
// add a new vertex for the struct or trait instance
Vertex structInstanceVertex = GraphHelper.createVertexWithoutIdentity(
graphService.getBlueprintsGraph(), structInstance.getTypeName(), id);
titanGraph, structInstance.getTypeName(), id);
LOG.debug("created vertex {} for struct {}", structInstanceVertex, attributeInfo.name);
// map all the attributes to this newly created vertex
......@@ -716,7 +714,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
// add a new vertex for the struct or trait instance
final String traitName = traitInstance.getTypeName();
Vertex traitInstanceVertex = GraphHelper.createVertexWithoutIdentity(
graphService.getBlueprintsGraph(), traitInstance.getTypeName(), typedInstanceId);
titanGraph, traitInstance.getTypeName(), typedInstanceId);
LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
// map all the attributes to this newly created vertex
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
......@@ -30,6 +30,8 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.discovery.SearchIndexer;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
......@@ -55,8 +57,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
private final TitanGraph titanGraph;
@Inject
public GraphBackedSearchIndexer(GraphService graphService) throws MetadataException {
this.titanGraph = ((TitanGraphService) graphService).getTitanGraph();
public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider)
throws RepositoryException {
this.titanGraph = graphProvider.get();
initialize();
}
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
......@@ -24,6 +24,7 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.persistence.Id;
import org.slf4j.Logger;
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java
......@@ -20,10 +20,9 @@ package org.apache.hadoop.metadata.repository.graph;
import com.google.inject.throwingproviders.CheckedProvider;
import com.tinkerpop.blueprints.Graph;
import org.apache.commons.configuration.ConfigurationException;
public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
@Override
T get() throws ConfigurationException;
T get();
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphService.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph;
import java.util.Set;
/**
* A blueprints based graph service.
*/
public interface GraphService {
/**
* Returns an handle to the graph db.
*
* @return an handle to the graph db
*/
Graph getBlueprintsGraph();
KeyIndexableGraph getIndexableGraph();
TransactionalGraph getTransactionalGraph();
Set<String> getVertexIndexedKeys();
Set<String> getEdgeIndexedKeys();
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphServiceConfigurator.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
public class GraphServiceConfigurator extends PropertyBasedConfigurator<GraphService> {
private static final String PROPERTY_NAME = "metadata.graph.impl.class";
private static final String DEFAULT_IMPL_CLASS = TitanGraphService.class.getName();
private static final String CONFIG_PATH = "application.properties";
public GraphServiceConfigurator() {
super("metadata.graph.propertyName", "metadata.graph.defaultImplClass",
"metadata.graph.configurationPath", PROPERTY_NAME,
DEFAULT_IMPL_CLASS, CONFIG_PATH);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/PropertyBasedConfigurator.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import java.util.Properties;
public abstract class PropertyBasedConfigurator<T> {
private final String propertyName;
private final String defaultImplClass;
private final String configurationPath;
PropertyBasedConfigurator(String propertyNameProp, String defaultImplClassProp,
String configurationPathProp, String propertyNameDefaultProp,
String defaultImplClassDefaultProp, String configPathDefaultProp) {
Properties props = System.getProperties();
this.propertyName = props.getProperty(propertyNameProp,
propertyNameDefaultProp);
this.defaultImplClass = props.getProperty(defaultImplClassProp,
defaultImplClassDefaultProp);
this.configurationPath = props.getProperty(configurationPathProp,
configPathDefaultProp);
}
PropertyBasedConfigurator(String propertyNameProp, String defaultImplClassProp,
String configurationPathProp) {
Properties props = System.getProperties();
this.propertyName = props.getProperty(propertyNameProp);
this.defaultImplClass = props.getProperty(defaultImplClassProp);
this.configurationPath = props.getProperty(configurationPathProp);
}
public String getPropertyName() {
return propertyName;
}
public String getDefaultImplClass() {
return defaultImplClass;
}
public String getConfigurationPath() {
return configurationPath;
}
public Configuration getConfiguration() {
String path = getConfigurationPath();
Configuration config = null;
try {
config = new PropertiesConfiguration(path);
} catch (ConfigurationException e) {
config = new PropertiesConfiguration();
}
return config;
}
public String getClassName() {
Configuration config = getConfiguration();
String propName = getPropertyName();
String defaultClass = getDefaultImplClass();
return config.getString(propName, defaultClass);
}
@SuppressWarnings("unchecked")
public Class<? extends T> getImplClass() {
String className = getClassName();
Class<? extends T> ret = null;
try {
ret = (Class<? extends T>) PropertyBasedConfigurator.class
.getClassLoader().loadClass(className);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
return ret;
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanBootstrap.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.attribute.Cmp;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
public class TitanBootstrap {
private final TitanGraph graph;
public TitanBootstrap() {
graph = TitanFactory.build().set("storage.backend", "inmemory").open();
}
public static String vertexString(final Vertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
properties.append(propertyKey)
.append("=").append(vertex.getProperty(propertyKey))
.append(", ");
}
return "v[" + vertex.getId() + "], Properties[" + properties + "]";
}
public static String edgeString(final Edge edge) {
return "e[" + edge.getLabel() + "], ["
+ edge.getVertex(Direction.OUT).getProperty("name")
+ " -> " + edge.getLabel() + " -> "
+ edge.getVertex(Direction.IN).getProperty("name")
+ "]";
}
public static void main(String[] args) {
TitanBootstrap bootstrap = new TitanBootstrap();
TitanGraph graph = bootstrap.getGraph();
try {
Vertex harish = bootstrap.createVertex();
harish.setProperty("name", "harish");
Vertex venkatesh = bootstrap.createVertex();
venkatesh.setProperty("name", "venkatesh");
harish.addEdge("buddy", venkatesh);
for (Vertex v : graph.getVertices()) {
System.out.println("v = " + vertexString(v));
for (Edge e : v.getEdges(Direction.OUT)) {
System.out.println("e = " + edgeString(e));
}
}
System.out.println("====================");
GraphQuery graphQuery = graph.query()
.has("name", Cmp.EQUAL, "harish");
for (Vertex v : graphQuery.vertices()) {
System.out.println("v = " + vertexString(v));
}
graphQuery = graph.query()
.has("name", Cmp.EQUAL, "venkatesh");
for (Edge e : graphQuery.edges()) {
System.out.println("e = " + edgeString(e));
}
} finally {
graph.shutdown();
}
}
public TitanGraph getGraph() {
return graph;
}
public Vertex createVertex() {
return graph.addVertex(null);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java
......@@ -25,35 +25,48 @@ import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import javax.inject.Singleton;
import java.util.Iterator;
/**
* Default implementation for Graph Provider that doles out Titan Graph.
*/
public class TitanGraphProvider implements GraphProvider<TitanGraph> {
private static final String SYSTEM_PROP = "";
private static final String DEFAULT_PATH = "graph.properties";
private final String configPath;
private static final String CONFIG_PATH = "application.properties";
/**
* Constant for the configuration property that indicates the prefix.
*/
private static final String METADATA_PREFIX = "metadata.graph.";
private static Configuration getConfiguration() throws ConfigurationException {
PropertiesConfiguration configProperties = new PropertiesConfiguration(CONFIG_PATH);
Configuration graphConfig = new PropertiesConfiguration();
public TitanGraphProvider() {
configPath = System.getProperties().getProperty(SYSTEM_PROP,
DEFAULT_PATH);
final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) {
String key = iterator.next();
if (key.startsWith(METADATA_PREFIX)) {
String value = (String) configProperties.getProperty(key);
key = key.substring(METADATA_PREFIX.length());
graphConfig.setProperty(key, value);
}
}
public Configuration getConfiguration() throws ConfigurationException {
return new PropertiesConfiguration(configPath);
return graphConfig;
}
@Override
@Singleton
public TitanGraph get() throws ConfigurationException {
TitanGraph graph = null;
public TitanGraph get() {
Configuration config;
try {
config = getConfiguration();
} catch (ConfigurationException e) {
throw new RuntimeException(e);
}
graph = TitanFactory.open(config);
return graph;
return TitanFactory.open(config);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphService.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.IOException;
import java.util.Iterator;
import java.util.Set;
/**
* Default implementation for Graph service backed by Titan.
*/
@Singleton
public class TitanGraphService implements GraphService {
private static final Logger LOG = LoggerFactory.getLogger(TitanGraphService.class);
/**
* Constant for the configuration property that indicates the prefix.
*/
private static final String INDEXER_PREFIX = "metadata.indexer.vertex.";
private final TitanGraph titanGraph;
/**
* Initialize this service through injection with a custom Provider.
*
* @param graph graph implementation
* @throws ConfigurationException
*/
@Inject
public TitanGraphService(GraphProvider<TitanGraph> graph) throws ConfigurationException {
// TODO reimplement to save the Provider and initialize the graph inside the start() method
this.titanGraph = graph.get();
initialize();
}
private static Configuration getConfiguration(String filename, String prefix)
throws ConfigurationException {
PropertiesConfiguration configProperties = new PropertiesConfiguration(filename);
Configuration graphConfig = new PropertiesConfiguration();
final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) {
String key = iterator.next();
if (key.startsWith(prefix)) {
String value = (String) configProperties.getProperty(key);
key = key.substring(prefix.length());
graphConfig.setProperty(key, value);
}
}
return graphConfig;
}
/**
* Initializes this Service. The starting of Titan is handled by the Provider
* @throws ConfigurationException
*/
public void initialize() throws ConfigurationException {
// todo - create Edge Cardinality Constraints
LOG.info("Initialized titanGraph db: {}", titanGraph);
Set<String> vertexIndexedKeys = getVertexIndexedKeys();
LOG.info("Init vertex property keys: {}", vertexIndexedKeys);
Set<String> edgeIndexedKeys = getEdgeIndexedKeys();
LOG.info("Init edge property keys: {}", edgeIndexedKeys);
}
/**
* Stops the service. This method blocks until the service has completely
* shut down.
*/
public void stop() {
if (titanGraph != null) {
titanGraph.shutdown();
}
}
/**
* A version of stop() that is designed to be usable in Java7 closure
* clauses. Implementation classes MUST relay this directly to
* {@link #stop()}
*
* @throws java.io.IOException
* never
* @throws RuntimeException
* on any failure during the stop operation
*/
public void close() throws IOException {
stop();
}
@Override
public Graph getBlueprintsGraph() {
return titanGraph;
}
@Override
public KeyIndexableGraph getIndexableGraph() {
return titanGraph;
}
@Override
public TransactionalGraph getTransactionalGraph() {
return titanGraph;
}
public TitanGraph getTitanGraph() {
return titanGraph;
}
@Override
public Set<String> getVertexIndexedKeys() {
// this must use the graph API instead of setting this value as a class member - it can change after creation
return getIndexableGraph().getIndexedKeys(Vertex.class);
}
@Override
public Set<String> getEdgeIndexedKeys() {
// this must use the graph API instead of setting this value as a class member - it can change after creation
return getIndexableGraph().getIndexedKeys(Edge.class);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/mapping/StructTypeMapper.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph.mapping;
public class StructTypeMapper {
}
repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphTypeStore.java
......@@ -26,9 +26,8 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.graph.Constants;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.apache.hadoop.metadata.typesystem.TypesDef;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
......@@ -52,7 +51,6 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
public class GraphTypeStore implements ITypeStore {
private static final String PROPERTY_PREFIX = "type.";
......@@ -67,8 +65,8 @@ public class GraphTypeStore implements ITypeStore {
private final TitanGraph titanGraph;
@Inject
public GraphTypeStore(GraphService graphService) {
titanGraph = ((TitanGraphService)graphService).getTitanGraph();
public GraphTypeStore(GraphProvider<TitanGraph> graphProvider) {
titanGraph = graphProvider.get();
}
@Override
......
repository/src/test/java/org/apache/hadoop/metadata/RepositoryServiceLoadingTest.java
......@@ -18,10 +18,9 @@
package org.apache.hadoop.metadata;
import junit.framework.Assert;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.testng.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -37,18 +36,11 @@ import javax.inject.Inject;
public class RepositoryServiceLoadingTest {
@Inject
GraphService gs;
@BeforeClass
public void setUp() throws Exception {
}
@AfterClass
public void tearDown() throws Exception {
}
private GraphProvider<TitanGraph> graphProvider;
@Test
public void testGetGraphService() throws Exception {
Assert.assertNotNull(gs);
Assert.assertNotNull(graphProvider);
Assert.assertNotNull(graphProvider.get());
}
}
repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -29,8 +29,7 @@ import org.apache.hadoop.metadata.query.HiveTitanSample;
import org.apache.hadoop.metadata.query.QueryTestsUtils;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphHelper;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
......@@ -56,7 +55,7 @@ import java.io.File;
public class GraphBackedDiscoveryServiceTest {
@Inject
private GraphService graphService;
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
......@@ -82,7 +81,7 @@ public class GraphBackedDiscoveryServiceTest {
}
private void setupSampleData() throws ScriptException {
TitanGraph titanGraph = ((TitanGraphService) graphService).getTitanGraph();
TitanGraph titanGraph = graphProvider.get();
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
......
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -25,6 +25,7 @@ import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.TestUtils;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedStruct;
......@@ -72,7 +73,8 @@ public class GraphBackedMetadataRepositoryTest {
private static final String PII = "PII";
@Inject
private TitanGraphService titanGraphService;
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
......@@ -84,10 +86,7 @@ public class GraphBackedMetadataRepositoryTest {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
// start the injected graph service
titanGraphService.initialize();
new GraphBackedSearchIndexer(titanGraphService);
new GraphBackedSearchIndexer(graphProvider);
TestUtils.defineDeptEmployeeTypes(typeSystem);
createHiveTypes();
......@@ -184,7 +183,7 @@ public class GraphBackedMetadataRepositoryTest {
}
private Vertex getTableEntityVertex() {
TitanGraph graph = titanGraphService.getTitanGraph();
TitanGraph graph = graphProvider.get();
GraphQuery query = graph.query()
.has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
......
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperScaleTest.java
......@@ -25,6 +25,7 @@ import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.Struct;
......@@ -59,7 +60,7 @@ public class GraphRepoMapperScaleTest {
private static final String TABLE_NAME = "bar";
@Inject
private TitanGraphService titanGraphService;
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
......@@ -69,10 +70,7 @@ public class GraphRepoMapperScaleTest {
@BeforeClass
public void setUp() throws Exception {
// start the injected graph service
titanGraphService.initialize();
searchIndexer = new GraphBackedSearchIndexer(titanGraphService);
searchIndexer = new GraphBackedSearchIndexer(graphProvider);
typeSystem = TypeSystem.getInstance();
......@@ -115,7 +113,7 @@ public class GraphRepoMapperScaleTest {
}
private void searchWithOutIndex(String key, String value) {
TitanGraph graph = titanGraphService.getTitanGraph();
TitanGraph graph = graphProvider.get();
long start = System.currentTimeMillis();
int count = 0;
try {
......@@ -131,7 +129,7 @@ public class GraphRepoMapperScaleTest {
}
private void searchWithIndex(String key, String value) {
TitanGraph graph = titanGraphService.getTitanGraph();
TitanGraph graph = graphProvider.get();
long start = System.currentTimeMillis();
int count = 0;
try {
......
repository/src/test/java/org/apache/hadoop/metadata/repository/typestore/GraphTypeStoreTest.java
......@@ -27,7 +27,7 @@ import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.TestUtils;
import org.apache.hadoop.metadata.repository.graph.GraphHelper;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.apache.hadoop.metadata.typesystem.TypesDef;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
......@@ -46,7 +46,8 @@ import java.util.List;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphTypeStoreTest {
@Inject
private TitanGraphService titanGraphService;
private GraphProvider<TitanGraph> graphProvider;
@Inject
private ITypeStore typeStore;
......@@ -54,9 +55,6 @@ public class GraphTypeStoreTest {
@BeforeClass
public void setUp() throws Exception {
// start the injected graph service
titanGraphService.initialize();
ts = TypeSystem.getInstance();
ts.reset();
TestUtils.defineDeptEmployeeTypes(ts);
......@@ -69,7 +67,7 @@ public class GraphTypeStoreTest {
}
private void dumpGraph() {
TitanGraph graph = titanGraphService.getTitanGraph();
TitanGraph graph = graphProvider.get();
for (Vertex v : graph.getVertices()) {
System.out.println("****v = " + GraphHelper.vertexString(v));
for (Edge e : v.getEdges(Direction.OUT)) {
......
repository/src/test/java/org/apache/hadoop/metadata/services/TitanGraphServiceTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.services;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.repository.graph.TitanGraphService;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
/**
* Unit test for TitanGraphService.
*/
@Guice(modules = RepositoryMetadataModule.class)
public class TitanGraphServiceTest {
@Inject
TitanGraphService titanGraphService;
@BeforeClass
public void setUp() throws Exception {
titanGraphService.initialize();
}
@AfterClass
public void tearDown() throws Exception {
titanGraphService.close();
}
@Test
public void testStart() throws Exception {
Assert.assertNotNull(titanGraphService.getBlueprintsGraph());
}
@Test
public void testGetBlueprintsGraph() throws Exception {
Assert.assertNotNull(titanGraphService.getBlueprintsGraph());
}
@Test
public void testGetIndexableGraph() throws Exception {
Assert.assertNotNull(titanGraphService.getIndexableGraph());
}
@Test
public void testGetTransactionalGraph() throws Exception {
Assert.assertNotNull(titanGraphService.getTransactionalGraph());
}
@Test
public void testGetTitanGraph() throws Exception {
Assert.assertNotNull(titanGraphService.getTitanGraph());
}
}
repository/src/test/resources/application.properties
......@@ -16,20 +16,20 @@
# limitations under the License.
#
# GraphService implementation
metadata.graph.impl.class=org.apache.hadoop.metadata.repository.graph.TitanGraphService
# Graph implementation
#metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
######### Graph Database Configs #########
# Graph Storage
metadata.graph.storage.backend=inmemory
# Graph Search Index
#metadata.graph.index.search.backend=elasticsearch
#metadata.graph.index.search.directory=target/data/es
#metadata.graph.index.search.elasticsearch.client-only=false
#metadata.graph.index.search.elasticsearch.local-mode=true
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=./data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
######### Security Properties #########
# SSL config
metadata.enableTLS=false
######### Security Properties #########
repository/src/test/resources/graph.properties deleted 100644 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
storage.backend=inmemory
# Graph Search Index
index.search.backend=elasticsearch
index.search.directory=target/data/es
index.search.elasticsearch.client-only=false
index.search.elasticsearch.local-mode=true
src/conf/application.properties
......@@ -17,8 +17,16 @@
#
######### Graph Database Configs #########
# Graph implementation
metadata.graph.impl.class=org.apache.hadoop.metadata.repository.graph.TitanGraphService
# Graph Storage
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=./data/berkeley
# Graph Search Index
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=./data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
######### Security Properties #########
......
src/conf/graph.properties deleted 100644 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Graph Storage
storage.backend=berkeleyje
storage.directory=./data/berkeley
# Graph Search Index
index.search.backend=elasticsearch
index.search.directory=./data/es
index.search.elasticsearch.client-only=false
index.search.elasticsearch.local-mode=true
webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java
......@@ -39,8 +39,8 @@ import java.util.Map;
public class GuiceServletConfig extends GuiceServletContextListener {
private static final Logger LOG = LoggerFactory
.getLogger(GuiceServletConfig.class);
private static final Logger LOG = LoggerFactory.getLogger(GuiceServletConfig.class);
private static final String GUICE_CTX_PARAM = "guice.packages";
@Override
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java
......@@ -31,16 +31,13 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -186,120 +183,4 @@ public class MetadataDiscoveryResource {
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Return a list of Vertices and Edges that emanate from the provided GUID to the depth
* specified.
*
* GET http://host/api/metadata/discovery/search/relationships/{guid}
*
* edgesToFollow = comma-separated list of Labels to follow. Sample query:
* http://host/api/metadata/discovery/search/relationships/1?depth=3&edgesToFollow=Likes,Has
*/
@GET
@Path("/search/relationships/{guid}")
@Produces({MediaType.APPLICATION_JSON})
public Response getLineageResults(@PathParam("guid") final String guid,
@DefaultValue("1") @QueryParam("depth") final int depth,
@QueryParam("edgesToFollow") final String edgesToFollow) {
LOG.info("Performing GUID lineage search for guid= {}", guid);
Preconditions.checkNotNull(guid, "Invalid argument: \"guid\" cannot be null.");
Preconditions
.checkNotNull(edgesToFollow, "Invalid argument: \"edgesToFollow\" cannot be null.");
// Parent JSON Object
JSONObject response = new JSONObject();
Map<String, HashMap<String, JSONObject>> resultMap = discoveryService
.relationshipWalk(guid, depth, edgesToFollow);
try {
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
if (resultMap.containsKey("vertices")) {
response.put("vertices", new JSONObject(resultMap.get("vertices")));
}
if (resultMap.containsKey("edges")) {
response.put("edges", new JSONObject(resultMap.get("edges")));
}
} catch (JSONException e) {
throw new WebApplicationException(
Servlets.getErrorResponse("Search: Error building JSON result set.",
Response.Status.INTERNAL_SERVER_ERROR));
}
LOG.debug("JSON result:" + response.toString());
return Response.ok(response).build();
}
/**
* Return a list of Vertices and Edges that match the given query.
*
* GET http://host/api/metadata/discovery/search/fulltext
*
* Sample query:
* http://host/api/metadata/discovery/search/fulltext?depth=1&property=Name&text=Zack
*
* Comma separated list of types as qeury.
*/
@GET
@Path("/search/fulltext")
@Produces({MediaType.APPLICATION_JSON})
public Response getFullTextResults(@QueryParam("text") final String searchText,
@DefaultValue("1") @QueryParam("depth") final int depth,
@DefaultValue("guid") @QueryParam("property") final String
prop) {
LOG.info("Performing full text search for vertices with property {} matching= {}", prop,
searchText);
Preconditions.checkNotNull(searchText, "Invalid argument: \"text\" cannot be null.");
Preconditions.checkNotNull(prop, "Invalid argument: \"prop\" cannot be null.");
// Parent JSON Object
JSONObject response = new JSONObject();
Map<String, HashMap<String, JSONObject>> resultMap = discoveryService.textSearch(
searchText, depth, prop);
try {
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
if (resultMap.containsKey("vertices")) {
response.put("vertices", resultMap.get("vertices"));
}
if (resultMap.containsKey("edges")) {
response.put("edges", resultMap.get("edges"));
}
} catch (JSONException e) {
throw new WebApplicationException(
Servlets.getErrorResponse("Search: Error building JSON result set.",
Response.Status.INTERNAL_SERVER_ERROR));
}
LOG.debug("JSON result:" + response.toString());
return Response.ok(response).build();
}
/**
* Return a list Vertices and Edges in the index.
*
* GET http://host/api/metadata/discovery/getIndexedFields
*
* No parameters taken.
*/
@GET
@Path("/getIndexedFields")
@Produces({MediaType.APPLICATION_JSON})
public Response getLineageResults() {
JSONObject response = new JSONObject();
try {
response.put("indexed_fields:", discoveryService.getGraphIndexedFields());
} catch (JSONException e) {
throw new WebApplicationException(
Servlets.getErrorResponse("Search: Error building JSON result set.",
Response.Status.INTERNAL_SERVER_ERROR));
}
LOG.debug("JSON result:" + response.toString());
return Response.ok(response).build();
}
}
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java
......@@ -18,6 +18,7 @@
package org.apache.hadoop.metadata.web.resources;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
......@@ -28,7 +29,7 @@ import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.repository.graph.GraphService;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import org.apache.hadoop.metadata.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
......@@ -75,11 +76,12 @@ public class RexsterGraphResource {
public static final String IN_IDS = "inIds";
public static final String BOTH_IDS = "bothIds";
private static final Logger LOG = LoggerFactory.getLogger(RexsterGraphResource.class);
private final GraphService graphService;
private TitanGraph graph;
@Inject
public RexsterGraphResource(GraphService graphService) {
this.graphService = graphService;
public RexsterGraphResource(GraphProvider<TitanGraph> graphProvider) {
this.graph = graphProvider.get();
}
private static void validateInputs(String errorMsg, String... inputs) {
......@@ -94,15 +96,15 @@ public class RexsterGraphResource {
}
protected Graph getGraph() {
return graphService.getBlueprintsGraph();
return graph;
}
protected Set<String> getVertexIndexedKeys() {
return graphService.getVertexIndexedKeys();
return graph.getIndexedKeys(Vertex.class);
}
protected Set<String> getEdgeIndexedKeys() {
return graphService.getEdgeIndexedKeys();
return graph.getIndexedKeys(Edge.class);
}
/**
......
webapp/src/main/resources/application.properties
......@@ -16,7 +16,18 @@
# limitations under the License.
#
# GraphService implementation
metadata.graph.impl.class=org.apache.hadoop.metadata.repository.graph.TitanGraphService
######### Graph Database Configs #########
# Graph Storage
metadata.graph.storage.backend=inmemory
# Graph Search Index
metadata.graph.index.search.backend=lucene
metadata.graph.index.search.directory=webapp/target/data/lucene
######### Security Properties #########
# SSL config
metadata.enableTLS=false
######### Security Properties #########
webapp/src/main/resources/graph.properties deleted 100644 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
storage.backend=inmemory
# Graph Search Index
index.search.backend=lucene
index.search.directory=webapp/target/data/es
\ No newline at end of file
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryJerseyResourceIT.java
......@@ -145,64 +145,6 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
Assert.assertEquals(response.getString("queryType"), "dsl");
}
@Test
public void testFullTextUriExists() throws Exception {
WebResource resource = service
.path("api/metadata/discovery/search/fulltext")
.queryParam("depth", "0").queryParam("text", "foo").queryParam("property", "Name");
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertNotEquals(clientResponse.getStatus(),
Response.Status.NOT_FOUND.getStatusCode());
}
@Test
public void testGetIndexedProperties() throws Exception {
WebResource resource = service
.path("api/metadata/discovery/getIndexedFields");
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertNotEquals(clientResponse.getStatus(),
Response.Status.NOT_FOUND.getStatusCode());
}
@Test
public void testLineageUriExists() throws Exception {
WebResource resource = service
.path("api/metadata/discovery/search/relationships/1")
.queryParam("depth", "1").queryParam("edgesToFollow", "bar");
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertNotEquals(clientResponse.getStatus(),
Response.Status.NOT_FOUND.getStatusCode());
}
@Test(dependsOnMethods = "testFullTextUriExists")
public void testSearchForText() throws Exception {
WebResource resource = service
.path("api/metadata/discovery/search/fulltext")
.queryParam("depth", "3").queryParam("text", "bar")
.queryParam("property", "hive_table.name");
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
//TODO - Assure zero vertices and edges.
Assert.assertNotEquals(clientResponse.getStatus(),
Response.Status.NOT_FOUND.getStatusCode());
}
private void createTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> dslTestTypeDefinition =
TypesUtil.createClassTypeDef("dsl_test_type",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment