Skip to content

A
atlas
Commit 8cff9589 by Venkatesh Seetharam
Options

Adds Apache License Headers, enables rat-check, find-bugs and checkstyle…

Adds Apache License Headers, enables rat-check, find-bugs and checkstyle plugins. Contributed by Venkatesh Seetharam
parent 9822e9c7
Showing with 76 additions and 145 deletions
common/pom.xml
...@@ -121,11 +121,6 @@ ...@@ -121,11 +121,6 @@
<groupId>com.thinkaurelius.titan</groupId> <groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId> <artifactId>titan-core</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
</dependency>
</dependencies> </dependencies>
<build> <build>
......
hivetypes/src/test/resources/graph.properties
...@@ -16,7 +16,8 @@ ...@@ -16,7 +16,8 @@
# limitations under the License. # limitations under the License.
# #
storage.backend=inmemory storage.backend=berkeleyje
storage.directory=target/data/berkeley
# Graph Search Index # Graph Search Index
index.search.backend=elasticsearch index.search.backend=elasticsearch
......
pom.xml
...@@ -579,6 +579,13 @@ ...@@ -579,6 +579,13 @@
<version>${titan.version}</version> <version>${titan.version}</version>
</dependency> </dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId>
<version>${titan.version}</version>
<!--<scope>test</scope>-->
</dependency>
<!-- metadata modules --> <!-- metadata modules -->
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId> <groupId>org.apache.hadoop.metadata</groupId>
......
repository/pom.xml
...@@ -118,12 +118,12 @@ ...@@ -118,12 +118,12 @@
<dependency> <dependency>
<groupId>com.thinkaurelius.titan</groupId> <groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId> <artifactId>titan-es</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.thinkaurelius.titan</groupId> <groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-es</artifactId> <artifactId>titan-lucene</artifactId>
</dependency> </dependency>
<dependency> <dependency>
......
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
...@@ -30,6 +30,8 @@ import com.google.inject.throwingproviders.ThrowingProviderBinder; ...@@ -30,6 +30,8 @@ import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.hadoop.metadata.discovery.DiscoveryService; import org.apache.hadoop.metadata.discovery.DiscoveryService;
import org.apache.hadoop.metadata.discovery.GraphBackedDiscoveryService; import org.apache.hadoop.metadata.discovery.GraphBackedDiscoveryService;
import org.apache.hadoop.metadata.repository.SearchIndexer;
import org.apache.hadoop.metadata.repository.graph.GraphBackedSearchIndexer;
import org.apache.hadoop.metadata.services.DefaultMetadataService; import org.apache.hadoop.metadata.services.DefaultMetadataService;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository; import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphProvider; import org.apache.hadoop.metadata.repository.graph.GraphProvider;
...@@ -51,6 +53,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { ...@@ -51,6 +53,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private Class<? extends MetadataRepository> metadataRepoClass; private Class<? extends MetadataRepository> metadataRepoClass;
private Class<? extends MetadataService> metadataService; private Class<? extends MetadataService> metadataService;
private Class<? extends DiscoveryService> discoveryService; private Class<? extends DiscoveryService> discoveryService;
private Class<? extends SearchIndexer> searchIndexer;
public RepositoryMetadataModule() { public RepositoryMetadataModule() {
GraphServiceConfigurator gsp = new GraphServiceConfigurator(); GraphServiceConfigurator gsp = new GraphServiceConfigurator();
...@@ -60,6 +63,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { ...@@ -60,6 +63,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
this.metadataRepoClass = GraphBackedMetadataRepository.class; this.metadataRepoClass = GraphBackedMetadataRepository.class;
this.metadataService = DefaultMetadataService.class; this.metadataService = DefaultMetadataService.class;
this.discoveryService = GraphBackedDiscoveryService.class; this.discoveryService = GraphBackedDiscoveryService.class;
this.searchIndexer = GraphBackedSearchIndexer.class;
} }
protected void configure() { protected void configure() {
...@@ -82,5 +86,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { ...@@ -82,5 +86,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the DiscoveryService interface to an implementation // bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(discoveryService); bind(DiscoveryService.class).to(discoveryService);
bind(SearchIndexer.class).to(searchIndexer);
} }
} }
repository/src/main/java/org/apache/hadoop/metadata/repository/SearchIndexer.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
/**
* Interface for indexing types.
*/
public interface SearchIndexer extends TypesChangeListener {
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
...@@ -620,9 +620,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -620,9 +620,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} }
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name; String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
// todo: move this to the indexer
GraphHelper.createPropertyKey(titanGraph.getManagementSystem(), propertyName);
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType(); IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
StringBuilder buffer = new StringBuilder(); StringBuilder buffer = new StringBuilder();
...@@ -652,9 +649,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -652,9 +649,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} }
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name; String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
// todo: move this to the indexer
GraphHelper.createPropertyKey(titanGraph.getManagementSystem(), propertyName);
StringBuilder buffer = new StringBuilder(); StringBuilder buffer = new StringBuilder();
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType(); IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
for (Map.Entry entry : collection.entrySet()) { for (Map.Entry entry : collection.entrySet()) {
...@@ -719,11 +713,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -719,11 +713,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id); referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
} else { } else {
referenceVertex = idToVertexMap.get(id); referenceVertex = idToVertexMap.get(id);
/*
ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
mapInstanceToVertex(id, typedInstance, referenceVertex,
classType.fieldMapping().fields, idToVertexMap);
*/
} }
if (referenceVertex != null) { if (referenceVertex != null) {
...@@ -998,38 +987,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -998,38 +987,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
propertyNameWithSuffix.lastIndexOf("."), propertyNameWithSuffix.length()); propertyNameWithSuffix.lastIndexOf("."), propertyNameWithSuffix.length());
values.put(key, mapVertexToCollectionEntry( values.put(key, mapVertexToCollectionEntry(
instanceVertex, attributeInfo, elementType, propertyNameWithSuffix)); instanceVertex, attributeInfo, elementType, propertyNameWithSuffix));
/*
switch (valueType.getTypeCategory()) {
case PRIMITIVE:
values.put(key, instanceVertex.getProperty(propertyNameWithSuffix));
break;
case ENUM:
values.put(key, instanceVertex.getProperty(propertyNameWithSuffix));
break;
case ARRAY:
case MAP:
case TRAIT:
// do nothing
break;
case STRUCT:
ITypedStruct structInstance = getStructInstanceFromVertex(instanceVertex,
valueType, attributeInfo.name, propertyNameWithSuffix);
values.put(key, structInstance);
break;
case CLASS:
Object idOrInstance = mapClassReferenceToVertex(
instanceVertex, attributeInfo, propertyNameWithSuffix);
values.put(key, idOrInstance);
break;
default:
break;
}
*/
} }
typedInstance.set(attributeInfo.name, values); typedInstance.set(attributeInfo.name, values);
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
...@@ -18,9 +18,8 @@ ...@@ -18,9 +18,8 @@
package org.apache.hadoop.metadata.repository.graph; package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.Cardinality; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.PropertyKey; import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Compare; import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Edge;
...@@ -91,21 +90,15 @@ public final class GraphHelper { ...@@ -91,21 +90,15 @@ public final class GraphHelper {
return results.hasNext() ? results.next() : null; return results.hasNext() ? results.next() : null;
} }
public static PropertyKey createPropertyKey(TitanManagement management, String propertyName) { public static Vertex findVertexByGUIDUsingIndex(TitanGraph titanGraph,
PropertyKey propertyKey = management.getPropertyKey(propertyName); String value) {
if (propertyKey == null) { LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
propertyKey = management
.makePropertyKey(propertyName)
.dataType(String.class)
.cardinality(Cardinality.SET)
.make();
management.buildIndex("by" + propertyName, Vertex.class)
.addKey(propertyKey)
.buildCompositeIndex();
}
return propertyKey; TitanIndexQuery query = titanGraph.indexQuery("index_" + Constants.GUID_PROPERTY_KEY,
Constants.GUID_PROPERTY_KEY + " = " + value);
Iterator<TitanIndexQuery.Result<Vertex>> results = query.vertices().iterator();
// returning one since guid should be unique
return results.hasNext() ? results.next().getElement() : null;
} }
public static String vertexString(final Vertex vertex) { public static String vertexString(final Vertex vertex) {
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphService.java
...@@ -18,25 +18,6 @@ ...@@ -18,25 +18,6 @@
package org.apache.hadoop.metadata.repository.graph; package org.apache.hadoop.metadata.repository.graph;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.thinkaurelius.titan.core.PropertyKey;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex; import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement; import com.thinkaurelius.titan.core.schema.TitanManagement;
...@@ -45,6 +26,20 @@ import com.tinkerpop.blueprints.Graph; ...@@ -45,6 +26,20 @@ import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.KeyIndexableGraph; import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph; import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
/** /**
* Default implementation for Graph service backed by Titan. * Default implementation for Graph service backed by Titan.
...@@ -115,12 +110,11 @@ public class TitanGraphService implements GraphService { ...@@ -115,12 +110,11 @@ public class TitanGraphService implements GraphService {
return graphConfig; return graphConfig;
} }
/** /**
* Initializes the indices for the graph. * Initializes the indices for the graph.
* @throws ConfigurationException * @throws ConfigurationException
*/ */
// TODO move this functionality to the MetadataRepository? // TODO move this functionality to the SearchIndexer?
protected void createIndicesForVertexKeys() throws ConfigurationException { protected void createIndicesForVertexKeys() throws ConfigurationException {
if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) { if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph"); LOG.info("Indexes already exist for titanGraph");
...@@ -133,61 +127,7 @@ public class TitanGraphService implements GraphService { ...@@ -133,61 +127,7 @@ public class TitanGraphService implements GraphService {
TitanGraphIndex graphIndex = mgmt.buildIndex(Constants.INDEX_NAME, Vertex.class) TitanGraphIndex graphIndex = mgmt.buildIndex(Constants.INDEX_NAME, Vertex.class)
.buildMixedIndex(Constants.BACKING_INDEX); .buildMixedIndex(Constants.BACKING_INDEX);
PropertyKey guidKey = mgmt
.makePropertyKey(Constants.GUID_PROPERTY_KEY)
.dataType(String.class).make();
mgmt.buildIndex("byGUID", Vertex.class)
.addKey(guidKey)
.unique()
.buildCompositeIndex();
PropertyKey typeKey = mgmt
.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY)
.dataType(String.class).make();
mgmt.buildIndex("byType", Vertex.class)
.addKey(typeKey)
.buildCompositeIndex();
//TODO - Once we can get the TypeSystem to give me actual types, the below will be modified and replace the current
//indexer config.
/*
Iterator<String> i = typeSystem.getTypeNames().iterator();
// Get a list of property names to iterate through...
List<String> propList = new ArrayList<>();
while (i.hasNext()) {
String currType = i.next();
Iterator<String> typeDefIterator = null;
while (typeDefIterator.hasNext()) {
// Pull the property name and index, so we can register the name
// and look up the type.
String prop = "";
String type = "";
boolean isUnique = false;
// Add the key.
LOG.info("Adding property: " + prop + " to index as type: "
+ type);
mgmt.addIndexKey(graphIndex, mgmt.makePropertyKey(prop)
.dataType(type.getClass()).make());
}
mgmt.commit();
LOG.info("Index creation complete.");
}
*/
Configuration indexConfig = getConfiguration("indexer.properties", INDEXER_PREFIX); Configuration indexConfig = getConfiguration("indexer.properties", INDEXER_PREFIX);
// Properties are formatted: prop_name:type;prop_name:type // Properties are formatted: prop_name:type;prop_name:type
// E.g. Name:String;Date:Long // E.g. Name:String;Date:Long
if (!indexConfig.isEmpty()) { if (!indexConfig.isEmpty()) {
...@@ -241,7 +181,6 @@ public class TitanGraphService implements GraphService { ...@@ -241,7 +181,6 @@ public class TitanGraphService implements GraphService {
mgmt.commit(); mgmt.commit();
LOG.info("Index creation complete."); LOG.info("Index creation complete.");
} }
} }
/** /**
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
...@@ -27,6 +27,7 @@ import org.apache.hadoop.metadata.json.TypesSerialization; ...@@ -27,6 +27,7 @@ import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.listener.TypedInstanceChangeListener; import org.apache.hadoop.metadata.listener.TypedInstanceChangeListener;
import org.apache.hadoop.metadata.listener.TypesChangeListener; import org.apache.hadoop.metadata.listener.TypesChangeListener;
import org.apache.hadoop.metadata.repository.MetadataRepository; import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.SearchIndexer;
import org.apache.hadoop.metadata.storage.RepositoryException; import org.apache.hadoop.metadata.storage.RepositoryException;
import org.apache.hadoop.metadata.types.IDataType; import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.TypeSystem; import org.apache.hadoop.metadata.types.TypeSystem;
...@@ -56,9 +57,12 @@ public class DefaultMetadataService implements MetadataService { ...@@ -56,9 +57,12 @@ public class DefaultMetadataService implements MetadataService {
private final MetadataRepository repository; private final MetadataRepository repository;
@Inject @Inject
DefaultMetadataService(MetadataRepository repository) throws MetadataException { DefaultMetadataService(MetadataRepository repository,
SearchIndexer searchIndexer) throws MetadataException {
this.typeSystem = TypeSystem.getInstance(); this.typeSystem = TypeSystem.getInstance();
this.repository = repository; this.repository = repository;
registerListener(searchIndexer);
} }
/** /**
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment