Commit 8cff9589 by Venkatesh Seetharam

Adds Apache License Headers, enables rat-check, find-bugs and checkstyle…

Adds Apache License Headers, enables rat-check, find-bugs and checkstyle plugins. Contributed by Venkatesh Seetharam
parent 9822e9c7
......@@ -121,11 +121,6 @@
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
</dependency>
</dependencies>
<build>
......
......@@ -16,7 +16,8 @@
# limitations under the License.
#
storage.backend=inmemory
storage.backend=berkeleyje
storage.directory=target/data/berkeley
# Graph Search Index
index.search.backend=elasticsearch
......
......@@ -579,6 +579,13 @@
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId>
<version>${titan.version}</version>
<!--<scope>test</scope>-->
</dependency>
<!-- metadata modules -->
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
......
......@@ -118,12 +118,12 @@
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
<artifactId>titan-es</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-es</artifactId>
<artifactId>titan-lucene</artifactId>
</dependency>
<dependency>
......
......@@ -30,6 +30,8 @@ import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.hadoop.metadata.discovery.DiscoveryService;
import org.apache.hadoop.metadata.discovery.GraphBackedDiscoveryService;
import org.apache.hadoop.metadata.repository.SearchIndexer;
import org.apache.hadoop.metadata.repository.graph.GraphBackedSearchIndexer;
import org.apache.hadoop.metadata.services.DefaultMetadataService;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
......@@ -51,6 +53,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private Class<? extends MetadataRepository> metadataRepoClass;
private Class<? extends MetadataService> metadataService;
private Class<? extends DiscoveryService> discoveryService;
private Class<? extends SearchIndexer> searchIndexer;
public RepositoryMetadataModule() {
GraphServiceConfigurator gsp = new GraphServiceConfigurator();
......@@ -60,6 +63,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
this.metadataRepoClass = GraphBackedMetadataRepository.class;
this.metadataService = DefaultMetadataService.class;
this.discoveryService = GraphBackedDiscoveryService.class;
this.searchIndexer = GraphBackedSearchIndexer.class;
}
protected void configure() {
......@@ -82,5 +86,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(discoveryService);
bind(SearchIndexer.class).to(searchIndexer);
}
}
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
/**
* Interface for indexing types.
*/
public interface SearchIndexer extends TypesChangeListener {
}
......@@ -620,9 +620,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
// todo: move this to the indexer
GraphHelper.createPropertyKey(titanGraph.getManagementSystem(), propertyName);
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
StringBuilder buffer = new StringBuilder();
......@@ -652,9 +649,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
// todo: move this to the indexer
GraphHelper.createPropertyKey(titanGraph.getManagementSystem(), propertyName);
StringBuilder buffer = new StringBuilder();
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
for (Map.Entry entry : collection.entrySet()) {
......@@ -719,11 +713,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
} else {
referenceVertex = idToVertexMap.get(id);
/*
ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
mapInstanceToVertex(id, typedInstance, referenceVertex,
classType.fieldMapping().fields, idToVertexMap);
*/
}
if (referenceVertex != null) {
......@@ -998,38 +987,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
propertyNameWithSuffix.lastIndexOf("."), propertyNameWithSuffix.length());
values.put(key, mapVertexToCollectionEntry(
instanceVertex, attributeInfo, elementType, propertyNameWithSuffix));
/*
switch (valueType.getTypeCategory()) {
case PRIMITIVE:
values.put(key, instanceVertex.getProperty(propertyNameWithSuffix));
break;
case ENUM:
values.put(key, instanceVertex.getProperty(propertyNameWithSuffix));
break;
case ARRAY:
case MAP:
case TRAIT:
// do nothing
break;
case STRUCT:
ITypedStruct structInstance = getStructInstanceFromVertex(instanceVertex,
valueType, attributeInfo.name, propertyNameWithSuffix);
values.put(key, structInstance);
break;
case CLASS:
Object idOrInstance = mapClassReferenceToVertex(
instanceVertex, attributeInfo, propertyNameWithSuffix);
values.put(key, idOrInstance);
break;
default:
break;
}
*/
}
typedInstance.set(attributeInfo.name, values);
......
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.SearchIndexer;
import org.apache.hadoop.metadata.types.AttributeInfo;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.DataTypes;
import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.StructType;
import org.apache.hadoop.metadata.types.TraitType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Map;
/**
* Adds index for properties of a given type when its added before any instances are added.
*/
public class GraphBackedSearchIndexer implements SearchIndexer {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
private final TitanGraph titanGraph;
@Inject
public GraphBackedSearchIndexer(GraphService graphService) throws MetadataException {
this.titanGraph = ((TitanGraphService) graphService).getTitanGraph();
initialize();
}
/**
* Initializes the indices for the graph - create indices for Global Vertex Keys
*/
private void initialize() {
TitanManagement management = titanGraph.getManagementSystem();
if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
LOG.info("Global indexes already exist for graph");
return;
}
LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
try {
createIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true);
createIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
createIndex(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
} finally {
management.commit();
}
LOG.info("Index creation for global keys complete.");
// dumpIndexKeys();
}
/*
private void dumpIndexKeys() {
for (TitanGraphIndex index : management.getGraphIndexes(Vertex.class)) {
System.out.println("index.getName() = " + index.getName());
for (PropertyKey key : index.getFieldKeys()) {
System.out.println("key.getName() = " + key.getName());
System.out.println("key = " + key);
}
}
for (TitanGraphIndex index : management.getGraphIndexes(Edge.class)) {
System.out.println("index.getName() = " + index.getName());
for (PropertyKey key : index.getFieldKeys()) {
System.out.println("key.getName() = " + key.getName());
System.out.println("key = " + key);
}
}
}
*/
/**
* This is upon adding a new type to Store.
*
* @param typeName type name
* @param dataType data type
* @throws org.apache.hadoop.metadata.MetadataException
*/
@Override
public void onAdd(String typeName, IDataType dataType) throws MetadataException {
LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);
try {
TitanManagement management = titanGraph.getManagementSystem();
addIndexForType(management, dataType);
management.commit();
LOG.info("Index creation for type {} complete", typeName);
} catch (Exception e) {
LOG.error("Error creating index for type {}", dataType, e);
// management.rollback();
}
// dumpIndexKeys();
}
private void addIndexForType(TitanManagement management, IDataType dataType) {
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
case ENUM:
case ARRAY:
case MAP:
// do nothing since these are NOT types
break;
case STRUCT:
StructType structType = (StructType) dataType;
createIndexForFields(management, structType, structType.fieldMapping().fields);
break;
case TRAIT:
TraitType traitType = (TraitType) dataType;
createIndexForFields(management, traitType, traitType.fieldMapping().fields);
break;
case CLASS:
ClassType classType = (ClassType) dataType;
createIndexForFields(management, classType, classType.fieldMapping().fields);
break;
default:
throw new IllegalArgumentException("bad data type" + dataType);
}
}
private void createIndexForFields(TitanManagement management,
IDataType dataType, Map<String, AttributeInfo> fields) {
for (AttributeInfo field : fields.values()) {
if (field.isIndexable) {
createIndexForAttribute(management, dataType.getName(), field);
}
}
}
private void createIndexForAttribute(TitanManagement management,
String typeName, AttributeInfo field) {
final String propertyName = typeName + "." + field.name;
switch (field.dataType().getTypeCategory()) {
case PRIMITIVE:
createIndex(management, propertyName,
getPrimitiveClass(field.dataType()), field.isUnique);
break;
case ENUM:
createIndex(management, propertyName, Integer.class, field.isUnique);
break;
case ARRAY:
case MAP:
// index the property holder for element names
createIndex(management, propertyName, String.class, field.isUnique);
break;
case STRUCT:
StructType structType = (StructType) field.dataType();
createIndexForFields(management, structType, structType.fieldMapping().fields);
break;
case TRAIT:
// do nothing since this is NOT contained in other types
break;
case CLASS:
// this is only A reference, index the attribute for edge
createEdgeIndex(management, propertyName, String.class);
break;
default:
throw new IllegalArgumentException("bad data type" + field.dataType().getName());
}
}
private Class getPrimitiveClass(IDataType dataType) {
if (dataType == DataTypes.STRING_TYPE) {
return String.class;
} else if (dataType == DataTypes.SHORT_TYPE) {
return Short.class;
} else if (dataType == DataTypes.INT_TYPE) {
return Integer.class;
} else if (dataType == DataTypes.BIGINTEGER_TYPE) {
return BigInteger.class;
} else if (dataType == DataTypes.BOOLEAN_TYPE) {
return Boolean.class;
} else if (dataType == DataTypes.BYTE_TYPE) {
return Byte.class;
} else if (dataType == DataTypes.LONG_TYPE) {
return Long.class;
} else if (dataType == DataTypes.FLOAT_TYPE) {
return Float.class;
} else if (dataType == DataTypes.DOUBLE_TYPE) {
return Double.class;
} else if (dataType == DataTypes.BIGDECIMAL_TYPE) {
return BigDecimal.class;
}
throw new IllegalArgumentException("unknown data type " + dataType);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName, Class propertyClass) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, false, Vertex.class);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName,
Class propertyClass, boolean isUnique) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, isUnique, Vertex.class);
}
private static PropertyKey createEdgeIndex(TitanManagement management,
String propertyName, Class propertyClass) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, false, Edge.class);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName, Class propertyClass,
Cardinality cardinality, boolean isUnique,
Class<? extends Element> elementClass) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
.makePropertyKey(propertyName)
.dataType(propertyClass)
.cardinality(cardinality)
.make();
TitanManagement.IndexBuilder indexBuilder = management
.buildIndex("index_" + propertyName, elementClass)
.addKey(propertyKey);
if (isUnique) {
indexBuilder = indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
return propertyKey;
}
}
......@@ -18,9 +18,8 @@
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
......@@ -91,21 +90,15 @@ public final class GraphHelper {
return results.hasNext() ? results.next() : null;
}
public static PropertyKey createPropertyKey(TitanManagement management, String propertyName) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
.makePropertyKey(propertyName)
.dataType(String.class)
.cardinality(Cardinality.SET)
.make();
management.buildIndex("by" + propertyName, Vertex.class)
.addKey(propertyKey)
.buildCompositeIndex();
}
public static Vertex findVertexByGUIDUsingIndex(TitanGraph titanGraph,
String value) {
LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
return propertyKey;
TitanIndexQuery query = titanGraph.indexQuery("index_" + Constants.GUID_PROPERTY_KEY,
Constants.GUID_PROPERTY_KEY + " = " + value);
Iterator<TitanIndexQuery.Result<Vertex>> results = query.vertices().iterator();
// returning one since guid should be unique
return results.hasNext() ? results.next().getElement() : null;
}
public static String vertexString(final Vertex vertex) {
......
......@@ -18,25 +18,6 @@
package org.apache.hadoop.metadata.repository.graph;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.thinkaurelius.titan.core.PropertyKey;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
......@@ -45,6 +26,20 @@ import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
/**
* Default implementation for Graph service backed by Titan.
......@@ -115,12 +110,11 @@ public class TitanGraphService implements GraphService {
return graphConfig;
}
/**
* Initializes the indices for the graph.
* @throws ConfigurationException
*/
// TODO move this functionality to the MetadataRepository?
// TODO move this functionality to the SearchIndexer?
protected void createIndicesForVertexKeys() throws ConfigurationException {
if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph");
......@@ -133,61 +127,7 @@ public class TitanGraphService implements GraphService {
TitanGraphIndex graphIndex = mgmt.buildIndex(Constants.INDEX_NAME, Vertex.class)
.buildMixedIndex(Constants.BACKING_INDEX);
PropertyKey guidKey = mgmt
.makePropertyKey(Constants.GUID_PROPERTY_KEY)
.dataType(String.class).make();
mgmt.buildIndex("byGUID", Vertex.class)
.addKey(guidKey)
.unique()
.buildCompositeIndex();
PropertyKey typeKey = mgmt
.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY)
.dataType(String.class).make();
mgmt.buildIndex("byType", Vertex.class)
.addKey(typeKey)
.buildCompositeIndex();
//TODO - Once we can get the TypeSystem to give me actual types, the below will be modified and replace the current
//indexer config.
/*
Iterator<String> i = typeSystem.getTypeNames().iterator();
// Get a list of property names to iterate through...
List<String> propList = new ArrayList<>();
while (i.hasNext()) {
String currType = i.next();
Iterator<String> typeDefIterator = null;
while (typeDefIterator.hasNext()) {
// Pull the property name and index, so we can register the name
// and look up the type.
String prop = "";
String type = "";
boolean isUnique = false;
// Add the key.
LOG.info("Adding property: " + prop + " to index as type: "
+ type);
mgmt.addIndexKey(graphIndex, mgmt.makePropertyKey(prop)
.dataType(type.getClass()).make());
}
mgmt.commit();
LOG.info("Index creation complete.");
}
*/
Configuration indexConfig = getConfiguration("indexer.properties", INDEXER_PREFIX);
// Properties are formatted: prop_name:type;prop_name:type
// E.g. Name:String;Date:Long
if (!indexConfig.isEmpty()) {
......@@ -241,7 +181,6 @@ public class TitanGraphService implements GraphService {
mgmt.commit();
LOG.info("Index creation complete.");
}
}
/**
......
......@@ -27,6 +27,7 @@ import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.listener.TypedInstanceChangeListener;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.SearchIndexer;
import org.apache.hadoop.metadata.storage.RepositoryException;
import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.TypeSystem;
......@@ -56,9 +57,12 @@ public class DefaultMetadataService implements MetadataService {
private final MetadataRepository repository;
@Inject
DefaultMetadataService(MetadataRepository repository) throws MetadataException {
DefaultMetadataService(MetadataRepository repository,
SearchIndexer searchIndexer) throws MetadataException {
this.typeSystem = TypeSystem.getInstance();
this.repository = repository;
registerListener(searchIndexer);
}
/**
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment