Skip to content

A
atlas
Commit 072aee8d by TJBChris
Options

Merge branch 'master' of https://github.com/hortonworks/metadata.git

parents b5539fc1 8cff9589
Showing with 76 additions and 145 deletions
common/pom.xml
......@@ -121,11 +121,6 @@
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
</dependency>
</dependencies>
<build>
......
hivetypes/src/test/resources/graph.properties
......@@ -16,7 +16,8 @@
# limitations under the License.
#
storage.backend=inmemory
storage.backend=berkeleyje
storage.directory=target/data/berkeley
# Graph Search Index
index.search.backend=elasticsearch
......
pom.xml
......@@ -579,6 +579,13 @@
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId>
<version>${titan.version}</version>
<!--<scope>test</scope>-->
</dependency>
<!-- metadata modules -->
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
......
repository/pom.xml
......@@ -118,12 +118,12 @@
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
<artifactId>titan-es</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-es</artifactId>
<artifactId>titan-lucene</artifactId>
</dependency>
<dependency>
......
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
......@@ -30,6 +30,8 @@ import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.hadoop.metadata.discovery.DiscoveryService;
import org.apache.hadoop.metadata.discovery.GraphBackedDiscoveryService;
import org.apache.hadoop.metadata.repository.SearchIndexer;
import org.apache.hadoop.metadata.repository.graph.GraphBackedSearchIndexer;
import org.apache.hadoop.metadata.services.DefaultMetadataService;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphProvider;
......@@ -51,6 +53,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private Class<? extends MetadataRepository> metadataRepoClass;
private Class<? extends MetadataService> metadataService;
private Class<? extends DiscoveryService> discoveryService;
private Class<? extends SearchIndexer> searchIndexer;
public RepositoryMetadataModule() {
GraphServiceConfigurator gsp = new GraphServiceConfigurator();
......@@ -60,6 +63,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
this.metadataRepoClass = GraphBackedMetadataRepository.class;
this.metadataService = DefaultMetadataService.class;
this.discoveryService = GraphBackedDiscoveryService.class;
this.searchIndexer = GraphBackedSearchIndexer.class;
}
protected void configure() {
......@@ -82,5 +86,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(discoveryService);
bind(SearchIndexer.class).to(searchIndexer);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/SearchIndexer.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
/**
* Interface for indexing types.
*/
public interface SearchIndexer extends TypesChangeListener {
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -620,9 +620,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
// todo: move this to the indexer
GraphHelper.createPropertyKey(titanGraph.getManagementSystem(), propertyName);
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
StringBuilder buffer = new StringBuilder();
......@@ -652,9 +649,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
// todo: move this to the indexer
GraphHelper.createPropertyKey(titanGraph.getManagementSystem(), propertyName);
StringBuilder buffer = new StringBuilder();
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
for (Map.Entry entry : collection.entrySet()) {
......@@ -719,11 +713,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
} else {
referenceVertex = idToVertexMap.get(id);
/*
ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
mapInstanceToVertex(id, typedInstance, referenceVertex,
classType.fieldMapping().fields, idToVertexMap);
*/
}
if (referenceVertex != null) {
......@@ -998,38 +987,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
propertyNameWithSuffix.lastIndexOf("."), propertyNameWithSuffix.length());
values.put(key, mapVertexToCollectionEntry(
instanceVertex, attributeInfo, elementType, propertyNameWithSuffix));
/*
switch (valueType.getTypeCategory()) {
case PRIMITIVE:
values.put(key, instanceVertex.getProperty(propertyNameWithSuffix));
break;
case ENUM:
values.put(key, instanceVertex.getProperty(propertyNameWithSuffix));
break;
case ARRAY:
case MAP:
case TRAIT:
// do nothing
break;
case STRUCT:
ITypedStruct structInstance = getStructInstanceFromVertex(instanceVertex,
valueType, attributeInfo.name, propertyNameWithSuffix);
values.put(key, structInstance);
break;
case CLASS:
Object idOrInstance = mapClassReferenceToVertex(
instanceVertex, attributeInfo, propertyNameWithSuffix);
values.put(key, idOrInstance);
break;
default:
break;
}
*/
}
typedInstance.set(attributeInfo.name, values);
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
......@@ -18,9 +18,8 @@
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
......@@ -91,21 +90,15 @@ public final class GraphHelper {
return results.hasNext() ? results.next() : null;
}
public static PropertyKey createPropertyKey(TitanManagement management, String propertyName) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
.makePropertyKey(propertyName)
.dataType(String.class)
.cardinality(Cardinality.SET)
.make();
management.buildIndex("by" + propertyName, Vertex.class)
.addKey(propertyKey)
.buildCompositeIndex();
}
public static Vertex findVertexByGUIDUsingIndex(TitanGraph titanGraph,
String value) {
LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
return propertyKey;
TitanIndexQuery query = titanGraph.indexQuery("index_" + Constants.GUID_PROPERTY_KEY,
Constants.GUID_PROPERTY_KEY + " = " + value);
Iterator<TitanIndexQuery.Result<Vertex>> results = query.vertices().iterator();
// returning one since guid should be unique
return results.hasNext() ? results.next().getElement() : null;
}
public static String vertexString(final Vertex vertex) {
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphService.java
......@@ -18,25 +18,6 @@
package org.apache.hadoop.metadata.repository.graph;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.thinkaurelius.titan.core.PropertyKey;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
......@@ -45,6 +26,20 @@ import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
/**
* Default implementation for Graph service backed by Titan.
......@@ -115,12 +110,11 @@ public class TitanGraphService implements GraphService {
return graphConfig;
}
/**
* Initializes the indices for the graph.
* @throws ConfigurationException
*/
// TODO move this functionality to the MetadataRepository?
// TODO move this functionality to the SearchIndexer?
protected void createIndicesForVertexKeys() throws ConfigurationException {
if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph");
......@@ -133,61 +127,7 @@ public class TitanGraphService implements GraphService {
TitanGraphIndex graphIndex = mgmt.buildIndex(Constants.INDEX_NAME, Vertex.class)
.buildMixedIndex(Constants.BACKING_INDEX);
PropertyKey guidKey = mgmt
.makePropertyKey(Constants.GUID_PROPERTY_KEY)
.dataType(String.class).make();
mgmt.buildIndex("byGUID", Vertex.class)
.addKey(guidKey)
.unique()
.buildCompositeIndex();
PropertyKey typeKey = mgmt
.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY)
.dataType(String.class).make();
mgmt.buildIndex("byType", Vertex.class)
.addKey(typeKey)
.buildCompositeIndex();
//TODO - Once we can get the TypeSystem to give me actual types, the below will be modified and replace the current
//indexer config.
/*
Iterator<String> i = typeSystem.getTypeNames().iterator();
// Get a list of property names to iterate through...
List<String> propList = new ArrayList<>();
while (i.hasNext()) {
String currType = i.next();
Iterator<String> typeDefIterator = null;
while (typeDefIterator.hasNext()) {
// Pull the property name and index, so we can register the name
// and look up the type.
String prop = "";
String type = "";
boolean isUnique = false;
// Add the key.
LOG.info("Adding property: " + prop + " to index as type: "
+ type);
mgmt.addIndexKey(graphIndex, mgmt.makePropertyKey(prop)
.dataType(type.getClass()).make());
}
mgmt.commit();
LOG.info("Index creation complete.");
}
*/
Configuration indexConfig = getConfiguration("indexer.properties", INDEXER_PREFIX);
// Properties are formatted: prop_name:type;prop_name:type
// E.g. Name:String;Date:Long
if (!indexConfig.isEmpty()) {
......@@ -241,7 +181,6 @@ public class TitanGraphService implements GraphService {
mgmt.commit();
LOG.info("Index creation complete.");
}
}
/**
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
......@@ -27,6 +27,7 @@ import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.listener.TypedInstanceChangeListener;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.SearchIndexer;
import org.apache.hadoop.metadata.storage.RepositoryException;
import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.TypeSystem;
......@@ -56,9 +57,12 @@ public class DefaultMetadataService implements MetadataService {
private final MetadataRepository repository;
@Inject
DefaultMetadataService(MetadataRepository repository) throws MetadataException {
DefaultMetadataService(MetadataRepository repository,
SearchIndexer searchIndexer) throws MetadataException {
this.typeSystem = TypeSystem.getInstance();
this.repository = repository;
registerListener(searchIndexer);
}
/**
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment