Skip to content

A
atlas
Commit f7fe3750 by Suma Shivaprasad
Options

Fixed indexing failures for specific types and due to missing transaction…

Fixed indexing failures for specific types and due to missing transaction closures. Todo - Nested transactions with write lock on graph and nested read transaction in DMS.createType are not working with bdb
parent ed6b9bf4
Showing with 270 additions and 62 deletions
repository/src/main/java/org/apache/hadoop/metadata/GraphTransactionInterceptor.java
......@@ -44,7 +44,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
return response;
} catch (Throwable t){
titanGraph.rollback();
LOG.debug("graph rollback");
LOG.error("graph rollback due to exception ", t);
throw t;
}
}
......
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
......@@ -42,6 +42,7 @@ import org.apache.hadoop.metadata.services.MetadataService;
* Guice module for Repository module.
*/
public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
@Override
protected void configure() {
// special wiring for Titan Graph
......@@ -67,7 +68,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
bind(SearchIndexer.class).to(GraphBackedSearchIndexer.class).asEagerSingleton();
bind(SearchIndexer.class).to(GraphBackedSearchIndexer.class);
bind(LineageService.class).to(HiveLineageService.class).asEagerSingleton();
......@@ -75,4 +76,5 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
requestInjection(interceptor);
bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
}
}
repository/src/main/java/org/apache/hadoop/metadata/discovery/SearchIndexer.java
......@@ -19,6 +19,7 @@
package org.apache.hadoop.metadata.discovery;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
import org.apache.hadoop.metadata.repository.IndexException;
/**
......@@ -26,4 +27,11 @@ import org.apache.hadoop.metadata.listener.TypesChangeListener;
*/
public interface SearchIndexer extends TypesChangeListener {
/* Commit the indexes */
void commit() throws IndexException;
/* RollBack the index */
void rollback() throws IndexException;
}
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/DefaultGraphPersistenceStrategy.java
......@@ -19,6 +19,7 @@
package org.apache.hadoop.metadata.discovery.graph;
import com.thinkaurelius.titan.core.TitanVertex;
import org.apache.hadoop.metadata.GraphTransaction;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.query.Expressions;
import org.apache.hadoop.metadata.query.GraphPersistenceStrategies;
......@@ -90,6 +91,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
}
@Override
@GraphTransaction
public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
return metadataRepository.getIdFromVertex(dataTypeName, vertex);
}
......
repository/src/main/java/org/apache/hadoop/metadata/repository/IndexCreationException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository;
public class IndexCreationException extends IndexException {
public IndexCreationException() {
}
public IndexCreationException(String message) {
super(message);
}
public IndexCreationException(String message, Throwable cause) {
super(message, cause);
}
public IndexCreationException(Throwable cause) {
super(cause);
}
public IndexCreationException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/IndexException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.repository;
import org.apache.hadoop.metadata.MetadataException;
public class IndexException extends MetadataException {
public IndexException() {
}
public IndexException(String message) {
super(message);
}
public IndexException(String message, Throwable cause) {
super(message, cause);
}
public IndexException(Throwable cause) {
super(cause);
}
public IndexException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -181,6 +181,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
@Override
@GraphTransaction
public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.info("Retrieving entity list for type={}", entityType);
GraphQuery query = titanGraph.query()
......@@ -421,7 +422,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
public void createVerticesForClassTypes(
private void createVerticesForClassTypes(
List<ITypedReferenceableInstance> newInstances) throws MetadataException {
for (ITypedReferenceableInstance typedInstance : newInstances) {
final Id id = typedInstance.getId();
......@@ -927,6 +928,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public final class GraphToTypedInstanceMapper {
@GraphTransaction
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid,
Vertex instanceVertex)
throws MetadataException {
......@@ -950,7 +952,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return typedInstance;
}
public void mapVertexToInstanceTraits(Vertex instanceVertex,
private void mapVertexToInstanceTraits(Vertex instanceVertex,
ITypedReferenceableInstance typedInstance,
List<String> traits) throws MetadataException {
for (String traitName : traits) {
......@@ -961,6 +963,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
@GraphTransaction
public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
Map<String, AttributeInfo> fields)
throws MetadataException {
......@@ -972,7 +975,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
public void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
final IDataType dataType = attributeInfo.dataType();
......@@ -1022,7 +1026,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
public Object mapClassReferenceToVertex(Vertex instanceVertex,
private Object mapClassReferenceToVertex(Vertex instanceVertex,
AttributeInfo attributeInfo,
String relationshipLabel,
IDataType dataType) throws MetadataException {
......@@ -1052,7 +1056,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
@SuppressWarnings("unchecked")
public void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo,
String propertyName) throws MetadataException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
......@@ -1072,7 +1076,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
typedInstance.set(attributeInfo.name, values);
}
public Object mapVertexToCollectionEntry(Vertex instanceVertex,
private Object mapVertexToCollectionEntry(Vertex instanceVertex,
AttributeInfo attributeInfo,
IDataType elementType, Object value, String propertyName)
throws MetadataException {
......@@ -1156,7 +1160,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return null;
}
public Object mapClassReferenceToVertex(Vertex instanceVertex,
private Object mapClassReferenceToVertex(Vertex instanceVertex,
AttributeInfo attributeInfo,
String relationshipLabel,
IDataType dataType,
......@@ -1223,7 +1227,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
traitName, traitType, traitInstance);
}
public void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName,
private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName,
String traitName, TraitType traitType,
ITypedStruct traitInstance) throws MetadataException {
String relationshipLabel = getEdgeLabel(typedInstanceTypeName, traitName);
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
......@@ -18,21 +18,18 @@
package org.apache.hadoop.metadata.repository.graph;
import com.google.inject.Singleton;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.Order;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.discovery.SearchIndexer;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.IndexException;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
......@@ -52,19 +49,24 @@ import java.util.Map;
/**
* Adds index for properties of a given type when its added before any instances are added.
*/
@Singleton
public class GraphBackedSearchIndexer implements SearchIndexer {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
private final TitanGraph titanGraph;
private TitanManagement management;
@Inject
public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider)
throws RepositoryException {
this.titanGraph = graphProvider.get();
/* Create the transaction for indexing.
* Commit/rollback is expected to be called from the caller.
*/
management = titanGraph.getManagementSystem();
initialize();
}
......@@ -72,18 +74,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
* Initializes the indices for the graph - create indices for Global Vertex Keys
*/
private void initialize() {
TitanManagement management = titanGraph.getManagementSystem();
if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
LOG.info("Global indexes already exist for graph");
return;
}
/* This is called only once, which is the first time Atlas types are made indexable .*/
LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
.buildMixedIndex(Constants.BACKING_INDEX);
management.buildIndex(Constants.EDGE_INDEX, Edge.class)
.buildMixedIndex(Constants.BACKING_INDEX);
management.commit();
// create a composite index for guid as its unique
createCompositeIndex(Constants.GUID_INDEX,
......@@ -108,18 +109,20 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
//Indexes for graph backed type system store
createTypeStoreIndexes();
management.commit();
//Make sure we acquire another transaction after commit for subsequent indexing
management = titanGraph.getManagementSystem();
LOG.info("Index creation for global keys complete.");
}
private void createFullTextIndex() {
TitanManagement management = titanGraph.getManagementSystem();
PropertyKey fullText =
management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();
management.buildIndex(Constants.FULLTEXT_INDEX, Vertex.class)
.addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
.buildMixedIndex(Constants.BACKING_INDEX);
management.commit();
LOG.info("Created mixed index for {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
}
......@@ -151,8 +154,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
} catch (Throwable throwable) {
// gets handle to currently open transaction
titanGraph.getManagementSystem().rollback();
LOG.error("Error creating index for type {}", dataType, throwable);
throw new IndexException("Error while creating index for type " + dataType, throwable);
}
}
......@@ -285,7 +288,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
private PropertyKey createCompositeIndex(String indexName,
String propertyName, Class propertyClass,
boolean isUnique, Cardinality cardinality) {
TitanManagement management = titanGraph.getManagementSystem();
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
......@@ -303,7 +305,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
}
indexBuilder.buildCompositeIndex();
management.commit();
LOG.info("Created index for property {} in composite index {}", propertyName, indexName);
}
......@@ -312,7 +313,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
}
private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) {
TitanManagement management = titanGraph.getManagementSystem();
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
// ignored cardinality as Can only index single-valued property keys on vertices
......@@ -321,15 +322,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
.dataType(propertyClass)
.make();
if (propertyClass == Boolean.class) {
if (!checkIfMixedIndexApplicable(propertyClass)) {
LOG.debug("Creating composite index for property {} of type {} ", propertyName, propertyClass.getName());
//Use standard index as backing index only supports string, int and geo types
management.buildIndex(propertyName, Vertex.class).addKey(propertyKey).buildCompositeIndex();
management.commit();
LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
} else {
//Use backing index
LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey);
management.commit();
LOG.debug("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
}
LOG.info("Created mixed vertex index for property {}", propertyName);
}
......@@ -337,15 +341,41 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
return propertyKey;
}
private boolean checkIfMixedIndexApplicable(Class propertyClass) {
//TODO - Check why date types are failing in ES/Solr
if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class || propertyClass == Date.class) {
return false;
}
return true;
}
@Override
public void commit() throws IndexException {
try {
management.commit();
} catch(Exception e) {
LOG.error("Index commit failed" , e);
throw new IndexException("Index commit failed " , e);
}
}
@Override
public void rollback() throws IndexException {
try {
management.rollback();
} catch(Exception e) {
LOG.error("Index rollback failed " , e);
throw new IndexException("Index rollback failed " , e);
}
}
/* Commenting this out since we do not need an index for edge label here
private void createEdgeMixedIndex(String propertyName) {
TitanManagement management = titanGraph.getManagementSystem();
EdgeLabel edgeLabel = management.getEdgeLabel(propertyName);
if (edgeLabel == null) {
edgeLabel = management.makeEdgeLabel(propertyName).make();
management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
management.commit();
LOG.info("Created index for edge label {}", propertyName);
}
} */
}*/
}
repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java
......@@ -75,6 +75,7 @@ public class GraphBackedTypeStore implements ITypeStore {
}
@Override
@GraphTransaction
public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws MetadataException {
ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
for (String typeName : typeNames) {
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
......@@ -20,14 +20,19 @@ package org.apache.hadoop.metadata.services;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Provider;
import org.apache.hadoop.metadata.GraphTransaction;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.ParamChecker;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.classification.InterfaceAudience;
import org.apache.hadoop.metadata.discovery.SearchIndexer;
import org.apache.hadoop.metadata.listener.EntityChangeListener;
import org.apache.hadoop.metadata.listener.TypesChangeListener;
import org.apache.hadoop.metadata.repository.IndexCreationException;
import org.apache.hadoop.metadata.repository.IndexException;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
......@@ -53,6 +58,8 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.IOException;
import java.text.ParseException;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
......@@ -68,23 +75,23 @@ public class DefaultMetadataService implements MetadataService {
private static final Logger LOG =
LoggerFactory.getLogger(DefaultMetadataService.class);
private final Set<TypesChangeListener> typesChangeListeners = new LinkedHashSet<>();
private final Set<EntityChangeListener> entityChangeListeners
= new LinkedHashSet<>();
private final TypeSystem typeSystem;
private final MetadataRepository repository;
private final ITypeStore typeStore;
private final Provider<SearchIndexer> searchIndexProvider;
@Inject
DefaultMetadataService(MetadataRepository repository,
SearchIndexer searchIndexer, ITypeStore typeStore) throws MetadataException {
Provider<SearchIndexer> searchIndexProvider, ITypeStore typeStore) throws MetadataException {
this.typeStore = typeStore;
this.typeSystem = TypeSystem.getInstance();
this.repository = repository;
this.searchIndexProvider = searchIndexProvider;
restoreTypeSystem();
registerListener(searchIndexer);
}
private void restoreTypeSystem() {
......@@ -146,19 +153,18 @@ public class DefaultMetadataService implements MetadataService {
* @return a unique id for this type
*/
@Override
@GraphTransaction
public JSONObject createType(String typeDefinition) throws MetadataException {
ParamChecker.notEmpty(typeDefinition, "type definition cannot be empty");
TypesDef typesDef;
try {
typesDef = TypesSerialization.fromJson(typeDefinition);
if(typesDef.isEmpty()) {
if (typesDef.isEmpty()) {
throw new MetadataException("Invalid type definition");
}
} catch (Exception e) {
LOG.error("Unable to deserialize json={}", typeDefinition, e);
throw new IllegalArgumentException("Unable to deserialize json");
throw new IllegalArgumentException("Unable to deserialize json ", e);
}
try {
......@@ -167,9 +173,9 @@ public class DefaultMetadataService implements MetadataService {
try {
typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
onTypesAddedToRepo(typesAdded);
} catch(Throwable t) {
typeSystem.removeTypes(ImmutableList.copyOf(typesAdded.keySet()));
throw new MetadataException(t);
} catch (Throwable t) {
typeSystem.removeTypes(typesAdded);
throw new MetadataException("Unable to persist types ", t);
}
return new JSONObject() {{
......@@ -177,7 +183,7 @@ public class DefaultMetadataService implements MetadataService {
}};
} catch (JSONException e) {
LOG.error("Unable to create response for types={}", typeDefinition, e);
throw new MetadataException("Unable to create response");
throw new MetadataException("Unable to create response ", e);
}
}
......@@ -334,7 +340,7 @@ public class DefaultMetadataService implements MetadataService {
}
private ITypedStruct deserializeTraitInstance(String traitInstanceDefinition)
throws MetadataException {
throws MetadataException {
try {
Struct traitInstance = InstanceSerialization.fromJsonStruct(
......@@ -374,23 +380,21 @@ public class DefaultMetadataService implements MetadataService {
}
private void onTypesAddedToRepo(Map<String, IDataType> typesAdded) throws MetadataException {
for (TypesChangeListener listener : typesChangeListeners) {
final SearchIndexer indexer = searchIndexProvider.get();
try {
for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
listener.onAdd(entry.getKey(), entry.getValue());
indexer.onAdd(entry.getKey(), entry.getValue());
}
indexer.commit();
} catch(IndexCreationException ice) {
indexer.rollback();
throw ice;
}
}
public void registerListener(TypesChangeListener listener) {
typesChangeListeners.add(listener);
}
public void unregisterListener(TypesChangeListener listener) {
typesChangeListeners.remove(listener);
}
private void onEntityAddedToRepo(ITypedReferenceableInstance typedInstance)
throws MetadataException {
throws MetadataException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntityAdded(typedInstance);
......@@ -418,4 +422,5 @@ public class DefaultMetadataService implements MetadataService {
public void unregisterListener(EntityChangeListener listener) {
entityChangeListeners.remove(listener);
}
}
}
\ No newline at end of file
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperScaleTest.java
......@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......@@ -19,11 +19,24 @@
package org.apache.hadoop.metadata.repository.graph;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
import com.thinkaurelius.titan.diskstorage.configuration.ModifiableConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.metadata.GraphTransaction;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.PropertiesUtil;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
......@@ -42,16 +55,20 @@ import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
import org.junit.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
@Test
@Guice(modules = RepositoryMetadataModule.class)
public class GraphRepoMapperScaleTest {
private static final String DATABASE_TYPE = "hive_database_type";
......@@ -59,9 +76,39 @@ public class GraphRepoMapperScaleTest {
private static final String TABLE_TYPE = "hive_table_type";
private static final String TABLE_NAME = "bar";
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private static final String INDEX_DIR = System.getProperty("java.io.tmpdir", "/tmp") + "/titan-schema-test" + new Random().nextLong();
private GraphProvider<TitanGraph> graphProvider = new GraphProvider<TitanGraph>() {
private TitanGraph graph = null;
//Ensure separate directory for graph provider to avoid issues with index merging
@Override
public TitanGraph get() {
try {
if (graph == null) {
synchronized (GraphRepoMapperScaleTest.class) {
if (graph == null) {
ReadConfiguration config = new CommonsConfiguration() {{
set("storage.backend", "inmemory");
set("index.search.directory", INDEX_DIR);
set("index.search.backend", "elasticsearch");
set("index.search.elasticsearch.local-mode", "true");
set("index.search.elasticsearch.client-only", "false");
}};
GraphDatabaseConfiguration graphconfig = new GraphDatabaseConfiguration(config);
graphconfig.getBackend().clearStorage();
graph = TitanFactory.open(config);
}
}
}
} catch (BackendException e) {
e.printStackTrace();
}
return graph;
}
};
private GraphBackedMetadataRepository repositoryService;
private GraphBackedSearchIndexer searchIndexer;
......@@ -69,7 +116,11 @@ public class GraphRepoMapperScaleTest {
private String dbGUID;
@BeforeClass
@GraphTransaction
public void setUp() throws Exception {
//Make sure we can cleanup the index directory
repositoryService = new GraphBackedMetadataRepository(graphProvider);
searchIndexer = new GraphBackedSearchIndexer(graphProvider);
typeSystem = TypeSystem.getInstance();
......@@ -77,6 +128,16 @@ public class GraphRepoMapperScaleTest {
createHiveTypes();
}
@AfterClass
public void tearDown() throws Exception {
graphProvider.get().shutdown();
try {
FileUtils.deleteDirectory(new File(INDEX_DIR));
} catch(IOException ioe) {
System.err.println("Failed to cleanup index directory");
}
}
@Test
public void testSubmitEntity() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
......@@ -220,6 +281,7 @@ public class GraphRepoMapperScaleTest {
for (Map.Entry<String, IDataType> entry : types.entrySet()) {
searchIndexer.onAdd(entry.getKey(), entry.getValue());
}
searchIndexer.commit();
}
private ITypedReferenceableInstance createHiveTableInstance(
......@@ -270,3 +332,4 @@ public class GraphRepoMapperScaleTest {
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
}
repository/src/test/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -81,7 +81,6 @@ public class GraphBackedTypeStoreTest {
}
@Test (dependsOnMethods = "testStore")
@GraphTransaction
public void testRestore() throws Exception {
TypesDef types = typeStore.restore();
......
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
......@@ -307,8 +307,13 @@ public class TypeSystem {
return false;
}
public void removeTypes(ImmutableList<String> typeNames) {
public void removeTypes(Map<String, IDataType> typeNames) {
for(String typeName : typeNames.keySet()) {
types.remove(typeName);
IDataType dataType = typeNames.get(typeName);
final DataTypes.TypeCategory typeCategory = dataType.getTypeCategory();
typeCategoriesToTypeNamesMap.get(typeCategory).remove(typeName);
}
}
class TransientTypeSystem extends TypeSystem {
......
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java
......@@ -217,6 +217,9 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef("columnNames", DataTypes.arrayTypeName(DataTypes.STRING_TYPE)),
TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
TypesUtil.createOptionalAttrDef("parameters", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE)),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition("database",
"database", Multiplicity.REQUIRED, false, "database"));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment