Skip to content

A
atlas
Commit b10b34e0 by Shwetha G S
Options

Merge pull request #139 from sumashivaprasad/api_merged

Fixed indexing issues across data types not supported in backing index and API response codes
parents 189ba064 92afb649
Showing with 358 additions and 92 deletions
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/ABridge.java
......@@ -85,7 +85,7 @@ public abstract class ABridge implements IBridge {
ITypedReferenceableInstance refBean = null;
try {
refBean = type.convert(this.convertToReferencable(bean), Multiplicity.REQUIRED);
String id = repo.createEntity(refBean, type.getName());
String id = repo.createEntity(refBean);
return id;
} catch (IllegalArgumentException | IllegalAccessException e) {
// TODO Auto-generated catch block
......
repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
......@@ -44,7 +44,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
return response;
} catch (Throwable t){
titanGraph.rollback();
LOG.debug("graph rollback");
LOG.error("graph rollback due to exception ", t);
throw t;
}
}
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -41,6 +41,7 @@ import org.apache.atlas.services.MetadataService;
* Guice module for Repository module.
*/
public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
@Override
protected void configure() {
// special wiring for Titan Graph
......@@ -66,7 +67,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
bind(SearchIndexer.class).to(GraphBackedSearchIndexer.class).asEagerSingleton();
bind(SearchIndexer.class).to(GraphBackedSearchIndexer.class);
bind(LineageService.class).to(HiveLineageService.class).asEagerSingleton();
......@@ -74,4 +75,5 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
requestInjection(interceptor);
bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
}
}
repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.discovery;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.IndexException;
/**
......@@ -26,4 +27,11 @@ import org.apache.atlas.listener.TypesChangeListener;
*/
public interface SearchIndexer extends TypesChangeListener {
/* Commit the indexes */
void commit() throws IndexException;
/* RollBack the index */
void rollback() throws IndexException;
}
repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.discovery.graph;
import com.thinkaurelius.titan.core.TitanVertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.MetadataException;
import org.apache.atlas.query.Expressions;
import org.apache.atlas.query.GraphPersistenceStrategies;
......
repository/src/main/java/org/apache/atlas/repository/IndexCreationException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository;
public class IndexCreationException extends IndexException {
public IndexCreationException() {
}
public IndexCreationException(String message) {
super(message);
}
public IndexCreationException(String message, Throwable cause) {
super(message, cause);
}
public IndexCreationException(Throwable cause) {
super(cause);
}
public IndexCreationException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
repository/src/main/java/org/apache/atlas/repository/IndexException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository;
import org.apache.atlas.MetadataException;
public class IndexException extends MetadataException {
public IndexException() {
}
public IndexException(String message) {
super(message);
}
public IndexException(String message, Throwable cause) {
super(message, cause);
}
public IndexException(Throwable cause) {
super(cause);
}
public IndexException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -257,6 +257,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
((TitanVertex) instanceVertex)
.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
} catch (RepositoryException e) {
throw e;
} catch (Exception e) {
throw new RepositoryException(e);
}
......@@ -279,7 +281,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
List<String> traitNames = getTraitNames(instanceVertex);
if (!traitNames.contains(traitNameToBeDeleted)) {
throw new RepositoryException("Could not find trait=" + traitNameToBeDeleted
throw new EntityNotFoundException("Could not find trait=" + traitNameToBeDeleted
+ " in the repository for entity: " + guid);
}
......@@ -302,6 +304,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
updateTraits(instanceVertex, traitNames);
}
}
} catch (RepositoryException e) {
throw e;
} catch (Exception e) {
throw new RepositoryException(e);
}
......@@ -350,6 +354,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
instanceToGraphMapper.mapAttributesToVertex(getIdFromVertex(typeName, instanceVertex),
instance, instanceVertex, new HashMap<Id, Vertex>(),
attributeInfo, attributeInfo.dataType());
} catch (RepositoryException e) {
throw e;
} catch (Exception e) {
throw new RepositoryException(e);
}
......@@ -426,7 +432,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
public void createVerticesForClassTypes(
private void createVerticesForClassTypes(
List<ITypedReferenceableInstance> newInstances) throws MetadataException {
for (ITypedReferenceableInstance typedInstance : newInstances) {
final Id id = typedInstance.getId();
......@@ -955,7 +961,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return typedInstance;
}
public void mapVertexToInstanceTraits(Vertex instanceVertex,
private void mapVertexToInstanceTraits(Vertex instanceVertex,
ITypedReferenceableInstance typedInstance,
List<String> traits) throws MetadataException {
for (String traitName : traits) {
......@@ -977,7 +983,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
public void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
final IDataType dataType = attributeInfo.dataType();
......@@ -1027,7 +1034,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
public Object mapClassReferenceToVertex(Vertex instanceVertex,
private Object mapClassReferenceToVertex(Vertex instanceVertex,
AttributeInfo attributeInfo,
String relationshipLabel,
IDataType dataType) throws MetadataException {
......@@ -1057,7 +1064,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
@SuppressWarnings("unchecked")
public void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo,
String propertyName) throws MetadataException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
......@@ -1077,7 +1084,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
typedInstance.set(attributeInfo.name, values);
}
public Object mapVertexToCollectionEntry(Vertex instanceVertex,
private Object mapVertexToCollectionEntry(Vertex instanceVertex,
AttributeInfo attributeInfo,
IDataType elementType, Object value, String propertyName)
throws MetadataException {
......@@ -1161,7 +1168,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return null;
}
public Object mapClassReferenceToVertex(Vertex instanceVertex,
private Object mapClassReferenceToVertex(Vertex instanceVertex,
AttributeInfo attributeInfo,
String relationshipLabel,
IDataType dataType,
......@@ -1228,7 +1235,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
traitName, traitType, traitInstance);
}
public void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName,
private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName,
String traitName, TraitType traitType,
ITypedStruct traitInstance) throws MetadataException {
String relationshipLabel = getEdgeLabel(typedInstanceTypeName, traitName);
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -18,7 +18,6 @@
package org.apache.atlas.repository.graph;
import com.google.inject.Singleton;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
......@@ -30,6 +29,8 @@ import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.MetadataException;
import org.apache.atlas.discovery.SearchIndexer;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.IndexCreationException;
import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
......@@ -49,19 +50,24 @@ import java.util.Map;
/**
* Adds index for properties of a given type when its added before any instances are added.
*/
@Singleton
public class GraphBackedSearchIndexer implements SearchIndexer {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
private final TitanGraph titanGraph;
private TitanManagement management;
@Inject
public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider)
throws RepositoryException {
this.titanGraph = graphProvider.get();
/* Create the transaction for indexing.
* Commit/rollback is expected to be called from the caller.
*/
management = titanGraph.getManagementSystem();
initialize();
}
......@@ -69,18 +75,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
* Initializes the indices for the graph - create indices for Global Vertex Keys
*/
private void initialize() {
TitanManagement management = titanGraph.getManagementSystem();
if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
LOG.info("Global indexes already exist for graph");
return;
}
/* This is called only once, which is the first time Atlas types are made indexable .*/
LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
.buildMixedIndex(Constants.BACKING_INDEX);
management.buildIndex(Constants.EDGE_INDEX, Edge.class)
.buildMixedIndex(Constants.BACKING_INDEX);
management.commit();
// create a composite index for guid as its unique
createCompositeIndex(Constants.GUID_INDEX,
......@@ -105,18 +110,20 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
//Indexes for graph backed type system store
createTypeStoreIndexes();
management.commit();
//Make sure we acquire another transaction after commit for subsequent indexing
management = titanGraph.getManagementSystem();
LOG.info("Index creation for global keys complete.");
}
private void createFullTextIndex() {
TitanManagement management = titanGraph.getManagementSystem();
PropertyKey fullText =
management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();
management.buildIndex(Constants.FULLTEXT_INDEX, Vertex.class)
.addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
.buildMixedIndex(Constants.BACKING_INDEX);
management.commit();
LOG.info("Created mixed index for {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
}
......@@ -147,9 +154,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
LOG.info("Index creation for type {} complete", typeName);
} catch (Throwable throwable) {
// gets handle to currently open transaction
titanGraph.getManagementSystem().rollback();
LOG.error("Error creating index for type {}", dataType, throwable);
throw new IndexCreationException("Error while creating index for type " + dataType, throwable);
}
}
......@@ -282,7 +288,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
private PropertyKey createCompositeIndex(String indexName,
String propertyName, Class propertyClass,
boolean isUnique, Cardinality cardinality) {
TitanManagement management = titanGraph.getManagementSystem();
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
......@@ -300,8 +305,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
}
indexBuilder.buildCompositeIndex();
management.commit();
LOG.info("Created index for property {} in composite index {}", propertyName, indexName);
}
......@@ -309,7 +312,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
}
private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) {
TitanManagement management = titanGraph.getManagementSystem();
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
// ignored cardinality as Can only index single-valued property keys on vertices
......@@ -318,15 +321,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
.dataType(propertyClass)
.make();
if (propertyClass == Boolean.class) {
if (!checkIfMixedIndexApplicable(propertyClass)) {
LOG.debug("Creating composite index for property {} of type {} ", propertyName, propertyClass.getName());
//Use standard index as backing index only supports string, int and geo types
management.buildIndex(propertyName, Vertex.class).addKey(propertyKey).buildCompositeIndex();
management.commit();
LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
} else {
//Use backing index
LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey);
management.commit();
LOG.debug("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
}
LOG.info("Created mixed vertex index for property {}", propertyName);
}
......@@ -334,15 +339,41 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
return propertyKey;
}
private boolean checkIfMixedIndexApplicable(Class propertyClass) {
//TODO - Check why date types are failing in ES/Solr
if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class || propertyClass == Date.class) {
return false;
}
return true;
}
@Override
public void commit() throws IndexException {
try {
management.commit();
} catch (Exception e) {
LOG.error("Index commit failed" , e);
throw new IndexException("Index commit failed " , e);
}
}
@Override
public void rollback() throws IndexException {
try {
management.rollback();
} catch (Exception e) {
LOG.error("Index rollback failed " , e);
throw new IndexException("Index rollback failed " , e);
}
}
/* Commenting this out since we do not need an index for edge label here
private void createEdgeMixedIndex(String propertyName) {
TitanManagement management = titanGraph.getManagementSystem();
EdgeLabel edgeLabel = management.getEdgeLabel(propertyName);
if (edgeLabel == null) {
edgeLabel = management.makeEdgeLabel(propertyName).make();
management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
management.commit();
LOG.info("Created index for edge label {}", propertyName);
}
} */
}*/
}
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
......@@ -74,6 +74,7 @@ public class GraphBackedTypeStore implements ITypeStore {
}
@Override
@GraphTransaction
public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws MetadataException {
ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
for (String typeName : typeNames) {
......
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -20,14 +20,17 @@ package org.apache.atlas.services;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.inject.Provider;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.MetadataException;
import org.apache.atlas.MetadataServiceClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.discovery.SearchIndexer;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.IndexCreationException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -56,6 +59,9 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
......@@ -71,22 +77,22 @@ public class DefaultMetadataService implements MetadataService {
private static final Logger LOG =
LoggerFactory.getLogger(DefaultMetadataService.class);
private final Set<TypesChangeListener> typesChangeListeners = new LinkedHashSet<>();
private final Set<EntityChangeListener> entityChangeListeners
= new LinkedHashSet<>();
private final TypeSystem typeSystem;
private final MetadataRepository repository;
private final ITypeStore typeStore;
private final Set<Provider<SearchIndexer>> typeChangeListeners;
@Inject
DefaultMetadataService(MetadataRepository repository,
SearchIndexer searchIndexer, ITypeStore typeStore) throws MetadataException {
DefaultMetadataService(final MetadataRepository repository,
final Provider<SearchIndexer> searchIndexProvider, final ITypeStore typeStore) throws MetadataException {
this.typeStore = typeStore;
this.typeSystem = TypeSystem.getInstance();
this.repository = repository;
registerListener(searchIndexer);
this.typeChangeListeners = new LinkedHashSet<Provider<SearchIndexer>>() {{ add(searchIndexProvider); }};
restoreTypeSystem();
}
......@@ -149,30 +155,32 @@ public class DefaultMetadataService implements MetadataService {
* @return a unique id for this type
*/
@Override
@GraphTransaction
public JSONObject createType(String typeDefinition) throws MetadataException {
ParamChecker.notEmpty(typeDefinition, "type definition cannot be empty");
TypesDef typesDef;
try {
typesDef = TypesSerialization.fromJson(typeDefinition);
if(typesDef.isEmpty()) {
if (typesDef.isEmpty()) {
throw new MetadataException("Invalid type definition");
}
} catch (Exception e) {
LOG.error("Unable to deserialize json={}", typeDefinition, e);
throw new IllegalArgumentException("Unable to deserialize json");
throw new IllegalArgumentException("Unable to deserialize json ", e);
}
try {
final Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
try {
typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
/* Create indexes first so that if index creation fails then we rollback
the typesystem and also do not persist the graph
*/
onTypesAddedToRepo(typesAdded);
} catch(Throwable t) {
typeSystem.removeTypes(ImmutableList.copyOf(typesAdded.keySet()));
throw new MetadataException(t);
typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
} catch (Throwable t) {
typeSystem.removeTypes(typesAdded.keySet());
throw new MetadataException("Unable to persist types ", t);
}
return new JSONObject() {{
......@@ -180,7 +188,7 @@ public class DefaultMetadataService implements MetadataService {
}};
} catch (JSONException e) {
LOG.error("Unable to create response for types={}", typeDefinition, e);
throw new MetadataException("Unable to create response");
throw new MetadataException("Unable to create response ", e);
}
}
......@@ -294,7 +302,7 @@ public class DefaultMetadataService implements MetadataService {
// verify if the type exists
if (!typeSystem.isRegistered(entityType)) {
throw new MetadataException("type is not defined for : " + entityType);
throw new TypeNotFoundException("type is not defined for : " + entityType);
}
}
......@@ -328,8 +336,12 @@ public class DefaultMetadataService implements MetadataService {
final String traitName = traitInstance.getTypeName();
// ensure trait type is already registered with the TS
Preconditions.checkArgument(typeSystem.isRegistered(traitName),
"trait=%s should be defined in type system before it can be added", traitName);
if ( !typeSystem.isRegistered(traitName) ) {
String msg = String.format("trait=%s should be defined in type system before it can be added", traitName);
LOG.error(msg);
throw new TypeNotFoundException(msg);
}
// ensure trait is not already defined
Preconditions.checkArgument(!getTraitNames(guid).contains(traitName),
"trait=%s is already defined for entity=%s", traitName, guid);
......@@ -340,7 +352,7 @@ public class DefaultMetadataService implements MetadataService {
}
private ITypedStruct deserializeTraitInstance(String traitInstanceDefinition)
throws MetadataException {
throws MetadataException {
try {
Struct traitInstance = InstanceSerialization.fromJsonStruct(
......@@ -351,6 +363,8 @@ public class DefaultMetadataService implements MetadataService {
TraitType traitType = typeSystem.getDataType(TraitType.class, entityTypeName);
return traitType.convert(
traitInstance, Multiplicity.REQUIRED);
} catch ( TypeNotFoundException e ) {
throw e;
} catch (Exception e) {
throw new MetadataException("Error deserializing trait instance", e);
}
......@@ -370,9 +384,12 @@ public class DefaultMetadataService implements MetadataService {
ParamChecker.notEmpty(traitNameToBeDeleted, "Trait name cannot be null");
// ensure trait type is already registered with the TS
Preconditions.checkArgument(typeSystem.isRegistered(traitNameToBeDeleted),
"trait=%s should be defined in type system before it can be deleted",
traitNameToBeDeleted);
if ( !typeSystem.isRegistered(traitNameToBeDeleted)) {
final String msg = String.format("trait=%s should be defined in type system before it can be deleted",
traitNameToBeDeleted);
LOG.error(msg);
throw new TypeNotFoundException(msg);
}
repository.deleteTrait(guid, traitNameToBeDeleted);
......@@ -380,23 +397,28 @@ public class DefaultMetadataService implements MetadataService {
}
private void onTypesAddedToRepo(Map<String, IDataType> typesAdded) throws MetadataException {
for (TypesChangeListener listener : typesChangeListeners) {
for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
listener.onAdd(entry.getKey(), entry.getValue());
Map<SearchIndexer, Throwable> caughtExceptions = new HashMap<>();
for(Provider<SearchIndexer> indexerProvider : typeChangeListeners) {
final SearchIndexer indexer = indexerProvider.get();
try {
for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
indexer.onAdd(entry.getKey(), entry.getValue());
}
indexer.commit();
} catch (IndexCreationException ice) {
LOG.error("Index creation for listener {} failed ", indexerProvider, ice);
indexer.rollback();
caughtExceptions.put(indexer, ice);
}
}
}
public void registerListener(TypesChangeListener listener) {
typesChangeListeners.add(listener);
}
public void unregisterListener(TypesChangeListener listener) {
typesChangeListeners.remove(listener);
if (caughtExceptions.size() > 0) {
throw new IndexCreationException("Index creation failed for types " + typesAdded.keySet() + ". Aborting");
}
}
private void onEntityAddedToRepo(ITypedReferenceableInstance typedInstance)
throws MetadataException {
throws MetadataException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntityAdded(typedInstance);
......@@ -424,4 +446,5 @@ public class DefaultMetadataService implements MetadataService {
public void unregisterListener(EntityChangeListener listener) {
entityChangeListeners.remove(listener);
}
}
}
\ No newline at end of file
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......@@ -19,11 +19,20 @@
package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
import com.thinkaurelius.titan.diskstorage.configuration.ModifiableConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -42,16 +51,21 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.io.FileUtils;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
@Test
@Guice(modules = RepositoryMetadataModule.class)
public class GraphRepoMapperScaleTest {
private static final String DATABASE_TYPE = "hive_database_type";
......@@ -59,9 +73,39 @@ public class GraphRepoMapperScaleTest {
private static final String TABLE_TYPE = "hive_table_type";
private static final String TABLE_NAME = "bar";
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private static final String INDEX_DIR = System.getProperty("java.io.tmpdir", "/tmp") + "/atlas-test" + new Random().nextLong();
private GraphProvider<TitanGraph> graphProvider = new GraphProvider<TitanGraph>() {
private TitanGraph graph = null;
//Ensure separate directory for graph provider to avoid issues with index merging
@Override
public TitanGraph get() {
try {
if (graph == null) {
synchronized (GraphRepoMapperScaleTest.class) {
if (graph == null) {
ReadConfiguration config = new CommonsConfiguration() {{
set("storage.backend", "inmemory");
set("index.search.directory", INDEX_DIR);
set("index.search.backend", "elasticsearch");
set("index.search.elasticsearch.local-mode", "true");
set("index.search.elasticsearch.client-only", "false");
}};
GraphDatabaseConfiguration graphconfig = new GraphDatabaseConfiguration(config);
graphconfig.getBackend().clearStorage();
graph = TitanFactory.open(config);
}
}
}
} catch (BackendException e) {
e.printStackTrace();
}
return graph;
}
};
private GraphBackedMetadataRepository repositoryService;
private GraphBackedSearchIndexer searchIndexer;
......@@ -69,7 +113,11 @@ public class GraphRepoMapperScaleTest {
private String dbGUID;
@BeforeClass
@GraphTransaction
public void setUp() throws Exception {
//Make sure we can cleanup the index directory
repositoryService = new GraphBackedMetadataRepository(graphProvider);
searchIndexer = new GraphBackedSearchIndexer(graphProvider);
typeSystem = TypeSystem.getInstance();
......@@ -77,6 +125,16 @@ public class GraphRepoMapperScaleTest {
createHiveTypes();
}
@AfterClass
public void tearDown() throws Exception {
graphProvider.get().shutdown();
try {
FileUtils.deleteDirectory(new File(INDEX_DIR));
} catch(IOException ioe) {
System.err.println("Failed to cleanup index directory");
}
}
@Test
public void testSubmitEntity() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
......@@ -220,6 +278,7 @@ public class GraphRepoMapperScaleTest {
for (Map.Entry<String, IDataType> entry : types.entrySet()) {
searchIndexer.onAdd(entry.getKey(), entry.getValue());
}
searchIndexer.commit();
}
private ITypedReferenceableInstance createHiveTableInstance(
......@@ -270,3 +329,4 @@ public class GraphRepoMapperScaleTest {
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
}
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -81,7 +81,6 @@ public class GraphBackedTypeStoreTest {
}
@Test (dependsOnMethods = "testStore")
@GraphTransaction
public void testRestore() throws Exception {
TypesDef types = typeStore.restore();
......
typesystem/src/main/java/org/apache/atlas/TypeNotFoundException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
/**
* A simple wrapper for 404.
*/
public class TypeNotFoundException extends MetadataException {
public TypeNotFoundException() {
}
public TypeNotFoundException(String message) {
super(message);
}
public TypeNotFoundException(String message, Throwable cause) {
super(message, cause);
}
public TypeNotFoundException(Throwable cause) {
super(cause);
}
public TypeNotFoundException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -22,21 +22,14 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
import org.apache.atlas.MetadataException;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef;
import javax.inject.Singleton;
import java.lang.reflect.Constructor;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@Singleton
......@@ -159,7 +152,7 @@ public class TypeSystem {
return cls.cast(dT);
}
throw new MetadataException(String.format("Unknown datatype: %s", name));
throw new TypeNotFoundException(String.format("Unknown datatype: %s", name));
}
public StructType defineStructType(String name,
......@@ -306,8 +299,13 @@ public class TypeSystem {
return false;
}
public void removeTypes(ImmutableList<String> typeNames) {
public void removeTypes(Collection<String> typeNames) {
for(String typeName : typeNames) {
IDataType dataType = types.get(typeName);
final DataTypes.TypeCategory typeCategory = dataType.getTypeCategory();
typeCategoriesToTypeNamesMap.get(typeCategory).remove(typeName);
types.remove(typeName);
}
}
class TransientTypeSystem extends TypeSystem {
......@@ -567,9 +565,10 @@ public class TypeSystem {
}
Map<String, IDataType> defineTypes() throws MetadataException {
step1();
step2();
try {
step1();
step2();
step3();
step4();
} catch (MetadataException me) {
......
webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
......@@ -81,11 +81,12 @@ public class AuditFilter implements Filter {
final String who = getUserFromRequest(httpRequest);
final String fromHost = httpRequest.getRemoteHost();
final String fromAddress = httpRequest.getRemoteAddr();
final String whatRequest = httpRequest.getMethod();
final String whatURL = Servlets.getRequestURL(httpRequest);
final String whatAddrs = httpRequest.getLocalAddr();
LOG.debug("Audit: {}/{} performed request {} ({}) at time {}",
who, fromAddress, whatURL, whatAddrs, whenISO9601);
LOG.debug("Audit: {}/{} performed request {} {} ({}) at time {}",
who, fromAddress, whatRequest, whatURL, whatAddrs, whenISO9601);
audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
}
......
webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
......@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import org.apache.atlas.MetadataException;
import org.apache.atlas.MetadataServiceClient;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.types.ValueConversionException;
......@@ -48,7 +49,6 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.net.URI;
import java.util.List;
......@@ -300,11 +300,11 @@ public class EntityResource {
response.put(MetadataServiceClient.GUID, guid);
return Response.created(locationURI).entity(response).build();
} catch (EntityNotFoundException e) {
} catch (EntityNotFoundException | TypeNotFoundException e) {
LOG.error("An entity with GUID={} does not exist", guid, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
} catch (MetadataException | IOException | IllegalArgumentException e) {
} catch (MetadataException | IllegalArgumentException e) {
LOG.error("Unable to add trait for entity={}", guid, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
......@@ -338,7 +338,7 @@ public class EntityResource {
response.put(TRAIT_NAME, traitName);
return Response.ok(response).build();
} catch (EntityNotFoundException e) {
} catch (EntityNotFoundException | TypeNotFoundException e) {
LOG.error("An entity with GUID={} does not exist", guid, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
......@@ -424,7 +424,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
.accept(Servlets.JSON_MEDIA_TYPE)
.type(Servlets.JSON_MEDIA_TYPE)
.method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
}
@Test(dependsOnMethods = "testGetTraitNames")
......@@ -494,7 +494,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
.accept(Servlets.JSON_MEDIA_TYPE)
.type(Servlets.JSON_MEDIA_TYPE)
.method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
}
@Test(dependsOnMethods = "testAddTrait")
......@@ -532,7 +532,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
.accept(Servlets.JSON_MEDIA_TYPE)
.type(Servlets.JSON_MEDIA_TYPE)
.method(HttpMethod.DELETE, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
......
webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
......@@ -219,6 +219,9 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef("columnNames", DataTypes.arrayTypeName(DataTypes.STRING_TYPE)),
TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
TypesUtil.createOptionalAttrDef("parameters", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE)),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition("database",
"database", Multiplicity.REQUIRED, false, "database"));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment