Merge pull request #139 from sumashivaprasad/api_merged

addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/ABridge.java

@@ -85,7 +85,7 @@ public abstract class ABridge implements IBridge {
         ITypedReferenceableInstance refBean = null;
         try {
             refBean = type.convert(this.convertToReferencable(bean), Multiplicity.REQUIRED);
-            String id = repo.createEntity(refBean, type.getName());
+            String id = repo.createEntity(refBean);
             return id;
         } catch (IllegalArgumentException | IllegalAccessException e) {
             // TODO Auto-generated catch block

repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java

@@ -44,7 +44,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
             return response;
         } catch (Throwable t){
             titanGraph.rollback();
-            LOG.debug("graph rollback");
+            LOG.error("graph rollback due to exception ", t);
             throw t;
         }
     }

repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java

@@ -41,6 +41,7 @@ import org.apache.atlas.services.MetadataService;
  * Guice module for Repository module.
  */
 public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
+
     @Override
     protected void configure() {
         // special wiring for Titan Graph
@@ -66,7 +67,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
         // bind the DiscoveryService interface to an implementation
         bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
 
-        bind(SearchIndexer.class).to(GraphBackedSearchIndexer.class).asEagerSingleton();
+        bind(SearchIndexer.class).to(GraphBackedSearchIndexer.class);
 
         bind(LineageService.class).to(HiveLineageService.class).asEagerSingleton();
 
@@ -74,4 +75,5 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
         requestInjection(interceptor);
         bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
     }
+
 }

repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java

@@ -19,6 +19,7 @@
 package org.apache.atlas.discovery;
 
 import org.apache.atlas.listener.TypesChangeListener;
+import org.apache.atlas.repository.IndexException;
 
 
 /**
@@ -26,4 +27,11 @@ import org.apache.atlas.listener.TypesChangeListener;
  */
 public interface SearchIndexer extends TypesChangeListener {
 
+    /* Commit the indexes */
+
+   void commit() throws IndexException;
+
+    /* RollBack the index */
+   void rollback() throws IndexException;
+
 }

repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java

@@ -19,6 +19,7 @@
 package org.apache.atlas.discovery.graph;
 
 import com.thinkaurelius.titan.core.TitanVertex;
+import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.MetadataException;
 import org.apache.atlas.query.Expressions;
 import org.apache.atlas.query.GraphPersistenceStrategies;

repository/src/main/java/org/apache/atlas/repository/IndexCreationException.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository;
+
+public class IndexCreationException extends IndexException {
+
+    public IndexCreationException() {
+    }
+
+    public IndexCreationException(String message) {
+        super(message);
+    }
+
+    public IndexCreationException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public IndexCreationException(Throwable cause) {
+        super(cause);
+    }
+
+    public IndexCreationException(String message, Throwable cause, boolean enableSuppression,
+                          boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

repository/src/main/java/org/apache/atlas/repository/IndexException.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository;
+
+
+import org.apache.atlas.MetadataException;
+
+public class IndexException extends MetadataException {
+
+    public IndexException() {
+    }
+
+    public IndexException(String message) {
+        super(message);
+    }
+
+    public IndexException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public IndexException(Throwable cause) {
+        super(cause);
+    }
+
+    public IndexException(String message, Throwable cause, boolean enableSuppression,
+                          boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+}

repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java

@@ -257,6 +257,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             ((TitanVertex) instanceVertex)
                     .addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
 
+        } catch (RepositoryException e) {
+          throw e;
         } catch (Exception e) {
             throw new RepositoryException(e);
         }
@@ -279,7 +281,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
 
             List<String> traitNames = getTraitNames(instanceVertex);
             if (!traitNames.contains(traitNameToBeDeleted)) {
-                throw new RepositoryException("Could not find trait=" + traitNameToBeDeleted
+                throw new EntityNotFoundException("Could not find trait=" + traitNameToBeDeleted
                         + " in the repository for entity: " + guid);
             }
 
@@ -302,6 +304,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                     updateTraits(instanceVertex, traitNames);
                 }
             }
+        } catch (RepositoryException e) {
+            throw e;
         } catch (Exception e) {
             throw new RepositoryException(e);
         }
@@ -350,6 +354,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             instanceToGraphMapper.mapAttributesToVertex(getIdFromVertex(typeName, instanceVertex),
                     instance, instanceVertex, new HashMap<Id, Vertex>(),
                     attributeInfo, attributeInfo.dataType());
+        } catch (RepositoryException e) {
+            throw e;
         } catch (Exception e) {
             throw new RepositoryException(e);
         }
@@ -426,7 +432,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             }
         }
 
-        public void createVerticesForClassTypes(
+        private void createVerticesForClassTypes(
                 List<ITypedReferenceableInstance> newInstances) throws MetadataException {
             for (ITypedReferenceableInstance typedInstance : newInstances) {
                 final Id id = typedInstance.getId();
@@ -955,7 +961,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             return typedInstance;
         }
 
-        public void mapVertexToInstanceTraits(Vertex instanceVertex,
+        private void mapVertexToInstanceTraits(Vertex instanceVertex,
                                               ITypedReferenceableInstance typedInstance,
                                               List<String> traits) throws MetadataException {
             for (String traitName : traits) {
@@ -977,7 +983,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             }
         }
 
-        public void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
+
+        private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
                                          AttributeInfo attributeInfo) throws MetadataException {
             LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
             final IDataType dataType = attributeInfo.dataType();
@@ -1027,7 +1034,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             }
         }
 
-        public Object mapClassReferenceToVertex(Vertex instanceVertex,
+        private Object mapClassReferenceToVertex(Vertex instanceVertex,
                                                 AttributeInfo attributeInfo,
                                                 String relationshipLabel,
                                                 IDataType dataType) throws MetadataException {
@@ -1057,7 +1064,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         }
 
         @SuppressWarnings("unchecked")
-        public void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+        private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
                                              AttributeInfo attributeInfo,
                                              String propertyName) throws MetadataException {
             LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
@@ -1077,7 +1084,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             typedInstance.set(attributeInfo.name, values);
         }
 
-        public Object mapVertexToCollectionEntry(Vertex instanceVertex,
+        private Object mapVertexToCollectionEntry(Vertex instanceVertex,
                                                  AttributeInfo attributeInfo,
                                                  IDataType elementType, Object value, String propertyName)
             throws MetadataException {
@@ -1161,7 +1168,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             return null;
         }
 
-        public Object mapClassReferenceToVertex(Vertex instanceVertex,
+        private Object mapClassReferenceToVertex(Vertex instanceVertex,
                                                 AttributeInfo attributeInfo,
                                                 String relationshipLabel,
                                                 IDataType dataType,
@@ -1228,7 +1235,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                     traitName, traitType, traitInstance);
         }
 
-        public void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName,
+        private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName,
                                              String traitName, TraitType traitType,
                                              ITypedStruct traitInstance) throws MetadataException {
             String relationshipLabel = getEdgeLabel(typedInstanceTypeName, traitName);

repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java

@@ -18,7 +18,6 @@
 
 package org.apache.atlas.repository.graph;
 
-import com.google.inject.Singleton;
 import com.thinkaurelius.titan.core.Cardinality;
 import com.thinkaurelius.titan.core.PropertyKey;
 import com.thinkaurelius.titan.core.TitanGraph;
@@ -30,6 +29,8 @@ import com.tinkerpop.blueprints.Vertex;
 import org.apache.atlas.MetadataException;
 import org.apache.atlas.discovery.SearchIndexer;
 import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.IndexCreationException;
+import org.apache.atlas.repository.IndexException;
 import org.apache.atlas.repository.RepositoryException;
 import org.apache.atlas.typesystem.types.AttributeInfo;
 import org.apache.atlas.typesystem.types.ClassType;
@@ -49,19 +50,24 @@ import java.util.Map;
 /**
  * Adds index for properties of a given type when its added before any instances are added.
  */
-@Singleton
 public class GraphBackedSearchIndexer implements SearchIndexer {
 
     private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
 
     private final TitanGraph titanGraph;
 
+    private TitanManagement management;
+
     @Inject
     public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider)
         throws RepositoryException {
 
         this.titanGraph = graphProvider.get();
 
+        /* Create the transaction for indexing.
+         * Commit/rollback is expected to be called from the caller.
+         */
+        management = titanGraph.getManagementSystem();
         initialize();
     }
 
@@ -69,18 +75,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
      * Initializes the indices for the graph - create indices for Global Vertex Keys
      */
     private void initialize() {
-        TitanManagement management = titanGraph.getManagementSystem();
         if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
             LOG.info("Global indexes already exist for graph");
             return;
         }
 
+        /* This is called only once, which is the first time Atlas types are made indexable .*/
         LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
         management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
                 .buildMixedIndex(Constants.BACKING_INDEX);
         management.buildIndex(Constants.EDGE_INDEX, Edge.class)
                 .buildMixedIndex(Constants.BACKING_INDEX);
-        management.commit();
 
         // create a composite index for guid as its unique
         createCompositeIndex(Constants.GUID_INDEX,
@@ -105,18 +110,20 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
         //Indexes for graph backed type system store
         createTypeStoreIndexes();
 
+        management.commit();
+        //Make sure we acquire another transaction after commit for subsequent indexing
+        management = titanGraph.getManagementSystem();
+
         LOG.info("Index creation for global keys complete.");
     }
 
     private void createFullTextIndex() {
-        TitanManagement management = titanGraph.getManagementSystem();
         PropertyKey fullText =
                 management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();
 
         management.buildIndex(Constants.FULLTEXT_INDEX, Vertex.class)
                 .addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
                 .buildMixedIndex(Constants.BACKING_INDEX);
-        management.commit();
         LOG.info("Created mixed index for {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
     }
 
@@ -147,9 +154,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
             LOG.info("Index creation for type {} complete", typeName);
 
         } catch (Throwable throwable) {
-            // gets handle to currently open transaction
-            titanGraph.getManagementSystem().rollback();
             LOG.error("Error creating index for type {}", dataType, throwable);
+            throw new IndexCreationException("Error while creating index for type " + dataType, throwable);
         }
     }
 
@@ -282,7 +288,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
     private PropertyKey createCompositeIndex(String indexName,
                                              String propertyName, Class propertyClass,
                                              boolean isUnique, Cardinality cardinality) {
-        TitanManagement management = titanGraph.getManagementSystem();
         PropertyKey propertyKey = management.getPropertyKey(propertyName);
         if (propertyKey == null) {
             propertyKey = management
@@ -300,8 +305,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
             }
 
             indexBuilder.buildCompositeIndex();
-            management.commit();
-
             LOG.info("Created index for property {} in composite index {}", propertyName, indexName);
         }
 
@@ -309,7 +312,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
     }
 
     private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) {
-        TitanManagement management = titanGraph.getManagementSystem();
+
         PropertyKey propertyKey = management.getPropertyKey(propertyName);
         if (propertyKey == null) {
             // ignored cardinality as Can only index single-valued property keys on vertices
@@ -318,15 +321,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
                     .dataType(propertyClass)
                     .make();
 
-            if (propertyClass == Boolean.class) {
+            if (!checkIfMixedIndexApplicable(propertyClass)) {
+                LOG.debug("Creating composite index for property {} of type {} ", propertyName, propertyClass.getName());
                 //Use standard index as backing index only supports string, int and geo types
                 management.buildIndex(propertyName, Vertex.class).addKey(propertyKey).buildCompositeIndex();
-                management.commit();
+                LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
             } else {
                 //Use backing index
+                LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
                 TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
                 management.addIndexKey(vertexIndex, propertyKey);
-                management.commit();
+                LOG.debug("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
             }
             LOG.info("Created mixed vertex index for property {}", propertyName);
         }
@@ -334,15 +339,41 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
         return propertyKey;
     }
 
+    private boolean checkIfMixedIndexApplicable(Class propertyClass) {
+        //TODO - Check why date types are failing in ES/Solr
+        if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class || propertyClass == Date.class) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void commit() throws IndexException {
+        try {
+            management.commit();
+        } catch (Exception e) {
+            LOG.error("Index commit failed" , e);
+            throw new IndexException("Index commit failed " , e);
+        }
+    }
+
+    @Override
+    public void rollback() throws IndexException {
+        try {
+            management.rollback();
+        } catch (Exception e) {
+            LOG.error("Index rollback failed " , e);
+            throw new IndexException("Index rollback failed " , e);
+        }
+    }
+
     /* Commenting this out since we do not need an index for edge label here
     private void createEdgeMixedIndex(String propertyName) {
-        TitanManagement management = titanGraph.getManagementSystem();
         EdgeLabel edgeLabel = management.getEdgeLabel(propertyName);
         if (edgeLabel == null) {
             edgeLabel = management.makeEdgeLabel(propertyName).make();
             management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
-            management.commit();
             LOG.info("Created index for edge label {}", propertyName);
         }
-    } */
+    }*/
 }

repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java

@@ -74,6 +74,7 @@ public class GraphBackedTypeStore implements ITypeStore {
     }
 
     @Override
+    @GraphTransaction
     public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws MetadataException {
         ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
         for (String typeName : typeNames) {

repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java

@@ -20,14 +20,17 @@ package org.apache.atlas.services;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.inject.Provider;
 import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.MetadataException;
 import org.apache.atlas.MetadataServiceClient;
 import org.apache.atlas.ParamChecker;
+import org.apache.atlas.TypeNotFoundException;
 import org.apache.atlas.classification.InterfaceAudience;
 import org.apache.atlas.discovery.SearchIndexer;
 import org.apache.atlas.listener.EntityChangeListener;
 import org.apache.atlas.listener.TypesChangeListener;
+import org.apache.atlas.repository.IndexCreationException;
 import org.apache.atlas.repository.MetadataRepository;
 import org.apache.atlas.repository.typestore.ITypeStore;
 import org.apache.atlas.typesystem.ITypedReferenceableInstance;
@@ -56,6 +59,9 @@ import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -71,22 +77,22 @@ public class DefaultMetadataService implements MetadataService {
     private static final Logger LOG =
             LoggerFactory.getLogger(DefaultMetadataService.class);
 
-    private final Set<TypesChangeListener> typesChangeListeners = new LinkedHashSet<>();
     private final Set<EntityChangeListener> entityChangeListeners
             = new LinkedHashSet<>();
 
     private final TypeSystem typeSystem;
     private final MetadataRepository repository;
     private final ITypeStore typeStore;
+    private final Set<Provider<SearchIndexer>> typeChangeListeners;
 
     @Inject
-    DefaultMetadataService(MetadataRepository repository,
-                           SearchIndexer searchIndexer, ITypeStore typeStore) throws MetadataException {
+    DefaultMetadataService(final MetadataRepository repository,
+                           final Provider<SearchIndexer> searchIndexProvider, final ITypeStore typeStore) throws MetadataException {
         this.typeStore = typeStore;
         this.typeSystem = TypeSystem.getInstance();
         this.repository = repository;
 
-        registerListener(searchIndexer);
+        this.typeChangeListeners = new LinkedHashSet<Provider<SearchIndexer>>() {{ add(searchIndexProvider); }};
         restoreTypeSystem();
     }
 
@@ -149,30 +155,32 @@ public class DefaultMetadataService implements MetadataService {
      * @return a unique id for this type
      */
     @Override
-    @GraphTransaction
     public JSONObject createType(String typeDefinition) throws MetadataException {
         ParamChecker.notEmpty(typeDefinition, "type definition cannot be empty");
 
         TypesDef typesDef;
         try {
             typesDef = TypesSerialization.fromJson(typeDefinition);
-            if(typesDef.isEmpty()) {
+            if (typesDef.isEmpty()) {
                 throw new MetadataException("Invalid type definition");
             }
         } catch (Exception e) {
             LOG.error("Unable to deserialize json={}", typeDefinition, e);
-            throw new IllegalArgumentException("Unable to deserialize json");
+            throw new IllegalArgumentException("Unable to deserialize json ", e);
         }
 
         try {
             final Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
 
             try {
-                typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
+                /* Create indexes first so that if index creation fails then we rollback
+                   the typesystem and also do not persist the graph
+                 */
                 onTypesAddedToRepo(typesAdded);
-            } catch(Throwable t) {
-                typeSystem.removeTypes(ImmutableList.copyOf(typesAdded.keySet()));
-                throw new MetadataException(t);
+                typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
+            } catch (Throwable t) {
+                typeSystem.removeTypes(typesAdded.keySet());
+                throw new MetadataException("Unable to persist types ", t);
             }
 
             return new JSONObject() {{
@@ -180,7 +188,7 @@ public class DefaultMetadataService implements MetadataService {
             }};
         } catch (JSONException e) {
             LOG.error("Unable to create response for types={}", typeDefinition, e);
-            throw new MetadataException("Unable to create response");
+            throw new MetadataException("Unable to create response ", e);
         }
     }
 
@@ -294,7 +302,7 @@ public class DefaultMetadataService implements MetadataService {
 
         // verify if the type exists
         if (!typeSystem.isRegistered(entityType)) {
-            throw new MetadataException("type is not defined for : " + entityType);
+            throw new TypeNotFoundException("type is not defined for : " + entityType);
         }
     }
 
@@ -328,8 +336,12 @@ public class DefaultMetadataService implements MetadataService {
         final String traitName = traitInstance.getTypeName();
 
         // ensure trait type is already registered with the TS
-        Preconditions.checkArgument(typeSystem.isRegistered(traitName),
-                "trait=%s should be defined in type system before it can be added", traitName);
+        if ( !typeSystem.isRegistered(traitName) ) {
+            String msg = String.format("trait=%s should be defined in type system before it can be added", traitName);
+            LOG.error(msg);
+            throw new TypeNotFoundException(msg);
+        }
+
         // ensure trait is not already defined
         Preconditions.checkArgument(!getTraitNames(guid).contains(traitName),
                 "trait=%s is already defined for entity=%s", traitName, guid);
@@ -340,7 +352,7 @@ public class DefaultMetadataService implements MetadataService {
     }
 
     private ITypedStruct deserializeTraitInstance(String traitInstanceDefinition)
-        throws MetadataException {
+            throws MetadataException {
 
         try {
             Struct traitInstance = InstanceSerialization.fromJsonStruct(
@@ -351,6 +363,8 @@ public class DefaultMetadataService implements MetadataService {
             TraitType traitType = typeSystem.getDataType(TraitType.class, entityTypeName);
             return traitType.convert(
                     traitInstance, Multiplicity.REQUIRED);
+        } catch ( TypeNotFoundException e ) {
+            throw e;
         } catch (Exception e) {
             throw new MetadataException("Error deserializing trait instance", e);
         }
@@ -370,9 +384,12 @@ public class DefaultMetadataService implements MetadataService {
         ParamChecker.notEmpty(traitNameToBeDeleted, "Trait name cannot be null");
 
         // ensure trait type is already registered with the TS
-        Preconditions.checkArgument(typeSystem.isRegistered(traitNameToBeDeleted),
-                "trait=%s should be defined in type system before it can be deleted",
-                traitNameToBeDeleted);
+        if ( !typeSystem.isRegistered(traitNameToBeDeleted)) {
+            final String msg = String.format("trait=%s should be defined in type system before it can be deleted",
+                    traitNameToBeDeleted);
+            LOG.error(msg);
+            throw new TypeNotFoundException(msg);
+        }
 
         repository.deleteTrait(guid, traitNameToBeDeleted);
 
@@ -380,23 +397,28 @@ public class DefaultMetadataService implements MetadataService {
     }
 
     private void onTypesAddedToRepo(Map<String, IDataType> typesAdded) throws MetadataException {
-        for (TypesChangeListener listener : typesChangeListeners) {
-            for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
-                listener.onAdd(entry.getKey(), entry.getValue());
+        Map<SearchIndexer, Throwable> caughtExceptions = new HashMap<>();
+        for(Provider<SearchIndexer> indexerProvider : typeChangeListeners) {
+            final SearchIndexer indexer = indexerProvider.get();
+            try {
+                for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
+                    indexer.onAdd(entry.getKey(), entry.getValue());
+                }
+                indexer.commit();
+            } catch (IndexCreationException ice) {
+                LOG.error("Index creation for listener {} failed ", indexerProvider, ice);
+                indexer.rollback();
+                caughtExceptions.put(indexer, ice);
             }
         }
-    }
-
-    public void registerListener(TypesChangeListener listener) {
-        typesChangeListeners.add(listener);
-    }
 
-    public void unregisterListener(TypesChangeListener listener) {
-        typesChangeListeners.remove(listener);
+        if (caughtExceptions.size() > 0) {
+           throw new IndexCreationException("Index creation failed for types " + typesAdded.keySet() + ". Aborting");
+        }
     }
 
     private void onEntityAddedToRepo(ITypedReferenceableInstance typedInstance)
-        throws MetadataException {
+            throws MetadataException {
 
         for (EntityChangeListener listener : entityChangeListeners) {
             listener.onEntityAdded(typedInstance);
@@ -424,4 +446,5 @@ public class DefaultMetadataService implements MetadataService {
     public void unregisterListener(EntityChangeListener listener) {
         entityChangeListeners.remove(listener);
     }
-}
+
+}
\ No newline at end of file

repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java

@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,11 +19,20 @@
 package org.apache.atlas.repository.graph;
 
 import com.google.common.collect.ImmutableList;
+import com.thinkaurelius.titan.core.TitanFactory;
 import com.thinkaurelius.titan.core.TitanGraph;
 import com.thinkaurelius.titan.core.TitanIndexQuery;
+import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
+import com.thinkaurelius.titan.diskstorage.BackendException;
+import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
+import com.thinkaurelius.titan.diskstorage.configuration.ModifiableConfiguration;
+import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration;
+import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration;
+import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
 import com.tinkerpop.blueprints.Compare;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.typesystem.ITypedReferenceableInstance;
@@ -42,16 +51,21 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.commons.io.FileUtils;
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
 import javax.inject.Inject;
+import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.Random;
 
 @Test
-@Guice(modules = RepositoryMetadataModule.class)
 public class GraphRepoMapperScaleTest {
 
     private static final String DATABASE_TYPE = "hive_database_type";
@@ -59,9 +73,39 @@ public class GraphRepoMapperScaleTest {
     private static final String TABLE_TYPE = "hive_table_type";
     private static final String TABLE_NAME = "bar";
 
-    @Inject
-    private GraphProvider<TitanGraph> graphProvider;
-    @Inject
+    private static final String INDEX_DIR = System.getProperty("java.io.tmpdir", "/tmp") + "/atlas-test" + new Random().nextLong();
+
+    private GraphProvider<TitanGraph> graphProvider = new GraphProvider<TitanGraph>() {
+
+        private TitanGraph graph = null;
+
+        //Ensure separate directory for graph provider to avoid issues with index merging
+        @Override
+        public TitanGraph get() {
+            try {
+                if (graph == null) {
+                    synchronized (GraphRepoMapperScaleTest.class) {
+                        if (graph == null) {
+                            ReadConfiguration config = new CommonsConfiguration() {{
+                                set("storage.backend", "inmemory");
+                                set("index.search.directory", INDEX_DIR);
+                                set("index.search.backend", "elasticsearch");
+                                set("index.search.elasticsearch.local-mode", "true");
+                                set("index.search.elasticsearch.client-only", "false");
+                            }};
+                            GraphDatabaseConfiguration graphconfig = new GraphDatabaseConfiguration(config);
+                            graphconfig.getBackend().clearStorage();
+                            graph = TitanFactory.open(config);
+                        }
+                    }
+                }
+            } catch (BackendException e) {
+                e.printStackTrace();
+            }
+            return graph;
+        }
+    };
+
     private GraphBackedMetadataRepository repositoryService;
 
     private GraphBackedSearchIndexer searchIndexer;
@@ -69,7 +113,11 @@ public class GraphRepoMapperScaleTest {
     private String dbGUID;
 
     @BeforeClass
+    @GraphTransaction
     public void setUp() throws Exception {
+        //Make sure we can cleanup the index directory
+        repositoryService = new GraphBackedMetadataRepository(graphProvider);
+
         searchIndexer = new GraphBackedSearchIndexer(graphProvider);
 
         typeSystem = TypeSystem.getInstance();
@@ -77,6 +125,16 @@ public class GraphRepoMapperScaleTest {
         createHiveTypes();
     }
 
+    @AfterClass
+    public void tearDown() throws Exception {
+        graphProvider.get().shutdown();
+        try {
+            FileUtils.deleteDirectory(new File(INDEX_DIR));
+        } catch(IOException ioe) {
+            System.err.println("Failed to cleanup index directory");
+        }
+    }
+
     @Test
     public void testSubmitEntity() throws Exception {
         Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
@@ -220,6 +278,7 @@ public class GraphRepoMapperScaleTest {
         for (Map.Entry<String, IDataType> entry : types.entrySet()) {
             searchIndexer.onAdd(entry.getKey(), entry.getValue());
         }
+        searchIndexer.commit();
     }
 
     private ITypedReferenceableInstance createHiveTableInstance(
@@ -270,3 +329,4 @@ public class GraphRepoMapperScaleTest {
         return tableType.convert(tableInstance, Multiplicity.REQUIRED);
     }
 }
+

repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java

@@ -81,7 +81,6 @@ public class GraphBackedTypeStoreTest {
     }
 
     @Test (dependsOnMethods = "testStore")
-    @GraphTransaction
     public void testRestore() throws Exception {
         TypesDef types = typeStore.restore();
 

typesystem/src/main/java/org/apache/atlas/TypeNotFoundException.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+/**
+ * A simple wrapper for 404.
+ */
+public class TypeNotFoundException extends MetadataException {
+    public TypeNotFoundException() {
+    }
+
+    public TypeNotFoundException(String message) {
+        super(message);
+    }
+
+    public TypeNotFoundException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public TypeNotFoundException(Throwable cause) {
+        super(cause);
+    }
+
+    public TypeNotFoundException(String message, Throwable cause, boolean enableSuppression,
+                                 boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java

@@ -22,21 +22,14 @@ import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Multimap;
 import org.apache.atlas.MetadataException;
+import org.apache.atlas.TypeNotFoundException;
 import org.apache.atlas.classification.InterfaceAudience;
 import org.apache.atlas.typesystem.TypesDef;
 
 import javax.inject.Singleton;
 import java.lang.reflect.Constructor;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
 @Singleton
@@ -159,7 +152,7 @@ public class TypeSystem {
             return cls.cast(dT);
         }
 
-        throw new MetadataException(String.format("Unknown datatype: %s", name));
+        throw new TypeNotFoundException(String.format("Unknown datatype: %s", name));
     }
 
     public StructType defineStructType(String name,
@@ -306,8 +299,13 @@ public class TypeSystem {
         return false;
     }
 
-    public void removeTypes(ImmutableList<String> typeNames) {
-
+    public void removeTypes(Collection<String> typeNames) {
+        for(String typeName : typeNames) {
+            IDataType dataType = types.get(typeName);
+            final DataTypes.TypeCategory typeCategory = dataType.getTypeCategory();
+            typeCategoriesToTypeNamesMap.get(typeCategory).remove(typeName);
+            types.remove(typeName);
+        }
     }
 
     class TransientTypeSystem extends TypeSystem {
@@ -567,9 +565,10 @@ public class TypeSystem {
         }
 
         Map<String, IDataType> defineTypes() throws MetadataException {
-            step1();
-            step2();
             try {
+                step1();
+                step2();
+
                 step3();
                 step4();
             } catch (MetadataException me) {

webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java

@@ -81,11 +81,12 @@ public class AuditFilter implements Filter {
         final String who = getUserFromRequest(httpRequest);
         final String fromHost = httpRequest.getRemoteHost();
         final String fromAddress = httpRequest.getRemoteAddr();
+        final String whatRequest = httpRequest.getMethod();
         final String whatURL = Servlets.getRequestURL(httpRequest);
         final String whatAddrs = httpRequest.getLocalAddr();
 
-        LOG.debug("Audit: {}/{} performed request {} ({}) at time {}",
-                who, fromAddress, whatURL, whatAddrs, whenISO9601);
+        LOG.debug("Audit: {}/{} performed request {} {} ({}) at time {}",
+                who, fromAddress, whatRequest, whatURL, whatAddrs, whenISO9601);
         audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
     }
 

webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java

@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
 import org.apache.atlas.MetadataException;
 import org.apache.atlas.MetadataServiceClient;
 import org.apache.atlas.ParamChecker;
+import org.apache.atlas.TypeNotFoundException;
 import org.apache.atlas.repository.EntityNotFoundException;
 import org.apache.atlas.services.MetadataService;
 import org.apache.atlas.typesystem.types.ValueConversionException;
@@ -48,7 +49,6 @@ import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
 import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
 import java.net.URI;
 import java.util.List;
 
@@ -300,11 +300,11 @@ public class EntityResource {
             response.put(MetadataServiceClient.GUID, guid);
 
             return Response.created(locationURI).entity(response).build();
-        } catch (EntityNotFoundException e) {
+        } catch (EntityNotFoundException | TypeNotFoundException e) {
             LOG.error("An entity with GUID={} does not exist", guid, e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
-        } catch (MetadataException | IOException | IllegalArgumentException e) {
+        } catch (MetadataException | IllegalArgumentException e) {
             LOG.error("Unable to add trait for entity={}", guid, e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
@@ -338,7 +338,7 @@ public class EntityResource {
             response.put(TRAIT_NAME, traitName);
 
             return Response.ok(response).build();
-        } catch (EntityNotFoundException e) {
+        } catch (EntityNotFoundException | TypeNotFoundException e) {
             LOG.error("An entity with GUID={} does not exist", guid, e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));

webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java

@@ -424,7 +424,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
                 .accept(Servlets.JSON_MEDIA_TYPE)
                 .type(Servlets.JSON_MEDIA_TYPE)
                 .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
-        Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
     }
 
     @Test(dependsOnMethods = "testGetTraitNames")
@@ -494,7 +494,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
                 .accept(Servlets.JSON_MEDIA_TYPE)
                 .type(Servlets.JSON_MEDIA_TYPE)
                 .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
-        Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
     }
 
     @Test(dependsOnMethods = "testAddTrait")
@@ -532,7 +532,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
                 .accept(Servlets.JSON_MEDIA_TYPE)
                 .type(Servlets.JSON_MEDIA_TYPE)
                 .method(HttpMethod.DELETE, ClientResponse.class);
-        Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
 
         String responseAsString = clientResponse.getEntity(String.class);
         Assert.assertNotNull(responseAsString);

webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java

@@ -219,6 +219,9 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
                 ImmutableList.<String>of(),
                 TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
                 TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
+                TypesUtil.createOptionalAttrDef("columnNames", DataTypes.arrayTypeName(DataTypes.STRING_TYPE)),
+                TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
+                TypesUtil.createOptionalAttrDef("parameters", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE)),
                 TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
                 new AttributeDefinition("database",
                         "database", Multiplicity.REQUIRED, false, "database"));