ATLAS-1230: updated AtlasTypeRegistry to support batch, atomic type updates

intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java

@@ -103,16 +103,6 @@ public class AtlasArrayType extends AtlasType {
     @Override
     public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
         elementType = typeRegistry.getType(elementTypeName);
-
-        if (elementType == null) {
-            String msg = elementTypeName + ": unknown element-type for array";
-
-            LOG.error(msg);
-
-            throw new AtlasBaseException(msg);
-        }
-
-        elementType.resolveReferences(typeRegistry);
     }
 
     @Override

intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java

@@ -70,7 +70,7 @@ public class AtlasClassificationType extends AtlasStructType {
         for (String superTypeName : classificationDef.getSuperTypes()) {
             AtlasType superType = typeRegistry.getType(superTypeName);
 
-            if (superType != null && superType instanceof AtlasClassificationType) {
+            if (superType instanceof AtlasClassificationType) {
                 AtlasClassificationType superClassificationType = (AtlasClassificationType)superType;
 
                 superClassificationType.resolveReferences(typeRegistry);
@@ -82,13 +82,8 @@ public class AtlasClassificationType extends AtlasStructType {
                     allS.addAll(superClassificationType.getAllSuperTypes());
                 }
             } else {
-                String msg = superTypeName + ((superType == null) ? ": unknown" : ": incompatible");
-
-                msg += (" supertype in classification " + classificationDef.getName());
-
-                LOG.error(msg);
-
-                throw new AtlasBaseException(msg);
+                throw new AtlasBaseException(superTypeName + ": incompatible supertype in classification "
+                                             + classificationDef.getName());
             }
         }
 

intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java

@@ -68,7 +68,7 @@ public class AtlasEntityType extends AtlasStructType {
         for (String superTypeName : entityDef.getSuperTypes()) {
             AtlasType superType = typeRegistry.getType(superTypeName);
 
-            if (superType != null && superType instanceof AtlasEntityType) {
+            if (superType instanceof AtlasEntityType) {
                 AtlasEntityType superEntityType = (AtlasEntityType)superType;
 
                 superEntityType.resolveReferences(typeRegistry);
@@ -80,13 +80,8 @@ public class AtlasEntityType extends AtlasStructType {
                     allS.addAll(superEntityType.getAllSuperTypes());
                 }
             } else {
-                String msg = superTypeName + ((superType == null) ? ": unknown" : ": incompatible");
-
-                msg += (" supertype in entity " + entityDef.getName());
-
-                LOG.error(msg);
-
-                throw new AtlasBaseException(msg);
+                throw new AtlasBaseException(superTypeName + ": incompatible supertype in entity "
+                                             + entityDef.getName());
             }
         }
 

intg/src/main/java/org/apache/atlas/type/AtlasMapType.java

@@ -88,25 +88,6 @@ public class AtlasMapType extends AtlasType {
     public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
         this.keyType   = typeRegistry.getType(keyTypeName);
         this.valueType = typeRegistry.getType(valueTypeName);
-
-        if (keyType == null) {
-            String msg = keyTypeName + ": unknown key-type for map";
-
-            LOG.error(msg);
-
-            throw new AtlasBaseException(msg);
-        }
-
-        if (valueType == null) {
-            String msg = valueTypeName + ": unknown value-type for map";
-
-            LOG.error(msg);
-
-            throw new AtlasBaseException(msg);
-        }
-
-        keyType.resolveReferences(typeRegistry);
-        valueType.resolveReferences(typeRegistry);
     }
 
     @Override

intg/src/main/java/org/apache/atlas/type/AtlasStructType.java

@@ -103,11 +103,6 @@ public class AtlasStructType extends AtlasType {
         for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
             AtlasType attrType = typeRegistry.getType(attributeDef.getTypeName());
 
-            if (attrType == null) {
-                throw new AtlasBaseException(attributeDef.getTypeName() + ": unknown type for attribute "
-                                             + structDef.getName() + "." + attributeDef.getName());
-            }
-
             resolveConstraints(attributeDef, attrType);
 
             Cardinality cardinality = attributeDef.getCardinality();
@@ -407,8 +402,8 @@ public class AtlasStructType extends AtlasType {
 
         if (StringUtils.isBlank(refAttribName)) {
             throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": "
-                        + CONSTRAINT_TYPE_MAPPED_FROM_REF + " invalid constraint. missing parameter "
-                        + CONSTRAINT_PARAM_REF_ATTRIBUTE);
+                        + " invalid constraint. missing parameter " + CONSTRAINT_PARAM_REF_ATTRIBUTE
+                        + " in " + CONSTRAINT_TYPE_MAPPED_FROM_REF + ". params=" + constraintDef.getParams());
         }
 
         AtlasStructType   structType = (AtlasStructType)attribType;
@@ -421,8 +416,8 @@ public class AtlasStructType extends AtlasType {
         }
 
         if (!StringUtils.equals(getTypeName(), refAttrib.getTypeName())) {
-            throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": invalid constraint. Datatype of"
-                    + CONSTRAINT_PARAM_REF_ATTRIBUTE + " " + structType.getTypeName() + "." + refAttribName
+            throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": invalid constraint. Datatype"
+                    + " of " + CONSTRAINT_PARAM_REF_ATTRIBUTE + " " + structType.getTypeName() + "." + refAttribName
                     + " should be " + getTypeName() + ", but found " + refAttrib.getTypeName());
         }
 

intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java

@@ -41,6 +41,7 @@ import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasStructType;
 import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -158,7 +159,11 @@ public final class  ModelTestUtil {
         }
 
         try {
-            typesRegistry.addType(ret);
+            AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry();
+
+            ttr.addType(ret);
+
+            typesRegistry.commitTransientTypeRegistry(ttr);
         } catch (AtlasBaseException excp) {
             LOG.error("failed to create enum-def", excp);
 
@@ -182,7 +187,11 @@ public final class  ModelTestUtil {
         ret.setAttributeDefs(newAttributeDefsWithAllBuiltInTypes(PREFIX_ATTRIBUTE_NAME));
 
         try {
-            typesRegistry.addType(ret);
+            AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry();
+
+            ttr.addType(ret);
+
+            typesRegistry.commitTransientTypeRegistry(ttr);
         } catch (AtlasBaseException excp) {
             LOG.error("failed to create struct-def", excp);
 
@@ -220,7 +229,11 @@ public final class  ModelTestUtil {
         }
 
         try {
-            typesRegistry.addType(ret);
+            AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry();
+
+            ttr.addType(ret);
+
+            typesRegistry.commitTransientTypeRegistry(ttr);
         } catch (AtlasBaseException excp) {
             LOG.error("failed to create entity-def", excp);
 
@@ -267,7 +280,11 @@ public final class  ModelTestUtil {
         }
 
         try {
-            typesRegistry.addType(ret);
+            AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry();
+
+            ttr.addType(ret);
+
+            typesRegistry.commitTransientTypeRegistry(ttr);
         } catch (AtlasBaseException excp) {
             LOG.error("failed to create classification-def", excp);
 
@@ -285,10 +302,14 @@ public final class  ModelTestUtil {
     public static AtlasEntity newEntity(AtlasEntityDef entityDef, AtlasTypeRegistry typesRegistry) {
         AtlasEntity ret = null;
 
+        try {
             AtlasType dataType = typesRegistry.getType(entityDef.getName());
 
-        if (dataType != null && dataType instanceof AtlasEntityType) {
-            ret = ((AtlasEntityType)dataType).createDefaultValue();
+            if (dataType instanceof AtlasEntityType) {
+                ret = ((AtlasEntityType) dataType).createDefaultValue();
+            }
+        } catch (AtlasBaseException excp) {
+            LOG.error("failed to get entity-type " + entityDef.getName(), excp);
         }
 
         return ret;
@@ -301,11 +322,15 @@ public final class  ModelTestUtil {
     public static AtlasStruct newStruct(AtlasStructDef structDef, AtlasTypeRegistry typesRegistry) {
         AtlasStruct ret = null;
 
+        try {
             AtlasType dataType = typesRegistry.getType(structDef.getName());
 
-        if (dataType != null && dataType instanceof AtlasStructType) {
+            if (dataType instanceof AtlasStructType) {
                 ret = ((AtlasStructType)dataType).createDefaultValue();
             }
+        } catch (AtlasBaseException excp) {
+            LOG.error("failed to get struct-type " + structDef.getName(), excp);
+        }
 
         return ret;
     }
@@ -318,11 +343,15 @@ public final class  ModelTestUtil {
                                                         AtlasTypeRegistry typesRegistry) {
         AtlasClassification ret = null;
 
+        try {
             AtlasType dataType = typesRegistry.getType(classificationDef.getName());
 
-        if (dataType != null && dataType instanceof AtlasClassificationType) {
+            if (dataType instanceof AtlasClassificationType) {
                 ret = ((AtlasClassificationType)dataType).createDefaultValue();
             }
+        } catch (AtlasBaseException excp) {
+            LOG.error("failed to get classification-type " + classificationDef.getName(), excp);
+        }
 
         return ret;
     }

intg/src/test/java/org/apache/atlas/model/instance/TestAtlasClassification.java

@@ -17,6 +17,7 @@
  */
 package org.apache.atlas.model.instance;
 
+import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.ModelTestUtil;
 import org.apache.atlas.model.typedef.AtlasClassificationDef;
 import org.apache.atlas.type.AtlasType;
@@ -31,7 +32,7 @@ import static org.testng.Assert.assertTrue;
 public class TestAtlasClassification {
 
     @Test
-    public void testClassificationSerDe() {
+    public void testClassificationSerDe() throws AtlasBaseException {
         AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDef();
         AtlasTypeRegistry      typeRegistry      = ModelTestUtil.getTypesRegistry();
         AtlasType              dataType          = typeRegistry.getType(classificationDef.getName());
@@ -50,7 +51,7 @@ public class TestAtlasClassification {
     }
 
     @Test
-    public void testClassificationSerDeWithSuperType() {
+    public void testClassificationSerDeWithSuperType() throws AtlasBaseException {
         AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDefWithSuperType();
         AtlasTypeRegistry      typeRegistry      = ModelTestUtil.getTypesRegistry();
         AtlasType              dataType          = typeRegistry.getType(classificationDef.getName());
@@ -69,7 +70,7 @@ public class TestAtlasClassification {
     }
 
     @Test
-    public void testClassificationSerDeWithSuperTypes() {
+    public void testClassificationSerDeWithSuperTypes() throws AtlasBaseException {
         AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDefWithSuperTypes();
         AtlasTypeRegistry      typeRegistry      = ModelTestUtil.getTypesRegistry();
         AtlasType              dataType          = typeRegistry.getType(classificationDef.getName());

intg/src/test/java/org/apache/atlas/model/instance/TestAtlasEntity.java

@@ -17,6 +17,7 @@
  */
 package org.apache.atlas.model.instance;
 
+import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.ModelTestUtil;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.type.AtlasType;
@@ -31,7 +32,7 @@ import static org.testng.Assert.assertTrue;
 public class TestAtlasEntity {
 
     @Test
-    public void testEntitySerDe() {
+    public void testEntitySerDe() throws AtlasBaseException {
         AtlasEntityDef    entityDef    = ModelTestUtil.getEntityDef();
         AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
         AtlasType         dataType     = typeRegistry.getType(entityDef.getName());
@@ -50,7 +51,7 @@ public class TestAtlasEntity {
     }
 
     @Test
-    public void testEntitySerDeWithSuperType() {
+    public void testEntitySerDeWithSuperType() throws AtlasBaseException {
         AtlasEntityDef    entityDef    = ModelTestUtil.getEntityDefWithSuperType();
         AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
         AtlasType         dataType     = typeRegistry.getType(entityDef.getName());
@@ -69,7 +70,7 @@ public class TestAtlasEntity {
     }
 
     @Test
-    public void testEntitySerDeWithSuperTypes() {
+    public void testEntitySerDeWithSuperTypes() throws AtlasBaseException {
         AtlasEntityDef    entityDef    = ModelTestUtil.getEntityDefWithSuperTypes();
         AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry();
         AtlasType         dataType     = typeRegistry.getType(entityDef.getName());

intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasEntityDef.java

@@ -84,10 +84,12 @@ public class TestAtlasEntityDef {
         entityDef.addSuperType(newSuperType);
 
         assertTrue(entityDef.hasSuperType(newSuperType));
+
+        entityDef.removeSuperType(newSuperType);
     }
 
     @Test
-    public void testEntityDefDefRemoveElement() {
+    public void testEntityDefRemoveElement() {
         AtlasEntityDef entityDef = ModelTestUtil.newEntityDefWithSuperTypes();
 
         for (String superType : entityDef.getSuperTypes()) {
@@ -114,6 +116,9 @@ public class TestAtlasEntityDef {
         for (String superType : newSuperTypes) {
             assertTrue(entityDef.hasSuperType(superType));
         }
+
+        // restore old sypertypes
+        entityDef.setSuperTypes(oldSuperTypes);
     }
 
     @Test

intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java

@@ -17,7 +17,11 @@
  */
 package org.apache.atlas.type;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.ModelTestUtil;
@@ -26,6 +30,7 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
+import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry;
 import org.testng.annotations.Test;
 
 import static org.testng.Assert.*;
@@ -127,8 +132,11 @@ public class TestAtlasEntityType {
         entityDefs.add(createColumnEntityDef());
 
         try {
-            typeRegistry.addTypesWithNoRefResolve(entityDefs);
-            typeRegistry.resolveReferences();
+            AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
+
+            ttr.addTypes(entityDefs);
+
+            typeRegistry.commitTransientTypeRegistry(ttr);
         } catch (AtlasBaseException excp) {
             failureMsg = excp.getMessage();
         }
@@ -144,7 +152,11 @@ public class TestAtlasEntityType {
         entityDefs.add(createTableEntityDef());
 
         try {
-            typeRegistry.addTypes(entityDefs);
+            AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
+
+            ttr.addTypes(entityDefs);
+
+            typeRegistry.commitTransientTypeRegistry(ttr);
         } catch (AtlasBaseException excp) {
             failureMsg = excp.getMessage();
         }
@@ -161,8 +173,11 @@ public class TestAtlasEntityType {
         entityDefs.add(createColumnEntityDef());
 
         try {
-            typeRegistry.addTypesWithNoRefResolve(entityDefs);
-            typeRegistry.resolveReferences();
+            AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
+
+            ttr.addTypes(entityDefs);
+
+            typeRegistry.commitTransientTypeRegistry(ttr);
         } catch (AtlasBaseException excp) {
             failureMsg = excp.getMessage();
         }
@@ -178,7 +193,11 @@ public class TestAtlasEntityType {
         entityDefs.add(createColumnEntityDef());
 
         try {
-            typeRegistry.addTypes(entityDefs);
+            AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry();
+
+            ttr.addTypes(entityDefs);
+
+            typeRegistry.commitTransientTypeRegistry(ttr);
         } catch (AtlasBaseException excp) {
             failureMsg = excp.getMessage();
         }

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1230 updated AtlasTypeRegistry to support batch, atomic type updates (mneethiraj)
 ATLAS-1229 Add TypeCategory and methods to access attribute definitiions in AtlasTypes (sumasai)
 ATLAS-1227 Added support for attribute constraints in the API (mneethiraj)
 ATLAS-1225 Optimize AtlasTypeDefGraphStore to use typeRegistry (mneethiraj via sumasai)

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasAbstractDefStoreV1.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v1;
+
+import org.apache.atlas.type.AtlasTypeRegistry;
+
+/**
+ * Abstract typedef-store for v1 format.
+ */
+public abstract class AtlasAbstractDefStoreV1 {
+    protected final AtlasTypeDefGraphStoreV1 typeDefStore;
+    protected final AtlasTypeRegistry        typeRegistry;
+
+    public AtlasAbstractDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
+        this.typeDefStore = typeDefStore;
+        this.typeRegistry = typeRegistry;
+    }
+}

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java

@@ -26,6 +26,9 @@ import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.AtlasClassificationDefStore;
 import org.apache.atlas.repository.util.FilterUtil;
+import org.apache.atlas.type.AtlasClassificationType;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.commons.collections.CollectionUtils;
 import org.slf4j.Logger;
@@ -33,21 +36,16 @@ import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 
 /**
  * ClassificationDef store in v1 format.
  */
-public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStore {
+public class AtlasClassificationDefStoreV1 extends AtlasAbstractDefStoreV1 implements AtlasClassificationDefStore {
     private static final Logger LOG = LoggerFactory.getLogger(AtlasClassificationDefStoreV1.class);
 
-    private final AtlasTypeDefGraphStoreV1 typeDefStore;
-
-    public AtlasClassificationDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore) {
-        super();
-
-        this.typeDefStore = typeDefStore;
+    public AtlasClassificationDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
+        super(typeDefStore, typeRegistry);
     }
 
     @Override
@@ -80,20 +78,17 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> AtlasClassificationDefStoreV1.create({})", classificationDefs);
         }
-        List<AtlasClassificationDef> classificationDefList = new LinkedList<>();
-        for (AtlasClassificationDef structDef : classificationDefs) {
-            try {
-                AtlasClassificationDef atlasClassificationDef = create(structDef);
-                classificationDefList.add(atlasClassificationDef);
-            } catch (AtlasBaseException baseException) {
-                LOG.error("Failed to create {}", structDef);
-                LOG.error("Exception: {}", baseException);
-            }
+
+        List<AtlasClassificationDef> ret = new ArrayList<>();
+
+        for (AtlasClassificationDef classificationDef : classificationDefs) {
+            ret.add(create(classificationDef));
         }
+
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasClassificationDefStoreV1.create({}, {})", classificationDefs, classificationDefList);
+            LOG.debug("<== AtlasClassificationDefStoreV1.create({}, {})", classificationDefs, ret);
         }
-        return classificationDefList;
+        return ret;
     }
 
     @Override
@@ -102,17 +97,17 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
             LOG.debug("==> AtlasClassificationDefStoreV1.getAll()");
         }
 
-        List<AtlasClassificationDef> classificationDefs = new LinkedList<>();
-        Iterator<AtlasVertex> verticesByCategory = typeDefStore.findTypeVerticesByCategory(TypeCategory.TRAIT);
-        while (verticesByCategory.hasNext()) {
-            AtlasClassificationDef classificationDef = toClassificationDef(verticesByCategory.next());
-            classificationDefs.add(classificationDef);
+        List<AtlasClassificationDef> ret = new ArrayList<>();
+
+        Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.TRAIT);
+        while (vertices.hasNext()) {
+            ret.add(toClassificationDef(vertices.next()));
         }
 
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasClassificationDefStoreV1.getAll()");
+            LOG.debug("<== AtlasClassificationDefStoreV1.getAll(): count={}", ret.size());
         }
-        return classificationDefs;
+        return ret;
     }
 
     @Override
@@ -211,22 +206,17 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
             LOG.debug("==> AtlasClassificationDefStoreV1.update({})", classificationDefs);
         }
 
-        List<AtlasClassificationDef> updatedClassificationDefs = new ArrayList<>();
+        List<AtlasClassificationDef> ret = new ArrayList<>();
 
         for (AtlasClassificationDef classificationDef : classificationDefs) {
-            try {
-                AtlasClassificationDef updatedDef = updateByName(classificationDef.getName(), classificationDef);
-                updatedClassificationDefs.add(updatedDef);
-            } catch (AtlasBaseException ex) {
-                LOG.error("Failed to update {}", classificationDef);
-            }
+            ret.add(updateByName(classificationDef.getName(), classificationDef));
         }
 
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasClassificationDefStoreV1.update({}): {}", classificationDefs, updatedClassificationDefs);
+            LOG.debug("<== AtlasClassificationDefStoreV1.update({}): {}", classificationDefs, ret);
         }
 
-        return updatedClassificationDefs;
+        return ret;
     }
 
     @Override
@@ -255,11 +245,7 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
         }
 
         for (String name : names) {
-            try {
             deleteByName(name);
-            } catch (AtlasBaseException ex) {
-                LOG.error("Failed to delete {}", name);
-            }
         }
 
         if (LOG.isDebugEnabled()) {
@@ -293,11 +279,7 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
         }
 
         for (String guid : guids) {
-            try {
             deleteByGuid(guid);
-            } catch (AtlasBaseException ex) {
-                LOG.error("Failed to delete {}", guid);
-            }
         }
 
         if (LOG.isDebugEnabled()) {
@@ -313,7 +295,6 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
         }
 
         List<AtlasClassificationDef> classificationDefs = new ArrayList<AtlasClassificationDef>();
-
         Iterator<AtlasVertex>        vertices           = typeDefStore.findTypeVerticesByCategory(TypeCategory.TRAIT);
 
         while(vertices.hasNext()) {
@@ -321,14 +302,11 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
             AtlasClassificationDef classificationDef = toClassificationDef(vertex);
 
             if (classificationDef != null) {
-                classificationDefs.add(classificationDef); // TODO: add only if this passes filter
+                classificationDefs.add(classificationDef);
             }
         }
 
-        if (CollectionUtils.isNotEmpty(classificationDefs)) {
         CollectionUtils.filter(classificationDefs, FilterUtil.getPredicateFromSearchFilter(filter));
-        }
-
 
         AtlasClassificationDefs ret = new AtlasClassificationDefs(classificationDefs);
 
@@ -339,8 +317,15 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
         return ret;
     }
 
-    private void toVertex(AtlasClassificationDef classificationDef, AtlasVertex vertex) {
-        AtlasStructDefStoreV1.toVertex(classificationDef, vertex, typeDefStore);
+    private void toVertex(AtlasClassificationDef classificationDef, AtlasVertex vertex) throws AtlasBaseException {
+        AtlasType type = typeRegistry.getType(classificationDef.getName());
+
+        if (type.getTypeCategory() != AtlasType.TypeCategory.CLASSIFICATION) {
+            throw new AtlasBaseException(classificationDef.getName() + ": not a classification type");
+        }
+
+        AtlasStructDefStoreV1.toVertex(classificationDef, (AtlasClassificationType)type,
+                                       vertex, typeDefStore, typeRegistry);
 
         typeDefStore.createSuperTypeEdges(vertex, classificationDef.getSuperTypes());
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java

@@ -21,11 +21,13 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.SearchFilter;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEntityDef.AtlasEntityDefs;
-import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.AtlasEntityDefStore;
 import org.apache.atlas.repository.util.FilterUtil;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.commons.collections.CollectionUtils;
 import org.slf4j.Logger;
@@ -33,21 +35,16 @@ import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 
 /**
  * EntityDef store in v1 format.
  */
-public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
+public class AtlasEntityDefStoreV1 extends AtlasAbstractDefStoreV1 implements AtlasEntityDefStore {
     private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityDefStoreV1.class);
 
-    private final AtlasTypeDefGraphStoreV1 typeDefStore;
-
-    public AtlasEntityDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore) {
-        super();
-
-        this.typeDefStore = typeDefStore;
+    public AtlasEntityDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
+        super(typeDefStore, typeRegistry);
     }
 
     @Override
@@ -80,20 +77,15 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> AtlasEntityDefStoreV1.create({})", entityDefs);
         }
-        List<AtlasEntityDef> entityDefList = new LinkedList<>();
-        for (AtlasEntityDef structDef : entityDefs) {
-            try {
-                AtlasEntityDef atlasEntityDef = create(structDef);
-                entityDefList.add(atlasEntityDef);
-            } catch (AtlasBaseException baseException) {
-                LOG.error("Failed to create {}", structDef);
-                LOG.error("Exception: {}", baseException);
-            }
+        List<AtlasEntityDef> ret = new ArrayList<>();
+
+        for (AtlasEntityDef entityDef : entityDefs) {
+            ret.add(create(entityDef));
         }
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasEntityDefStoreV1.create({}, {})", entityDefs, entityDefList);
+            LOG.debug("<== AtlasEntityDefStoreV1.create({}, {})", entityDefs, ret);
         }
-        return entityDefList;
+        return ret;
     }
 
     @Override
@@ -102,17 +94,18 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
             LOG.debug("==> AtlasEntityDefStoreV1.getAll()");
         }
 
-        List<AtlasEntityDef> entityDefs = new LinkedList<>();
-        Iterator<AtlasVertex> verticesByCategory = typeDefStore.findTypeVerticesByCategory(TypeCategory.CLASS);
-        while (verticesByCategory.hasNext()) {
-            AtlasEntityDef atlasEntityDef = toEntityDef(verticesByCategory.next());
-            entityDefs.add(atlasEntityDef);
+        List<AtlasEntityDef>  ret      = new ArrayList<>();
+        Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.CLASS);
+
+        while (vertices.hasNext()) {
+            ret.add(toEntityDef(vertices.next()));
         }
 
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasEntityDefStoreV1.getAll()");
+            LOG.debug("<== AtlasEntityDefStoreV1.getAll(): count={}", ret.size());
         }
-        return entityDefs;
+
+        return ret;
     }
 
     @Override
@@ -211,22 +204,17 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
             LOG.debug("==> AtlasEntityDefStoreV1.update({})", entityDefs);
         }
 
-        List<AtlasEntityDef> updatedEntityDefs = new ArrayList<>();
+        List<AtlasEntityDef> ret = new ArrayList<>();
 
         for (AtlasEntityDef entityDef : entityDefs) {
-            try {
-                AtlasEntityDef atlasEntityDef = updateByName(entityDef.getName(), entityDef);
-                updatedEntityDefs.add(atlasEntityDef);
-            } catch (AtlasBaseException ex) {
-                LOG.error("Failed to update {}", entityDef);
-            }
+            ret.add(updateByName(entityDef.getName(), entityDef));
         }
 
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasEntityDefStoreV1.update({}): {}", entityDefs, updatedEntityDefs);
+            LOG.debug("<== AtlasEntityDefStoreV1.update({}): {}", entityDefs, ret);
         }
 
-        return updatedEntityDefs;
+        return ret;
     }
 
     @Override
@@ -254,12 +242,8 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
             LOG.debug("==> AtlasEntityDefStoreV1.deleteByNames({})", names);
         }
 
-        List<AtlasStructDef> updatedDefs = new ArrayList<>();
-
         for (String name : names) {
-            try {
             deleteByName(name);
-            } catch (AtlasBaseException ex) {}
         }
 
         if (LOG.isDebugEnabled()) {
@@ -292,12 +276,8 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
             LOG.debug("==> AtlasEntityDefStoreV1.deleteByGuids({})", guids);
         }
 
-        List<AtlasStructDef> updatedDefs = new ArrayList<>();
-
         for (String guid : guids) {
-            try {
             deleteByGuid(guid);
-            } catch (AtlasBaseException ex) {}
         }
 
         if (LOG.isDebugEnabled()) {
@@ -312,7 +292,6 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
         }
 
         List<AtlasEntityDef>  entityDefs = new ArrayList<AtlasEntityDef>();
-
         Iterator<AtlasVertex> vertices   = typeDefStore.findTypeVerticesByCategory(TypeCategory.CLASS);
 
         while(vertices.hasNext()) {
@@ -324,10 +303,7 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
             }
         }
 
-        if (CollectionUtils.isNotEmpty(entityDefs)) {
         CollectionUtils.filter(entityDefs, FilterUtil.getPredicateFromSearchFilter(filter));
-        }
-
 
         AtlasEntityDefs ret = new AtlasEntityDefs(entityDefs);
 
@@ -338,8 +314,14 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
         return ret;
     }
 
-    private void toVertex(AtlasEntityDef entityDef, AtlasVertex vertex) {
-        AtlasStructDefStoreV1.toVertex(entityDef, vertex, typeDefStore);
+    private void toVertex(AtlasEntityDef entityDef, AtlasVertex vertex) throws AtlasBaseException {
+        AtlasType type = typeRegistry.getType(entityDef.getName());
+
+        if (type.getTypeCategory() != AtlasType.TypeCategory.ENTITY) {
+            throw new AtlasBaseException(entityDef.getName() + ": not a entity type");
+        }
+
+        AtlasStructDefStoreV1.toVertex(entityDef, (AtlasEntityType)type, vertex, typeDefStore, typeRegistry);
 
         typeDefStore.createSuperTypeEdges(vertex, entityDef.getSuperTypes());
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEnumDefStoreV1.java

@@ -26,6 +26,7 @@ import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.AtlasEnumDefStore;
 import org.apache.atlas.repository.util.FilterUtil;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -34,21 +35,16 @@ import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 
 /**
  * EnumDef store in v1 format.
  */
-public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
+public class AtlasEnumDefStoreV1 extends AtlasAbstractDefStoreV1 implements AtlasEnumDefStore {
     private static final Logger LOG = LoggerFactory.getLogger(AtlasEnumDefStoreV1.class);
 
-    private final AtlasTypeDefGraphStoreV1 typeDefStore;
-
-    public AtlasEnumDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore) {
-        super();
-
-        this.typeDefStore = typeDefStore;
+    public AtlasEnumDefStoreV1(AtlasTypeDefGraphStoreV1 typeDefStore, AtlasTypeRegistry typeRegistry) {
+        super(typeDefStore, typeRegistry);
     }
 
     @Override
@@ -77,24 +73,22 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
     }
 
     @Override
-    public List<AtlasEnumDef> create(List<AtlasEnumDef> atlasEnumDefs) throws AtlasBaseException {
+    public List<AtlasEnumDef> create(List<AtlasEnumDef> enumDefs) throws AtlasBaseException {
         if (LOG.isDebugEnabled()) {
-            LOG.debug("==> AtlasEnumDefStoreV1.create({})", atlasEnumDefs);
-        }
-        List<AtlasEnumDef> enumDefList = new LinkedList<>();
-        for (AtlasEnumDef enumDef : atlasEnumDefs) {
-            try {
-                AtlasEnumDef atlasEnumDef = create(enumDef);
-                enumDefList.add(atlasEnumDef);
-            } catch (AtlasBaseException baseException) {
-                LOG.error("Failed to create {}", enumDef);
-                LOG.error("Exception: {}", baseException);
+            LOG.debug("==> AtlasEnumDefStoreV1.create({})", enumDefs);
         }
+
+        List<AtlasEnumDef> ret = new ArrayList<>();
+
+        for (AtlasEnumDef enumDef : enumDefs) {
+            ret.add(create(enumDef));
         }
+
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasEnumDefStoreV1.create({}, {})", atlasEnumDefs, enumDefList);
+            LOG.debug("<== AtlasEnumDefStoreV1.create({}): {}", enumDefs, ret);
         }
-        return enumDefList;
+
+        return ret;
     }
 
     @Override
@@ -103,17 +97,18 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
             LOG.debug("==> AtlasEnumDefStoreV1.getAll()");
         }
 
-        List<AtlasEnumDef> enumDefs = new LinkedList<>();
-        Iterator<AtlasVertex> verticesByCategory = typeDefStore.findTypeVerticesByCategory(TypeCategory.ENUM);
-        while (verticesByCategory.hasNext()) {
-            AtlasEnumDef enumDef = toEnumDef(verticesByCategory.next());
-            enumDefs.add(enumDef);
+        List<AtlasEnumDef> ret = new ArrayList<>();
+
+        Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.ENUM);
+        while (vertices.hasNext()) {
+            ret.add(toEnumDef(vertices.next()));
         }
 
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasEnumDefStoreV1.getAll()");
+            LOG.debug("<== AtlasEnumDefStoreV1.getAll(): count={}", ret.size());
         }
-        return enumDefs;
+
+        return ret;
     }
 
     @Override
@@ -212,22 +207,17 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
             LOG.debug("==> AtlasEnumDefStoreV1.update({})", enumDefs);
         }
 
-        List<AtlasEnumDef> updatedEnumDefs = new ArrayList<>();
+        List<AtlasEnumDef> ret = new ArrayList<>();
 
         for (AtlasEnumDef enumDef : enumDefs) {
-            try {
-                AtlasEnumDef updatedDef = updateByName(enumDef.getName(), enumDef);
-                updatedEnumDefs.add(updatedDef);
-            } catch (AtlasBaseException ex) {
-                LOG.error("Failed to update {}", enumDef);
-            }
+            ret.add(updateByName(enumDef.getName(), enumDef));
         }
 
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== AtlasEnumDefStoreV1.update({}): {}", enumDefs, updatedEnumDefs);
+            LOG.debug("<== AtlasEnumDefStoreV1.update({}): {}", enumDefs, ret);
         }
 
-        return updatedEnumDefs;
+        return ret;
     }
 
     @Override
@@ -256,11 +246,7 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
         }
 
         for (String name : names) {
-            try {
             deleteByName(name);
-            } catch (AtlasBaseException ex) {
-                LOG.error("Failed to delete {}", name);
-            }
         }
 
         if (LOG.isDebugEnabled()) {
@@ -294,11 +280,7 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
         }
 
         for (String guid : guids) {
-            try {
             deleteByGuid(guid);
-            } catch (AtlasBaseException ex) {
-                LOG.error("Failed to delete {}", guid);
-            }
         }
 
         if (LOG.isDebugEnabled()) {
@@ -313,7 +295,6 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
         }
 
         List<AtlasEnumDef>    enumDefs = new ArrayList<AtlasEnumDef>();
-
         Iterator<AtlasVertex> vertices = typeDefStore.findTypeVerticesByCategory(TypeCategory.ENUM);
 
         while(vertices.hasNext()) {
@@ -325,9 +306,7 @@ public class AtlasEnumDefStoreV1 implements AtlasEnumDefStore {
             }
         }
 
-        if (CollectionUtils.isNotEmpty(enumDefs)) {
         CollectionUtils.filter(enumDefs, FilterUtil.getPredicateFromSearchFilter(filter));
-        }
 
         AtlasEnumDefs ret = new AtlasEnumDefs(enumDefs);
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java

@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v1;
 
 import com.google.common.base.Preconditions;
 
+import com.google.inject.Inject;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
 import org.apache.atlas.model.typedef.AtlasClassificationDef;
@@ -31,7 +32,12 @@ import org.apache.atlas.repository.graphdb.AtlasEdge;
 import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
 import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.AtlasClassificationDefStore;
+import org.apache.atlas.repository.store.graph.AtlasEntityDefStore;
+import org.apache.atlas.repository.store.graph.AtlasEnumDefStore;
+import org.apache.atlas.repository.store.graph.AtlasStructDefStore;
 import org.apache.atlas.repository.store.graph.AtlasTypeDefGraphStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -57,8 +63,9 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
 
     protected final AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance();
 
-    public AtlasTypeDefGraphStoreV1() {
-        super();
+    @Inject
+    public AtlasTypeDefGraphStoreV1(AtlasTypeRegistry typeRegistry) {
+        super(typeRegistry);
 
         LOG.info("==> AtlasTypeDefGraphStoreV1()");
 
@@ -72,16 +79,31 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
     }
 
     @Override
+    protected AtlasEnumDefStore getEnumDefStore(AtlasTypeRegistry typeRegistry) {
+        return new AtlasEnumDefStoreV1(this, typeRegistry);
+    }
+
+    @Override
+    protected AtlasStructDefStore getStructDefStore(AtlasTypeRegistry typeRegistry) {
+        return new AtlasStructDefStoreV1(this, typeRegistry);
+    }
+
+    @Override
+    protected AtlasClassificationDefStore getClassificationDefStore(AtlasTypeRegistry typeRegistry) {
+        return new AtlasClassificationDefStoreV1(this, typeRegistry);
+    }
+
+    @Override
+    protected AtlasEntityDefStore getEntityDefStore(AtlasTypeRegistry typeRegistry) {
+        return new AtlasEntityDefStoreV1(this, typeRegistry);
+    }
+
+    @Override
     public void init() throws AtlasBaseException {
         LOG.info("==> AtlasTypeDefGraphStoreV1.init()");
 
         super.init();
 
-        super.init(new AtlasEnumDefStoreV1(this),
-                   new AtlasStructDefStoreV1(this),
-                   new AtlasClassificationDefStoreV1(this),
-                   new AtlasEntityDefStoreV1(this));
-
         LOG.info("<== AtlasTypeDefGraphStoreV1.init()");
     }
 

webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java

@@ -64,7 +64,7 @@ public class TypesREST {
     private HttpServletRequest httpServletRequest;
 
     @Inject
-    public TypesREST(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry atlasTypeRegistry) {
+    public TypesREST(AtlasTypeDefStore typeDefStore) {
         LOG.info("new TypesREST");
         this.typeDefStore = typeDefStore;
     }