ATLAS-1225: Updated AtlasTypeDefGraphStore to use AtlasTypesRegistry, to reduce query to Graph DB

intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java

@@ -195,7 +195,16 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
         sb.append("AtlasStructDef{");
         super.toString(sb);
         sb.append(", attributeDefs=[");
-        dumpObjects(attributeDefs, sb);
+        if (CollectionUtils.isNotEmpty(attributeDefs)) {
+            int i = 0;
+            for (AtlasAttributeDef attributeDef : attributeDefs) {
+                attributeDef.toString(sb);
+                if (i > 0) {
+                    sb.append(", ");
+                }
+                i++;
+            }
+        }
         sb.append("]");
         sb.append('}');
 
@@ -287,7 +296,6 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
             }
         }
 
-
         public String getName() {
             return name;
         }
@@ -350,7 +358,6 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
             isIndexable = idexable;
         }
 
-
         public StringBuilder toString(StringBuilder sb) {
             if (sb == null) {
                 sb = new StringBuilder();
@@ -410,7 +417,6 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
         }
     }
 
-
     /**
      * REST serialization friendly list.
      */

intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java

@@ -114,7 +114,7 @@ public interface AtlasTypeDefStore {
     /*************************/
     /** EntityDef operation **/
     /*************************/
-    AtlasEntityDef createEntityDefs(AtlasEntityDef entityDef) throws AtlasBaseException;
+    AtlasEntityDef createEntityDef(AtlasEntityDef entityDef) throws AtlasBaseException;
 
     List<AtlasEntityDef> createEntityDefs(List<AtlasEntityDef> entityDefs) throws AtlasBaseException;
 

intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java

@@ -84,6 +84,22 @@ public class AtlasArrayType extends AtlasType {
         this.resolveReferences(typeRegistry);
     }
 
+    public String getElementTypeName() {
+        return elementTypeName;
+    }
+
+    public int getMinCount() {
+        return minCount;
+    }
+
+    public int getMaxCount() {
+        return maxCount;
+    }
+
+    public AtlasType getElementType() {
+        return elementType;
+    }
+
     @Override
     public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
         elementType = typeRegistry.getType(elementTypeName);

intg/src/main/java/org/apache/atlas/type/AtlasEnumType.java

@@ -22,6 +22,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
 
@@ -59,6 +60,10 @@ public class AtlasEnumType extends AtlasType {
     }
 
     @Override
+    public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+    }
+
+    @Override
     public Object createDefaultValue() {
         return defaultValue;
     }

intg/src/main/java/org/apache/atlas/type/AtlasMapType.java

@@ -64,6 +64,26 @@ public class AtlasMapType extends AtlasType {
         resolveReferences(typeRegistry);
     }
 
+    public String getKeyTypeName() {
+        return keyTypeName;
+    }
+
+    public String getValueTypeName() {
+        return valueTypeName;
+    }
+
+    public AtlasType getKeyType() {
+        return keyType;
+    }
+
+    public AtlasType getValueType() {
+        return valueType;
+    }
+
+    public void setKeyType(AtlasType keyType) {
+        this.keyType = keyType;
+    }
+
     @Override
     public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
         this.keyType   = typeRegistry.getType(keyTypeName);

intg/src/main/java/org/apache/atlas/type/AtlasStructType.java

@@ -38,6 +38,7 @@ public class AtlasStructType extends AtlasType {
 
     private Map<String, AtlasType> attrTypes = Collections.emptyMap();
 
+
     public AtlasStructType(AtlasStructDef structDef) {
         super(structDef.getName());
 
@@ -52,6 +53,10 @@ public class AtlasStructType extends AtlasType {
         this.resolveReferences(typeRegistry);
     }
 
+    public AtlasType getAttributeType(String attributeName) { return attrTypes.get(attributeName); }
+
+    public AtlasAttributeDef getAttributeDef(String attributeName) { return structDef.getAttribute(attributeName); }
+
     @Override
     public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
         Map<String, AtlasType> a = new HashMap<String, AtlasType>();
@@ -59,24 +64,20 @@ public class AtlasStructType extends AtlasType {
         for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
             AtlasType attrType = typeRegistry.getType(attributeDef.getTypeName());
 
-            if (attrType != null) {
-                Cardinality cardinality = attributeDef.getCardinality();
-
-                if (cardinality == Cardinality.LIST || cardinality == Cardinality.SET) {
-                    attrType = new AtlasArrayType(attrType,
-                                                  attributeDef.getValuesMinCount(),
-                                                  attributeDef.getValuesMaxCount());
-                }
-
-                a.put(attributeDef.getName(), attrType);
-            } else {
-                String msg = attributeDef.getTypeName() + ": unknown type for attribute "
-                             + structDef.getName() + "." + attributeDef.getName();
+            if (attrType == null) {
+                throw new AtlasBaseException(attributeDef.getTypeName() + ": unknown type for attribute "
+                                             + structDef.getName() + "." + attributeDef.getName());
+            }
 
-                LOG.error(msg);
+            Cardinality cardinality = attributeDef.getCardinality();
 
-                throw new AtlasBaseException(msg);
+            if (cardinality == Cardinality.LIST || cardinality == Cardinality.SET) {
+                attrType = new AtlasArrayType(attrType,
+                                              attributeDef.getValuesMinCount(),
+                                              attributeDef.getValuesMaxCount());
             }
+
+            a.put(attributeDef.getName(), attrType);
         }
 
         this.attrTypes = Collections.unmodifiableMap(a);

intg/src/main/java/org/apache/atlas/type/AtlasType.java

@@ -41,7 +41,6 @@ public abstract class AtlasType {
     }
 
     public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
-
     }
 
     public String getTypeName() { return typeName; }

intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java

@@ -157,7 +157,13 @@ public final class  ModelTestUtil {
             ret.setDefaultValue(ret.getElementDefs().get(idxDefault).getValue());
         }
 
-        typesRegistry.addEnumDef(ret);
+        try {
+            typesRegistry.addType(ret);
+        } catch (AtlasBaseException excp) {
+            LOG.error("failed to create enum-def", excp);
+
+            ret = null;
+        }
 
         return ret;
     }
@@ -176,7 +182,7 @@ public final class  ModelTestUtil {
         ret.setAttributeDefs(newAttributeDefsWithAllBuiltInTypes(PREFIX_ATTRIBUTE_NAME));
 
         try {
-            typesRegistry.addStructDef(ret);
+            typesRegistry.addType(ret);
         } catch (AtlasBaseException excp) {
             LOG.error("failed to create struct-def", excp);
 
@@ -214,7 +220,7 @@ public final class  ModelTestUtil {
         }
 
         try {
-            typesRegistry.addEntityDef(ret);
+            typesRegistry.addType(ret);
         } catch (AtlasBaseException excp) {
             LOG.error("failed to create entity-def", excp);
 
@@ -261,7 +267,7 @@ public final class  ModelTestUtil {
         }
 
         try {
-            typesRegistry.addClassificationDef(ret);
+            typesRegistry.addType(ret);
         } catch (AtlasBaseException excp) {
             LOG.error("failed to create classification-def", excp);
 

intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java

@@ -35,7 +35,7 @@ public class TestAtlasEntityType {
     private final List<Object>    invalidValues = new ArrayList<Object>();
 
     {
-        entityType = getEntityType(ModelTestUtil.getEntityDefWithSuperTypes());
+        entityType  = getEntityType(ModelTestUtil.getEntityDefWithSuperTypes());
 
         AtlasEntity         invalidValue1 = entityType.createDefaultValue();
         AtlasEntity         invalidValue2 = entityType.createDefaultValue();

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1225 Optimize AtlasTypeDefGraphStore to use typeRegistry (mneethiraj via sumasai)
 ATLAS-1221 build failure in windows SyntaxError: invalid syntax (zhangqiang2 via shwethags)
 ATLAS-1226 Servlet init-params in web.xml are unused (mneethiraj via shwethags)
 ATLAS-1224 Minor fixes for hive and falcon bridge twiki (ayubkhan via sumasai) 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java

@@ -31,8 +31,6 @@ import org.apache.commons.collections.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import sun.security.provider.certpath.Vertex;
-
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.LinkedList;

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java

@@ -21,6 +21,7 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.SearchFilter;
 import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
@@ -381,7 +382,7 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
                 String propertyKey = AtlasGraphUtilsV1.getPropertyKey(ret, attrName);
                 String attribJson  = atlasVertex.getProperty(propertyKey, String.class);
 
-                attributeDefs.add(toAttributeDefFromJson(attribJson));
+                attributeDefs.add(toAttributeDefFromJson(AtlasType.fromJson(attribJson, Map.class)));
             }
         }
         ret.setAttributeDefs(attributeDefs);
@@ -412,14 +413,17 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
     }
 
     private static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef) {
+        Boolean isComposite       = Boolean.FALSE;
+        String  reverseAttribName = null;
+
         Map<String, Object> attribInfo = new HashMap<String, Object>();
 
         attribInfo.put("name", attributeDef.getName());
         attribInfo.put("dataType", attributeDef.getTypeName());
         attribInfo.put("isUnique", attributeDef.isUnique());
         attribInfo.put("isIndexable", attributeDef.isIndexable());
-        attribInfo.put("isComposite", Boolean.FALSE);
-        attribInfo.put("reverseAttributeName", "");
+        attribInfo.put("isComposite", isComposite);
+        attribInfo.put("reverseAttributeName", reverseAttribName);
         Map<String, Object> multiplicity = new HashMap<String, Object>();
         multiplicity.put("lower", attributeDef.getValuesMinCount());
         multiplicity.put("upper", attributeDef.getValuesMaxCount());
@@ -430,23 +434,23 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
         return AtlasType.toJson(attribInfo);
     }
 
-    private static AtlasAttributeDef toAttributeDefFromJson(String json) {
-        Map attribInfo = AtlasType.fromJson(json, Map.class);
-
+    private static AtlasAttributeDef toAttributeDefFromJson(Map attribInfo) {
         AtlasAttributeDef ret = new AtlasAttributeDef();
 
         ret.setName((String) attribInfo.get("name"));
         ret.setTypeName((String) attribInfo.get("dataType"));
         ret.setUnique((Boolean) attribInfo.get("isUnique"));
         ret.setIndexable((Boolean) attribInfo.get("isIndexable"));
+
         /*
-        attributeMap.put("isComposite", isComposite);
-        attributeMap.put("reverseAttributeName", reverseAttributeName);
+        String  reverseAttribName = (String)attribInfo.get("reverseAttributeName");
+        Boolean isComposite       = (Boolean) attribInfo.get("isComposite");
         */
-        Map multiplicity = AtlasType.fromJson((String) attribInfo.get("multiplicity"), Map.class);
-        Number  minCount = (Number) multiplicity.get("lower");
-        Number  maxCount = (Number) multiplicity.get("upper");
-        Boolean isUnique = (Boolean) multiplicity.get("isUnique");
+        Map     multiplicity      = AtlasType.fromJson((String) attribInfo.get("multiplicity"), Map.class);
+        Number  minCount          = (Number) multiplicity.get("lower");
+        Number  maxCount          = (Number) multiplicity.get("upper");
+        Boolean isUnique          = (Boolean) multiplicity.get("isUnique");
+
 
         if (minCount == null || minCount.intValue() == 0) {
             ret.setOptional(true);

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java

@@ -60,15 +60,29 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
     public AtlasTypeDefGraphStoreV1() {
         super();
 
-        enumDefStore           = new AtlasEnumDefStoreV1(this);
-        structDefStore         = new AtlasStructDefStoreV1(this);
-        classificationDefStore = new AtlasClassificationDefStoreV1(this);
-        entityDefStore         = new AtlasEntityDefStoreV1(this);
+        LOG.info("==> AtlasTypeDefGraphStoreV1()");
+
+        try {
+            init();
+        } catch(AtlasBaseException excp) {
+            LOG.error("failed to initialize types from graph store", excp);
+        }
+
+        LOG.info("<== AtlasTypeDefGraphStoreV1()");
     }
 
     @Override
-    public void init() {
+    public void init() throws AtlasBaseException {
+        LOG.info("==> AtlasTypeDefGraphStoreV1.init()");
+
+        super.init();
+
+        super.init(new AtlasEnumDefStoreV1(this),
+                   new AtlasStructDefStoreV1(this),
+                   new AtlasClassificationDefStoreV1(this),
+                   new AtlasEntityDefStoreV1(this));
 
+        LOG.info("<== AtlasTypeDefGraphStoreV1.init()");
     }
 
     public AtlasGraph getAtlasGraph() { return atlasGraph; }

webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java

@@ -350,7 +350,7 @@ public class TypesREST {
         AtlasEntityDef ret = null;
 
         try {
-            ret = typeDefStore.createEntityDefs(entityDef);
+            ret = typeDefStore.createEntityDef(entityDef);
             return ret;
         } catch (AtlasBaseException ex) {
             throw new WebApplicationException(Servlets.getErrorResponse(ex, Response.Status.BAD_REQUEST));