ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai)

release-log.txt

@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai)
 ATLAS-525 Drop support for partitions, select query lineage, roles, principals, resource, hive_type...(sumasai via shwethags)
 ATLAS-599 HDFS Path Model (sumasai via yhemanth)
 ATLAS-553 Entity mutation - Fix issue with reordering of elements in array<class> with composite references (sumasai via shwethags)

repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java

@@ -30,13 +30,13 @@ import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
 import org.apache.atlas.typesystem.types.ClassType;
 import org.apache.atlas.typesystem.types.DataTypes;
 import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDependencySorter;
 import org.apache.atlas.typesystem.types.Multiplicity;
 import org.apache.atlas.typesystem.types.ObjectGraphWalker;
 import org.apache.atlas.typesystem.types.TraitType;
 import org.apache.atlas.typesystem.types.TypeSystem;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -46,11 +46,6 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 public class MemRepository implements IRepository {
 
-/*
-    public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
-    public static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd");
-*/
-
     final TypeSystem typeSystem;
     /*
      * A Store for each Class and Trait.
@@ -63,23 +58,6 @@ public class MemRepository implements IRepository {
         this.typeStores = new HashMap<>();
     }
 
-/*
-    @Override
-    public DateFormat getDateFormat() {
-        return dateFormat;
-    }
-
-    @Override
-    public DateFormat getTimestampFormat() {
-        return timestampFormat;
-    }
-
-    @Override
-    public boolean allowNullsInCollections() {
-        return false;
-    }
-*/
-
     @Override
     public Id newId(String typeName) {
         return new Id("" + ID_SEQ.incrementAndGet(), 0, typeName);
@@ -306,9 +284,8 @@ public class MemRepository implements IRepository {
                 cTypes.add((ClassType) h);
             }
         }
-
-        Collections.sort(tTypes);
-        Collections.sort(cTypes);
+        tTypes = HierarchicalTypeDependencySorter.sortTypes(tTypes);
+        cTypes = HierarchicalTypeDependencySorter.sortTypes(cTypes);
 
         for (TraitType tT : tTypes) {
             defineTrait(tT);

typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java

@@ -46,8 +46,7 @@ import java.util.Set;
  *            as SuperTypes.
  * @param <T> the class of the Instance of this DataType.
  */
-public abstract class HierarchicalType<ST extends HierarchicalType, T> extends AbstractDataType<T>
-        implements Comparable<ST> {
+public abstract class HierarchicalType<ST extends HierarchicalType, T> extends AbstractDataType<T> {
 
     public final TypeSystem typeSystem;
     public final Class<ST> superTypeClass;
@@ -367,19 +366,10 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
     }
 
     @Override
-    public int compareTo(ST o) {
-        String oName = o.getName();
-        try {
-            if (o.isSubType(getName())) {
-                return 1;
-            } else if (isSubType(oName)) {
-                return -1;
-            } else {
-                return getName().compareTo(oName);
-            }
-        } catch(AtlasException e) {
-            throw new RuntimeException(e);
-        }
+    public String toString() {
+
+        return "[name=" + name + ", description=" + description + 
+            ", superTypes=" + superTypes + ", immediateAttrs=" + immediateAttrs + "]";
     }
 
     public Set<String> getAllSuperTypeNames() {

typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorter.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.typesystem.types;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+
+/**
+ * Sorts hierarchical types by supertype dependency
+ */
+public class HierarchicalTypeDependencySorter {
+    
+    /**
+     * Sorts the specified hierarchical types by supertype dependencies, 
+     * such that any type A which is a supertype of type B
+     * will always be located earlier in the result list; that is, the supertype 
+     * A would be found at some index i and subtype B would be found at some index j,
+     * and i < j.
+     * 
+     * @param types  hierarchical types to be sorted
+     * @return hierarchical types sorted by supertype dependency
+     */
+    public static <T extends HierarchicalType> List<T> sortTypes(List<T> types) {
+        Map<String, T> typesByName = new HashMap<>();
+        for (T type : types) {
+            typesByName.put(type.name, type);
+        }
+        List<T> result = new ArrayList<>(types.size());
+        Set<T> processed = new HashSet<>();
+        for (T type : types) {
+            addToResult(type, result, processed, typesByName);
+        }
+        return result;
+    }
+    
+    private static <T extends HierarchicalType> void addToResult(T type, List<T> result, 
+        Set<T> processed, Map<String, T> typesByName) {
+        
+        if (processed.contains(type)) {
+            return;
+        }
+        processed.add(type);
+        ImmutableSet<String> superTypeNames = type.superTypes;
+        for (String superTypeName : superTypeNames) {
+            // Recursively add any supertypes first to the result.
+            T superType = typesByName.get(superTypeName);
+            if (superType != null) {
+                addToResult(superType, result, processed, typesByName);
+            }
+        }
+        result.add(type);
+    }
+}

typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java

@@ -34,8 +34,6 @@ import javax.inject.Singleton;
 import java.lang.reflect.Constructor;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -542,13 +540,13 @@ public class TypeSystem {
             for (String traitTypeName : traitNameToDefMap.keySet()) {
                 traitTypes.add(getDataType(TraitType.class, traitTypeName));
             }
-            Collections.sort(traitTypes);
+            traitTypes = HierarchicalTypeDependencySorter.sortTypes(traitTypes);
 
             List<ClassType> classTypes = new ArrayList<>();
             for (String classTypeName : classNameToDefMap.keySet()) {
                 classTypes.add(getDataType(ClassType.class, classTypeName));
             }
-            Collections.sort(classTypes);
+            classTypes = HierarchicalTypeDependencySorter.sortTypes(classTypes);
 
             for (StructTypeDefinition structDef : structDefs) {
                 constructStructureType(structDef);

typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorterTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.typesystem.types;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.atlas.AtlasException;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+
+public class HierarchicalTypeDependencySorterTest {
+
+    @BeforeMethod
+    public void setup() throws Exception {
+        TypeSystem ts = TypeSystem.getInstance();
+        ts.reset();
+    }
+    
+    @SuppressWarnings("rawtypes")
+    @Test
+    public void testSimpleModel() throws AtlasException {
+        TypeSystem ts = TypeSystem.getInstance();
+        HierarchicalType a = new ClassType(ts, "a", null, ImmutableSet.<String>of(), 0);
+        HierarchicalType b = new ClassType(ts, "B", null, ImmutableSet.of("a"), 0);
+        HierarchicalType c = new ClassType(ts, "C", null, ImmutableSet.of("B"), 0);
+
+        List<HierarchicalType> unsortedTypes = Arrays.asList(c, a, b);
+        List<HierarchicalType> sortedTypes = HierarchicalTypeDependencySorter.sortTypes(unsortedTypes);
+        Assert.assertEquals(sortedTypes.size(), 3);
+        Assert.assertEquals(sortedTypes.indexOf(a), 0);
+        Assert.assertEquals(sortedTypes.indexOf(b), 1);
+        Assert.assertEquals(sortedTypes.indexOf(c), 2);
+    }
+
+    @SuppressWarnings("rawtypes")
+    @Test
+    public void testLargerModel() throws Exception {
+        TypeSystem ts = TypeSystem.getInstance();
+        HierarchicalType testObjectType = new ClassType(ts, "TestObject", null, ImmutableSet.<String>of(), 0);
+        HierarchicalType testDataSetType = new ClassType(ts, "TestDataSet", null, ImmutableSet.of("TestObject"), 0);
+        HierarchicalType testColumnType = new ClassType(ts, "TestColumn", null, ImmutableSet.of("TestObject"), 0);
+        HierarchicalType testRelationalDataSetType = new ClassType(ts, "TestRelationalDataSet", null, ImmutableSet.of("TestDataSet"), 0);
+        HierarchicalType testTableType = new ClassType(ts, "Table", null, ImmutableSet.of("TestDataSet"), 0);
+        HierarchicalType testDataFileType = new ClassType(ts, "TestDataFile", null, ImmutableSet.of("TestRelationalDataSet"), 0);
+        HierarchicalType testDocumentType = new ClassType(ts, "TestDocument", null, ImmutableSet.of("TestDataSet"), 0);
+        HierarchicalType testAnnotationType = new ClassType(ts, "TestAnnotation", null, ImmutableSet.<String>of(), 0);
+        HierarchicalType myNewAnnotationType = new ClassType(ts, "MyNewAnnotation", null, ImmutableSet.of("TestAnnotation"), 0);
+
+        List<HierarchicalType> unsortedTypes = Arrays.asList(testTableType, testColumnType, myNewAnnotationType, testDataSetType,
+            testDataFileType, testAnnotationType, testRelationalDataSetType, testObjectType, testDocumentType);
+        List<HierarchicalType> sortedTypes = HierarchicalTypeDependencySorter.sortTypes(unsortedTypes);
+        // Verify that super types were sorted before their subtypes.
+        Assert.assertTrue(sortedTypes.indexOf(testObjectType) < sortedTypes.indexOf(testDataSetType));
+        Assert.assertTrue(sortedTypes.indexOf(testObjectType) < sortedTypes.indexOf(testColumnType));
+        Assert.assertTrue(sortedTypes.indexOf(testDataSetType) < sortedTypes.indexOf(testRelationalDataSetType));
+        Assert.assertTrue(sortedTypes.indexOf(testDataSetType) < sortedTypes.indexOf(testDocumentType));
+        Assert.assertTrue(sortedTypes.indexOf(testDataSetType) < sortedTypes.indexOf(testTableType));
+        Assert.assertTrue(sortedTypes.indexOf(testRelationalDataSetType) < sortedTypes.indexOf(testDataFileType));
+        Assert.assertTrue(sortedTypes.indexOf(testAnnotationType) < sortedTypes.indexOf(myNewAnnotationType));
+    }
+}

typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java

@@ -36,6 +36,7 @@ import java.util.HashMap;
 import java.util.List;
 
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
@@ -171,25 +172,6 @@ public class TypeSystemTest extends BaseTest {
     }
 
     @Test
-    public void testHierarchy() throws AtlasException {
-        HierarchicalTypeDefinition<ClassType> a = TypesUtil.createClassTypeDef("a", ImmutableSet.<String>of());
-        HierarchicalTypeDefinition<ClassType> b = TypesUtil.createClassTypeDef("B", ImmutableSet.of("a"));
-        HierarchicalTypeDefinition<ClassType> c = TypesUtil.createClassTypeDef("C", ImmutableSet.of("B"));
-
-        TypeSystem ts = getTypeSystem();
-        ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
-                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
-                ImmutableList.of(a, b, c));
-        ClassType ac = ts.getDataType(ClassType.class, "a");
-        ClassType bc = ts.getDataType(ClassType.class, "B");
-        ClassType cc = ts.getDataType(ClassType.class, "C");
-
-        assertTrue(ac.compareTo(bc) < 0);
-        assertTrue(bc.compareTo(cc) < 0);
-        assertTrue(ac.compareTo(cc) < 0);
-    }
-
-    @Test
     public void testTypeCategory() throws AtlasException {
         TypeSystem ts = getTypeSystem();
         ts.reset();
@@ -251,5 +233,37 @@ public class TypeSystemTest extends BaseTest {
         Assert.assertEquals(traitNames.size(), numTraits+2);
     }
 
+    @Test
+    public void testHierarchy() throws Exception {
+        HierarchicalTypeDefinition<ClassType> testObjectDef = TypesUtil.createClassTypeDef("TestObject", ImmutableSet.<String>of(),
+            createOptionalAttrDef("name", DataTypes.STRING_TYPE),
+            createOptionalAttrDef("description", DataTypes.STRING_TYPE),
+            createOptionalAttrDef("topAttribute", DataTypes.STRING_TYPE));
+        HierarchicalTypeDefinition<ClassType> testDataSetDef = TypesUtil.createClassTypeDef("TestDataSet", ImmutableSet.of("TestObject"));
+        HierarchicalTypeDefinition<ClassType> testColumnDef = TypesUtil.createClassTypeDef("TestColumn", ImmutableSet.of("TestObject"),
+            createRequiredAttrDef("name", DataTypes.STRING_TYPE));
+        HierarchicalTypeDefinition<ClassType> testRelationalDataSetDef = 
+            TypesUtil.createClassTypeDef("TestRelationalDataSet", ImmutableSet.of("TestDataSet"),
+                new AttributeDefinition("columns", DataTypes.arrayTypeName("TestColumn"),
+                    Multiplicity.OPTIONAL, true, null));
+        HierarchicalTypeDefinition<ClassType> testTableDef = TypesUtil.createClassTypeDef("TestTable", ImmutableSet.of("TestRelationalDataSet"),
+            createOptionalAttrDef("schema", DataTypes.STRING_TYPE));
+        HierarchicalTypeDefinition<ClassType> testDataFileDef = TypesUtil.createClassTypeDef("TestDataFile", ImmutableSet.of("TestRelationalDataSet"),
+            createOptionalAttrDef("urlString", DataTypes.STRING_TYPE));
+        HierarchicalTypeDefinition<ClassType> testDocumentDef = TypesUtil.createClassTypeDef("TestDocument", ImmutableSet.of("TestDataSet"),
+            createOptionalAttrDef("urlString", DataTypes.STRING_TYPE),
+            createOptionalAttrDef("encoding", DataTypes.STRING_TYPE));
+        HierarchicalTypeDefinition<ClassType> testAnnotationDef =TypesUtil.createClassTypeDef("TestAnnotation",  ImmutableSet.<String>of(),
+            createOptionalAttrDef("inheritedAttribute", DataTypes.STRING_TYPE));
+        HierarchicalTypeDefinition<ClassType> myNewAnnotationDef = TypesUtil.createClassTypeDef("MyNewAnnotation", ImmutableSet.of("TestAnnotation"),
+            createRequiredAttrDef("myNewAnnotationAttribute", DataTypes.STRING_TYPE));
+        getTypeSystem().defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+            ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+            ImmutableList.of(testObjectDef, testDataSetDef, testColumnDef, testRelationalDataSetDef, testTableDef, testDataFileDef, testDocumentDef, testAnnotationDef, myNewAnnotationDef));
 
+        // Verify that field mappings for MyNewAnnotation contains the attribute inherited from the TestAnnotation superclass.
+        // Prior to fix for ATLAS-573, the inherited attribute was missing.
+        ClassType dataType = getTypeSystem().getDataType(ClassType.class, "MyNewAnnotation");
+        Assert.assertTrue(dataType.fieldMapping.fields.containsKey("inheritedAttribute"));
+    }
 }