ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via shwethags)

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via shwethags)
 ATLAS-1199 Atlas UI not loading after fresh build due to jquery-asBreadcrumbs plugin upgrade (kevalbhatt via shwethags)
 ATLAS-1174 Framework to apply updates to types in the type-system (sarath.kum4r@gmail.com via shwethags)
 ATLAS-1155 Errors in Eclipse when I bring in the latest code (davidrad via shwethags)

repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java

@@ -18,12 +18,12 @@
 
 package org.apache.atlas.services;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.*;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,8 +31,10 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.List;
 
 public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar {
 
@@ -81,13 +83,46 @@ public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar {
             LOG.error("Error while deserializing JSON in {}", typeDefName);
             throw new ReservedTypesRegistrationException("Error while deserializing JSON in " + typeDefName, e);
         }
-        HierarchicalTypeDefinition<ClassType> classDef = typesDef.classTypesAsJavaList().get(0);
-        if (!typeSystem.isRegistered(classDef.typeName)) {
-            metadataService.createType(typeDefJSON);
-            LOG.info("Registered types in {}", typeDefName);
-        } else {
-            LOG.warn("class {} already registered, ignoring types in {}", classDef.typeName,
-                    typeDefName);
+        List<HierarchicalTypeDefinition<ClassType>> createClassDefList = new ArrayList<>();
+        List<HierarchicalTypeDefinition<TraitType>> createTraitDefList = new ArrayList<>();
+        List<EnumTypeDefinition> createEnumDefList = new ArrayList<>();
+        List<StructTypeDefinition> createStructDefList = new ArrayList<>();
+
+        for(HierarchicalTypeDefinition<ClassType> classTypeDef:typesDef.classTypesAsJavaList()){
+            if(!typeSystem.isRegistered(classTypeDef.typeName)){
+                LOG.debug("ClassType {} is not registered. Adding to create type list", classTypeDef.typeName);
+                createClassDefList.add(classTypeDef);
+            }
+        }
+
+        for(HierarchicalTypeDefinition<TraitType> traitTypeDef:typesDef.traitTypesAsJavaList()){
+            if(!typeSystem.isRegistered(traitTypeDef.typeName)){
+                LOG.debug("TraitType {} is not registered. Adding to create type list", traitTypeDef.typeName);
+                createTraitDefList.add(traitTypeDef);
+            }
+        }
+
+        for(StructTypeDefinition structTypeDef:typesDef.structTypesAsJavaList()){
+            if(!typeSystem.isRegistered(structTypeDef.typeName)){
+                LOG.debug("StructType {} is not registered. Adding to create type list", structTypeDef.typeName);
+                createStructDefList.add(structTypeDef);
+            }
+        }
+
+        for(EnumTypeDefinition enumTypeDef:typesDef.enumTypesAsJavaList()){
+            if(!typeSystem.isRegistered(enumTypeDef.name)){
+                LOG.debug("EnumType {} is not registered. Adding to create type list", enumTypeDef.name);
+                createEnumDefList.add(enumTypeDef);
+            }
+        }
+
+        TypesDef createTypes = TypesUtil.getTypesDef(ImmutableList.copyOf(createEnumDefList), ImmutableList.copyOf(createStructDefList),
+                ImmutableList.copyOf(createTraitDefList), ImmutableList.copyOf(createClassDefList));
+
+        String createTypeJSON = TypesSerialization.toJson(createTypes);
+        if(createTypeJSON != null) {
+            metadataService.createType(createTypeJSON);
+            LOG.info("Created types definition JSON {}", createTypeJSON);
         }
     }
 }

repository/src/test/java/org/apache/atlas/TestUtils.java

@@ -239,6 +239,55 @@ public final class TestUtils {
 
     public static final String NAME = "name";
 
+    public static TypesDef simpleType(){
+        HierarchicalTypeDefinition<ClassType> superTypeDefinition =
+                createClassTypeDef("h_type", ImmutableSet.<String>of(),
+                        createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+        StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
+                new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)});
+
+        HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
+                createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
+
+        EnumValue values[] = {new EnumValue("ONE", 1),};
+
+        EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values);
+        return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
+                ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition));
+    }
+
+    public static TypesDef simpleTypeUpdated(){
+        HierarchicalTypeDefinition<ClassType> superTypeDefinition =
+                createClassTypeDef("h_type", ImmutableSet.<String>of(),
+                        createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+        HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
+                createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
+                        createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+        StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
+                new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)});
+
+        HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
+                createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
+
+        EnumValue values[] = {new EnumValue("ONE", 1),};
+
+        EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values);
+        return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
+                ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition, newSuperTypeDefinition));
+    }
+
+    public static TypesDef simpleTypeUpdatedDiff() {
+        HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
+                createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
+                        createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+        return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(newSuperTypeDefinition));
+    }
+
     public static TypesDef defineHiveTypes() {
         String _description = "_description";
         HierarchicalTypeDefinition<ClassType> superTypeDefinition =

repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java

@@ -23,6 +23,8 @@ import org.apache.atlas.TestUtils;
 import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.json.TypesSerialization;
 import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.mockito.InOrder;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
@@ -55,17 +57,6 @@ public class ReservedTypesRegistrarTest {
     }
 
     @Test
-    public void testRegisterFirstChecksClassTypeIsRegistered() throws AtlasException {
-        ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
-        TypesDef typesDef = TestUtils.defineHiveTypes();
-        String typesJson = TypesSerialization.toJson(typesDef);
-        reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
-        InOrder inOrder = inOrder(typeSystem, metadataService);
-        inOrder.verify(typeSystem).isRegistered(typesDef.classTypesAsJavaList().get(0).typeName);
-        inOrder.verify(metadataService).createType(typesJson);
-    }
-
-    @Test
     public void testRegisterCreatesTypesUsingMetadataService() throws AtlasException {
         ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
         TypesDef typesDef = TestUtils.defineHiveTypes();
@@ -90,12 +81,23 @@ public class ReservedTypesRegistrarTest {
     }
 
     @Test
-    public void testShouldNotRegisterIfTypeIsAlreadyRegistered() throws AtlasException {
+    public void testCreateAndUpdateType() throws AtlasException{
         ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
-        TypesDef typesDef = TestUtils.defineHiveTypes();
+        TypesDef typesDef = TestUtils.simpleType();
         String typesJson = TypesSerialization.toJson(typesDef);
-        when(typeSystem.isRegistered(typesDef.classTypesAsJavaList().get(0).typeName)).thenReturn(true);
         reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
-        verifyZeroInteractions(metadataService);
+        verify(metadataService).createType(typesJson);
+
+        //test update simple type
+        TypesDef updatedTypesDef = TestUtils.simpleTypeUpdated();
+        String updatedTypesJson = TypesSerialization.toJson(updatedTypesDef);
+        TypesDef simpleTypeUpdatedDiff = TestUtils.simpleTypeUpdatedDiff();
+        String simpleTypeUpdatedDiffJson = TypesSerialization.toJson(simpleTypeUpdatedDiff);
+        when(typeSystem.isRegistered("h_type")).thenReturn(true);
+        when(typeSystem.isRegistered("t_type")).thenReturn(true);
+        when(typeSystem.isRegistered("s_type")).thenReturn(true);
+        when(typeSystem.isRegistered("e_type")).thenReturn(true);
+        reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", updatedTypesJson);
+        verify(metadataService).createType(simpleTypeUpdatedDiffJson);
     }
 }