ATLAS-1734: Import API update to make type import optional

intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java

@@ -44,6 +44,7 @@ public class AtlasImportRequest implements Serializable {
     public  static final String TRANSFORMS_KEY   = "transforms";
     private static final String START_POSITION_KEY = "startPosition";
     private static final String START_GUID_KEY = "startGuid";
+    private static final String UPDATE_TYPE_DEFINITION_KEY = "updateTypeDefinition";
 
     private Map<String, String> options;
 
@@ -76,19 +77,24 @@ public class AtlasImportRequest implements Serializable {
 
     @JsonIgnore
     public String getStartGuid() {
-        if (this.options == null || !this.options.containsKey(START_GUID_KEY)) {
-            return null;
-        }
-
-        return (String) this.options.get(START_GUID_KEY);
+        return getOptionForKey(START_GUID_KEY);
     }
 
     @JsonIgnore
     public String getStartPosition() {
-        if (this.options == null || !this.options.containsKey(START_POSITION_KEY)) {
+        return getOptionForKey(START_POSITION_KEY);
+    }
+
+    @JsonIgnore
+    public String getUpdateTypeDefs() {
+        return getOptionForKey(UPDATE_TYPE_DEFINITION_KEY);
+    }
+
+    private String getOptionForKey(String key) {
+        if (this.options == null || !this.options.containsKey(key)) {
             return null;
         }
 
-        return (String) this.options.get(START_GUID_KEY);
+        return (String) this.options.get(key);
     }
- }
+}

repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java

@@ -603,7 +603,7 @@ public class ExportService {
         }
     }
 
-    private class UniqueList<T> {
+    public static class UniqueList<T> {
         private final List<T>   list = new ArrayList<>();
         private final Set<T>    set = new HashSet<>();
 

repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java

@@ -21,12 +21,7 @@ import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.impexp.AtlasImportRequest;
 import org.apache.atlas.model.impexp.AtlasImportResult;
-import org.apache.atlas.model.typedef.AtlasClassificationDef;
-import org.apache.atlas.model.typedef.AtlasEntityDef;
-import org.apache.atlas.model.typedef.AtlasEnumDef;
-import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
-import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
 import org.apache.atlas.repository.store.graph.AtlasEntityStore;
 import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasTypeRegistry;
@@ -51,7 +46,6 @@ public class ImportService {
     private long startTimestamp;
     private long endTimestamp;
 
-
     public ImportService(final AtlasTypeDefStore typeDefStore, final AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
         this.typeDefStore = typeDefStore;
         this.entityStore = entityStore;
@@ -139,40 +133,12 @@ public class ImportService {
     }
 
     private void processTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
-        setGuidToEmpty(typeDefinitionMap);
-
-        AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typeDefinitionMap, this.typeRegistry);
-
-        if (!typesToCreate.isEmpty()) {
-            typeDefStore.createTypesDef(typesToCreate);
-
-            updateMetricsForTypesDef(typesToCreate, result);
-        }
-    }
-
-    private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) {
-        result.incrementMeticsCounter("typedef:classification", typeDefinitionMap.getClassificationDefs().size());
-        result.incrementMeticsCounter("typedef:enum", typeDefinitionMap.getEnumDefs().size());
-        result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size());
-        result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size());
-    }
-
-    private void setGuidToEmpty(AtlasTypesDef typesDef) {
-        for (AtlasEntityDef def : typesDef.getEntityDefs()) {
-            def.setGuid(null);
-        }
-
-        for (AtlasClassificationDef def : typesDef.getClassificationDefs()) {
-            def.setGuid(null);
+        if(result.getRequest().getUpdateTypeDefs() != null && !result.getRequest().getUpdateTypeDefs().equals("true")) {
+            return;
         }
 
-        for (AtlasEnumDef def : typesDef.getEnumDefs()) {
-            def.setGuid(null);
-        }
-
-        for (AtlasStructDef def : typesDef.getStructDefs()) {
-            def.setGuid(null);
-        }
+        ImportTypeDefProcessor importTypeDefProcessor = new ImportTypeDefProcessor(this.typeDefStore, this.typeRegistry);
+        importTypeDefProcessor.processTypes(typeDefinitionMap, result);
     }
 
     private void processEntities(ZipSource importSource, AtlasImportResult result) throws AtlasBaseException {

repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+
+public class ImportTypeDefProcessor {
+    private final AtlasTypeDefStore typeDefStore;
+    private final AtlasTypeRegistry typeRegistry;
+    private TypeAttributeDifference typeAttributeDifference;
+
+    public ImportTypeDefProcessor(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
+        this.typeDefStore = typeDefStore;
+        this.typeRegistry = typeRegistry;
+        this.typeAttributeDifference = new TypeAttributeDifference(typeDefStore, typeRegistry);
+    }
+
+    public void processTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
+        setGuidToEmpty(typeDefinitionMap);
+
+        typeAttributeDifference.updateTypes(typeDefinitionMap, result);
+
+        AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typeDefinitionMap, this.typeRegistry);
+        if (!typesToCreate.isEmpty()) {
+            typeDefStore.createTypesDef(typesToCreate);
+            updateMetricsForTypesDef(typesToCreate, result);
+
+        }
+    }
+
+    private void setGuidToEmpty(AtlasTypesDef typesDef) {
+        for (AtlasEntityDef def : typesDef.getEntityDefs()) {
+            def.setGuid(null);
+        }
+
+        for (AtlasClassificationDef def : typesDef.getClassificationDefs()) {
+            def.setGuid(null);
+        }
+
+        for (AtlasEnumDef def : typesDef.getEnumDefs()) {
+            def.setGuid(null);
+        }
+
+        for (AtlasStructDef def : typesDef.getStructDefs()) {
+            def.setGuid(null);
+        }
+    }
+
+    private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) {
+        result.incrementMeticsCounter("typedef:classification", typeDefinitionMap.getClassificationDefs().size());
+        result.incrementMeticsCounter("typedef:enum", typeDefinitionMap.getEnumDefs().size());
+        result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size());
+        result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size());
+    }
+}

repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TypeAttributeDifference {
+    private final AtlasTypeDefStore typeDefStore;
+    private final AtlasTypeRegistry typeRegistry;
+
+    public TypeAttributeDifference(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
+        this.typeDefStore = typeDefStore;
+        this.typeRegistry = typeRegistry;
+    }
+
+    public void updateTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
+
+        updateEntityDef(typeDefinitionMap, result);
+        updateClassificationDef(typeDefinitionMap, result);
+        updateEnumDef(typeDefinitionMap, result);
+        updateStructDef(typeDefinitionMap, result);
+    }
+
+    private void updateEntityDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
+        for (AtlasEntityDef def: typeDefinitionMap.getEntityDefs()) {
+            AtlasEntityDef existing = typeRegistry.getEntityDefByName(def.getName());
+            if(existing != null && addAttributes(existing, def)) {
+                typeDefStore.updateEntityDefByName(existing.getName(), existing);
+                result.incrementMeticsCounter("typedef:entitydef:update");
+            }
+        }
+    }
+
+    private void updateClassificationDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
+        for (AtlasClassificationDef def: typeDefinitionMap.getClassificationDefs()) {
+            AtlasClassificationDef existing = typeRegistry.getClassificationDefByName(def.getName());
+            if(existing != null && addAttributes(existing, def)) {
+                typeDefStore.updateClassificationDefByName(existing.getName(), existing);
+                result.incrementMeticsCounter("typedef:classification:update");
+            }
+        }
+    }
+
+    private void updateEnumDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
+        for (AtlasEnumDef def: typeDefinitionMap.getEnumDefs()) {
+            AtlasEnumDef existing = typeRegistry.getEnumDefByName(def.getName());
+            if(existing != null && addElements(existing, def)) {
+                typeDefStore.updateEnumDefByName(existing.getName(), existing);
+                result.incrementMeticsCounter("typedef:enum:update");
+            }
+        }
+    }
+
+    private void updateStructDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
+        for (AtlasStructDef def: typeDefinitionMap.getStructDefs()) {
+            AtlasStructDef existing = typeRegistry.getStructDefByName(def.getName());
+            if(existing != null && addAttributes(existing, def)) {
+                typeDefStore.updateStructDefByName(existing.getName(), existing);
+                result.incrementMeticsCounter("typedef:struct:update");
+            }
+        }
+    }
+
+    private boolean addElements(AtlasEnumDef existing, AtlasEnumDef incoming) {
+        return addElements(existing, getElementsAbsentInExisting(existing, incoming));
+    }
+
+    private boolean addAttributes(AtlasStructDef existing, AtlasStructDef incoming) {
+        return addAttributes(existing, getElementsAbsentInExisting(existing, incoming));
+    }
+
+    private List<AtlasStructDef.AtlasAttributeDef> getElementsAbsentInExisting(AtlasStructDef existing, AtlasStructDef incoming) {
+        List<AtlasStructDef.AtlasAttributeDef> difference = new ArrayList<>();
+        for (AtlasStructDef.AtlasAttributeDef attr : incoming.getAttributeDefs()) {
+            if(existing.getAttribute(attr.getName()) == null) {
+                difference.add(attr);
+            }
+        }
+
+        return difference;
+    }
+
+    private List<AtlasEnumDef.AtlasEnumElementDef> getElementsAbsentInExisting(AtlasEnumDef existing, AtlasEnumDef incoming) {
+        List<AtlasEnumDef.AtlasEnumElementDef> difference = new ArrayList<>();
+        for (AtlasEnumDef.AtlasEnumElementDef ed : incoming.getElementDefs()) {
+            if(existing.getElement(ed.getValue()) == null) {
+                difference.add(ed);
+            }
+        }
+
+        return difference;
+    }
+
+    private boolean addAttributes(AtlasStructDef def, List<AtlasStructDef.AtlasAttributeDef> list) {
+        for (AtlasStructDef.AtlasAttributeDef ad : list) {
+            def.addAttribute(ad);
+        }
+
+        return list.size() > 0;
+    }
+
+    private boolean addElements(AtlasEnumDef def, List<AtlasEnumDef.AtlasEnumElementDef> list) {
+        for (AtlasEnumDef.AtlasEnumElementDef ad : list) {
+            def.addElement(ad);
+        }
+
+        return list.size() > 0;
+    }
+}

repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java

@@ -40,6 +40,47 @@ public class AtlasImportRequestTest {
     }
 
     @Test
+    public void serializeOptions_VerifyAccessors() {
+        String guid = "\"abcd\"";
+        String pos = "\"1\"";
+        String trueVal = "\"true\"";
+
+        String jsonData = "{ \"options\": " +
+                "               {" +
+                "\"startGuid\":" + guid + "," +
+                "\"startPosition\":" + pos + "," +
+                "\"updateTypeDefinition\":" + trueVal +
+                "}" +
+                "}";
+
+        AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
+
+        assertNotNull(request);
+        assertNotNull(request.getStartGuid());
+        assertNotNull(request.getStartPosition());
+        assertNotNull(request.getUpdateTypeDefs());
+
+        assertEquals(request.getStartGuid(), guid.replace("\"", ""));
+        assertEquals(request.getStartPosition(), pos.replace("\"", ""));
+        assertEquals(request.getUpdateTypeDefs(), trueVal.replace("\"", ""));
+    }
+
+    @Test
+    public void optionsDefaultsTest() {
+        String jsonData = "{ \"options\": " +
+                "               {" +
+                    "}" +
+                "}";
+
+        AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
+
+        assertNotNull(request);
+        assertNull(request.getStartGuid());
+        assertNull(request.getStartPosition());
+        assertNull(request.getUpdateTypeDefs());
+    }
+
+    @Test
     public void serializeAtlasImportRequstFromJsonWithEmptyTransforms() {
         String jsonData = "{ \"options\": { \"transforms\": \"{ }\" } }";
 

repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java

@@ -20,6 +20,7 @@ package org.apache.atlas.repository.impexp;
 import com.google.inject.Inject;
 import org.apache.atlas.TestModules;
 import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
 import org.apache.atlas.repository.store.graph.AtlasEntityStore;
 import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasTypeRegistry;
@@ -31,10 +32,12 @@ import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
-import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource;
-import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
-import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runAndVerifyQuickStart_v1_Import;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
 
 @Guice(modules = TestModules.TestOnlyModule.class)
 public class ImportServiceTest {
@@ -55,7 +58,7 @@ public class ImportServiceTest {
     }
 
     @Test(dataProvider = "sales")
-    public void importDB1_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException {
+    public void importDB1(ZipSource zipSource) throws AtlasBaseException, IOException {
         loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
         runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
     }
@@ -66,7 +69,7 @@ public class ImportServiceTest {
     }
 
     @Test(dataProvider = "reporting")
-    public void importDB2_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException {
+    public void importDB2(ZipSource zipSource) throws AtlasBaseException, IOException {
         loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
         runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
     }
@@ -77,8 +80,57 @@ public class ImportServiceTest {
     }
 
     @Test(dataProvider = "logging")
-    public void importDB3_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException {
+    public void importDB3(ZipSource zipSource) throws AtlasBaseException, IOException {
         loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
         runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
     }
+
+    @DataProvider(name = "salesNewTypeAttrs")
+    public static Object[][] getDataFromSalesNewTypeAttrs(ITestContext context) throws IOException {
+        return getZipSource("salesNewTypeAttrs.zip");
+    }
+
+    @Test(dataProvider = "salesNewTypeAttrs", dependsOnMethods = "importDB1")
+    public void importDB4(ZipSource zipSource) throws AtlasBaseException, IOException {
+        loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
+        runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), getDefaultImportRequest(), zipSource);
+    }
+
+    @DataProvider(name = "salesNewTypeAttrs-next")
+    public static Object[][] getDataFromSalesNewTypeAttrsNext(ITestContext context) throws IOException {
+        return getZipSource("salesNewTypeAttrs-next.zip");
+    }
+
+
+    @Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4")
+    public void importDB5(ZipSource zipSource) throws AtlasBaseException, IOException {
+        final String newEnumDefName = "database_action";
+
+        assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
+
+        AtlasImportRequest request = getDefaultImportRequest();
+        Map<String, String> options = new HashMap<>();
+        options.put("updateTypeDefinition", "false");
+        request.setOptions(options);
+
+        runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), request, zipSource);
+        assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
+        assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 4);
+    }
+
+    @Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4")
+    public void importDB6(ZipSource zipSource) throws AtlasBaseException, IOException {
+        final String newEnumDefName = "database_action";
+
+        assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
+
+        AtlasImportRequest request = getDefaultImportRequest();
+        Map<String, String> options = new HashMap<>();
+        options.put("updateTypeDefinition", "true");
+        request.setOptions(options);
+
+        runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), request, zipSource);
+        assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
+        assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 8);
+    }
 }

repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import com.google.common.collect.Sets;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.testng.Assert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ImportServiceTestUtils {
+
+    public static void verifyImportedEntities(List<String> creationOrder, List<String> processedEntities) {
+        Set<String> lhs = com.google.common.collect.Sets.newHashSet(creationOrder);
+        Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities);
+        Set<String> difference = Sets.difference(lhs, rhs);
+
+        Assert.assertNotNull(difference);
+        Assert.assertEquals(difference.size(), 0);
+    }
+
+    public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) {
+        Map<String, Integer> metricsForCompare = getImportMetricsForCompare(importResult);
+        for (Map.Entry<String, Integer> entry : exportResult.getMetrics().entrySet()) {
+            if(entry.getKey().startsWith("entity") == false ||
+                    entry.getKey().contains("withExtInfo") ||
+                    entry.getKey().contains("Column") ||
+                    entry.getKey().contains("StorageDesc")) continue;
+
+            Assert.assertTrue(metricsForCompare.containsKey(entry.getKey()));
+            Assert.assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey()));
+        }
+    }
+
+    private static Map<String,Integer> getImportMetricsForCompare(AtlasImportResult result) {
+        Map<String, Integer> r = new HashMap<>();
+        for (Map.Entry<String, Integer> entry : result.getMetrics().entrySet()) {
+            r.put(entry.getKey().replace(":updated", "").replace(":created", ""), entry.getValue());
+        }
+
+        return r;
+    }
+
+
+    public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
+        AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName);
+        createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
+    }
+
+    private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+        AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesFromJson, typeRegistry);
+
+        if (!typesToCreate.isEmpty()) {
+            typeDefStore.createTypesDef(typesToCreate);
+        }
+    }
+
+    private static AtlasTypesDef getAtlasTypesDefFromFile(String fileName) throws IOException {
+        String sampleTypes = ZipFileResourceTestUtils.getModelJson(fileName);
+        return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
+    }
+
+    public static AtlasImportRequest getDefaultImportRequest() {
+        return new AtlasImportRequest();
+    }
+
+
+    public static AtlasImportResult runImportWithParameters(ImportService importService, AtlasImportRequest request, ZipSource source) throws AtlasBaseException, IOException {
+        final String requestingIP = "1.0.0.0";
+        final String hostName = "localhost";
+        final String userName = "admin";
+
+        AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP);
+        Assert.assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS);
+        return result;
+    }
+
+    public static void runAndVerifyQuickStart_v1_Import(ImportService importService, ZipSource zipSource) throws AtlasBaseException, IOException {
+        AtlasExportResult exportResult = zipSource.getExportResult();
+        List<String> creationOrder = zipSource.getCreationOrder();
+
+        AtlasImportRequest request = getDefaultImportRequest();
+        AtlasImportResult result = runImportWithParameters(importService, request, zipSource);
+
+        Assert.assertNotNull(result);
+        verifyImportedMetrics(exportResult, result);
+        verifyImportedEntities(creationOrder, result.getProcessedEntities());
+    }
+}

repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.powermock.reflect.Whitebox;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TypeAttributeDifferenceTest {
+    private TypeAttributeDifference typeAttributeDifference;
+
+    @BeforeClass
+    public void setup() {
+        typeAttributeDifference = new TypeAttributeDifference(null, null);
+    }
+
+    private List<AtlasEnumDef.AtlasEnumElementDef> getEnumElementDefs(int startIndex, String... names) {
+        int i = startIndex;
+        List<AtlasEnumDef.AtlasEnumElementDef> list = new ArrayList<>();
+        for (String s: names) {
+            list.add(new AtlasEnumDef.AtlasEnumElementDef(s, s, i++));
+        }
+
+        return list;
+    }
+
+    private List<AtlasStructDef.AtlasAttributeDef> getAtlasAttributeDefs(String... names) {
+        List<AtlasStructDef.AtlasAttributeDef> list = new ArrayList<>();
+        for (String s : names) {
+            list.add(new AtlasStructDef.AtlasAttributeDef(s, AtlasBaseTypeDef.ATLAS_TYPE_STRING));
+        }
+
+        return list;
+    }
+
+    private AtlasEntityDef getAtlasEntityDefWithAttributes(String... attributeNames) {
+        AtlasEntityDef e = new AtlasEntityDef();
+        for (AtlasStructDef.AtlasAttributeDef a : getAtlasAttributeDefs(attributeNames)) {
+            e.addAttribute(a);
+        }
+
+        return e;
+    }
+
+    @Test
+    public void entityDefWithNoAttributes() throws Exception {
+        AtlasEntityDef existing = new AtlasEntityDef();
+        AtlasEntityDef incoming = new AtlasEntityDef();
+        List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = new ArrayList<>();
+        List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+        Assert.assertEquals(actualAttributes, expectedAttributes);
+    }
+
+    private List<AtlasStructDef.AtlasAttributeDef> invokeGetAttributesAbsentInExisting(AtlasStructDef existing, AtlasStructDef incoming) throws Exception {
+        return Whitebox.invokeMethod(typeAttributeDifference, "getElementsAbsentInExisting", existing, incoming);
+    }
+
+    private List<AtlasEnumDef.AtlasEnumElementDef> invokeGetAttributesAbsentInExisting(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception {
+        return Whitebox.invokeMethod(typeAttributeDifference, "getElementsAbsentInExisting", existing, incoming);
+    }
+
+    private AtlasEnumDef getAtlasEnumWithAttributes(String... elements) {
+        AtlasEnumDef enumDef = new AtlasEnumDef();
+        for (AtlasEnumDef.AtlasEnumElementDef ed : getEnumElementDefs(0, elements)) {
+            enumDef.addElement(ed);
+        }
+
+        return enumDef;
+    }
+
+    @Test
+    public void bothSame_DifferenceIsEmptyList() throws Exception {
+        AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name", "qualifiedName");
+        AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name", "qualifiedName");
+
+        List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs();
+        List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+        Assert.assertEquals(actualAttributes, expectedAttributes);
+    }
+
+    @Test
+    public void different_ReturnsDifference() throws Exception {
+        AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name");
+        AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name", "qualifiedName");
+        List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs( "qualifiedName");
+
+        List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+        Assert.assertEquals(actualAttributes, expectedAttributes);
+    }
+
+    @Test
+    public void differentSubset_ReturnsDifference() throws Exception {
+        AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name", "qualifiedName");
+        AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name");
+        List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+        List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs();
+        Assert.assertEquals(actualAttributes, expectedAttributes);
+    }
+
+    @Test
+    public void differentEnumDef_ReturnsDifference () throws Exception {
+        AtlasEnumDef existing = getAtlasEnumWithAttributes("Alpha", "Bravo");
+        AtlasEnumDef incoming = getAtlasEnumWithAttributes("Alpha", "Bravo", "Delta", "Echo");
+        List<AtlasEnumDef.AtlasEnumElementDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+        List<AtlasEnumDef.AtlasEnumElementDef> expectedAttributes = getEnumElementDefs(2, "Delta", "Echo");
+        Assert.assertEquals(actualAttributes, expectedAttributes);
+    }
+
+    @Test
+    public void differentEnumDefs_ReturnsDifference () throws Exception {
+        AtlasEnumDef existing = getAtlasEnumWithAttributes("Alpha", "Bravo");
+        AtlasEnumDef incoming = getAtlasEnumWithAttributes("Alpha", "Bravo", "Delta", "Echo");
+        boolean ret = invokeUpdate(existing, incoming);
+
+        List<AtlasEnumDef.AtlasEnumElementDef> expectedAttributes = getEnumElementDefs(0, "Alpha", "Bravo", "Delta", "Echo");
+
+        Assert.assertTrue(ret, "Update took place");
+        Assert.assertEquals(existing.getElementDefs(), expectedAttributes);
+    }
+
+    private boolean invokeUpdate(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception {
+        return Whitebox.invokeMethod(typeAttributeDifference, "addElements", existing, incoming);
+    }
+}

repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+public class UniqueListTest {
+    private final String firstElement = "firstElement";
+    private ExportService.UniqueList<String> uniqueList;
+
+    @BeforeClass
+    public void setup() {
+        uniqueList = new ExportService.UniqueList();
+        uniqueList.add(firstElement);
+        uniqueList.add("def");
+        uniqueList.add("firstElement");
+        uniqueList.add("ghi");
+    }
+
+    @Test
+    public void add3Elements_ListHas2() {
+        assertEquals(3, uniqueList.size());
+    }
+
+    @Test
+    public void addAllList_ListHas2() {
+        ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
+        uniqueList2.addAll(uniqueList);
+
+        assertEquals(3, uniqueList2.size());
+    }
+
+    @Test
+    public void attemptClear_SizeIsZero() {
+        ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
+        uniqueList2.addAll(uniqueList);
+        uniqueList2.clear();
+
+        assertEquals(0, uniqueList2.size());
+    }
+
+    @Test
+    public void attemptOneRemove_SizeIsReduced() {
+        ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
+        uniqueList2.addAll(uniqueList);
+        String removedElement = uniqueList2.remove(0);
+
+        assertEquals(2, uniqueList2.size());
+        assertEquals(firstElement, removedElement);
+    }
+}

repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java

@@ -31,7 +31,6 @@ import org.apache.commons.io.FileUtils;
 import org.apache.solr.common.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testng.Assert;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -42,6 +41,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.testng.Assert.*;
+
 public class ZipFileResourceTestUtils {
     public static final Logger LOG = LoggerFactory.getLogger(ZipFileResourceTestUtils.class);
 
@@ -67,7 +68,7 @@ public class ZipFileResourceTestUtils {
         String filePath = userDir + "/../addons/models/" + fileName;
         File f = new File(filePath);
         String s = FileUtils.readFileToString(f);
-        Assert.assertFalse(StringUtils.isEmpty(s), "Model file read correctly!");
+        assertFalse(StringUtils.isEmpty(s), "Model file read correctly!");
 
         return s;
     }
@@ -84,8 +85,8 @@ public class ZipFileResourceTestUtils {
         Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities);
         Set<String> difference = Sets.difference(lhs, rhs);
 
-        Assert.assertNotNull(difference);
-        Assert.assertEquals(difference.size(), 0);
+        assertNotNull(difference);
+        assertEquals(difference.size(), 0);
     }
 
     public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) {
@@ -96,8 +97,8 @@ public class ZipFileResourceTestUtils {
                     entry.getKey().contains("Column") ||
                     entry.getKey().contains("StorageDesc")) continue;
 
-            Assert.assertTrue(metricsForCompare.containsKey(entry.getKey()));
-            Assert.assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey()));
+            assertTrue(metricsForCompare.containsKey(entry.getKey()), entry.getKey());
+            assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey()), entry.getKey());
         }
     }
 
@@ -140,7 +141,7 @@ public class ZipFileResourceTestUtils {
         final String userName = "admin";
 
         AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP);
-        Assert.assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS);
+        assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS);
         return result;
     }
 
@@ -151,7 +152,7 @@ public class ZipFileResourceTestUtils {
         AtlasImportRequest request = getDefaultImportRequest();
         AtlasImportResult result = runImportWithParameters(importService, request, zipSource);
 
-        Assert.assertNotNull(result);
+        assertNotNull(result);
         verifyImportedMetrics(exportResult, result);
         verifyImportedEntities(creationOrder, result.getProcessedEntities());
     }