Skip to content

A
atlas
Commit 8101883c by ashutoshm Committed by Madhan Neethiraj
Options

ATLAS-1734: Import API update to make type import optional

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 0e7f8ea4
Showing with 676 additions and 60 deletions
intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
...@@ -44,6 +44,7 @@ public class AtlasImportRequest implements Serializable { ...@@ -44,6 +44,7 @@ public class AtlasImportRequest implements Serializable {
public static final String TRANSFORMS_KEY = "transforms"; public static final String TRANSFORMS_KEY = "transforms";
private static final String START_POSITION_KEY = "startPosition"; private static final String START_POSITION_KEY = "startPosition";
private static final String START_GUID_KEY = "startGuid"; private static final String START_GUID_KEY = "startGuid";
private static final String UPDATE_TYPE_DEFINITION_KEY = "updateTypeDefinition";
private Map<String, String> options; private Map<String, String> options;
...@@ -76,19 +77,24 @@ public class AtlasImportRequest implements Serializable { ...@@ -76,19 +77,24 @@ public class AtlasImportRequest implements Serializable {
@JsonIgnore @JsonIgnore
public String getStartGuid() { public String getStartGuid() {
if (this.options == null || !this.options.containsKey(START_GUID_KEY)) { return getOptionForKey(START_GUID_KEY);
return null;
} }
return (String) this.options.get(START_GUID_KEY); @JsonIgnore
public String getStartPosition() {
return getOptionForKey(START_POSITION_KEY);
} }
@JsonIgnore @JsonIgnore
public String getStartPosition() { public String getUpdateTypeDefs() {
if (this.options == null || !this.options.containsKey(START_POSITION_KEY)) { return getOptionForKey(UPDATE_TYPE_DEFINITION_KEY);
return null;
} }
return (String) this.options.get(START_GUID_KEY); private String getOptionForKey(String key) {
if (this.options == null || !this.options.containsKey(key)) {
return null;
} }
return (String) this.options.get(key);
} }
}
repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
...@@ -603,7 +603,7 @@ public class ExportService { ...@@ -603,7 +603,7 @@ public class ExportService {
} }
} }
private class UniqueList<T> { public static class UniqueList<T> {
private final List<T> list = new ArrayList<>(); private final List<T> list = new ArrayList<>();
private final Set<T> set = new HashSet<>(); private final Set<T> set = new HashSet<>();
......
repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
...@@ -21,12 +21,7 @@ import org.apache.atlas.AtlasErrorCode; ...@@ -21,12 +21,7 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult; import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
...@@ -51,7 +46,6 @@ public class ImportService { ...@@ -51,7 +46,6 @@ public class ImportService {
private long startTimestamp; private long startTimestamp;
private long endTimestamp; private long endTimestamp;
public ImportService(final AtlasTypeDefStore typeDefStore, final AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) { public ImportService(final AtlasTypeDefStore typeDefStore, final AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
this.typeDefStore = typeDefStore; this.typeDefStore = typeDefStore;
this.entityStore = entityStore; this.entityStore = entityStore;
...@@ -139,40 +133,12 @@ public class ImportService { ...@@ -139,40 +133,12 @@ public class ImportService {
} }
private void processTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { private void processTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
setGuidToEmpty(typeDefinitionMap); if(result.getRequest().getUpdateTypeDefs() != null && !result.getRequest().getUpdateTypeDefs().equals("true")) {
return;
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typeDefinitionMap, this.typeRegistry);
if (!typesToCreate.isEmpty()) {
typeDefStore.createTypesDef(typesToCreate);
updateMetricsForTypesDef(typesToCreate, result);
}
}
private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) {
result.incrementMeticsCounter("typedef:classification", typeDefinitionMap.getClassificationDefs().size());
result.incrementMeticsCounter("typedef:enum", typeDefinitionMap.getEnumDefs().size());
result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size());
result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size());
} }
private void setGuidToEmpty(AtlasTypesDef typesDef) { ImportTypeDefProcessor importTypeDefProcessor = new ImportTypeDefProcessor(this.typeDefStore, this.typeRegistry);
for (AtlasEntityDef def : typesDef.getEntityDefs()) { importTypeDefProcessor.processTypes(typeDefinitionMap, result);
def.setGuid(null);
}
for (AtlasClassificationDef def : typesDef.getClassificationDefs()) {
def.setGuid(null);
}
for (AtlasEnumDef def : typesDef.getEnumDefs()) {
def.setGuid(null);
}
for (AtlasStructDef def : typesDef.getStructDefs()) {
def.setGuid(null);
}
} }
private void processEntities(ZipSource importSource, AtlasImportResult result) throws AtlasBaseException { private void processEntities(ZipSource importSource, AtlasImportResult result) throws AtlasBaseException {
......
repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.impexp;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
public class ImportTypeDefProcessor {
private final AtlasTypeDefStore typeDefStore;
private final AtlasTypeRegistry typeRegistry;
private TypeAttributeDifference typeAttributeDifference;
public ImportTypeDefProcessor(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
this.typeAttributeDifference = new TypeAttributeDifference(typeDefStore, typeRegistry);
}
public void processTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
setGuidToEmpty(typeDefinitionMap);
typeAttributeDifference.updateTypes(typeDefinitionMap, result);
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typeDefinitionMap, this.typeRegistry);
if (!typesToCreate.isEmpty()) {
typeDefStore.createTypesDef(typesToCreate);
updateMetricsForTypesDef(typesToCreate, result);
}
}
private void setGuidToEmpty(AtlasTypesDef typesDef) {
for (AtlasEntityDef def : typesDef.getEntityDefs()) {
def.setGuid(null);
}
for (AtlasClassificationDef def : typesDef.getClassificationDefs()) {
def.setGuid(null);
}
for (AtlasEnumDef def : typesDef.getEnumDefs()) {
def.setGuid(null);
}
for (AtlasStructDef def : typesDef.getStructDefs()) {
def.setGuid(null);
}
}
private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) {
result.incrementMeticsCounter("typedef:classification", typeDefinitionMap.getClassificationDefs().size());
result.incrementMeticsCounter("typedef:enum", typeDefinitionMap.getEnumDefs().size());
result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size());
result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size());
}
}
repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.impexp;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import java.util.ArrayList;
import java.util.List;
public class TypeAttributeDifference {
private final AtlasTypeDefStore typeDefStore;
private final AtlasTypeRegistry typeRegistry;
public TypeAttributeDifference(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
}
public void updateTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
updateEntityDef(typeDefinitionMap, result);
updateClassificationDef(typeDefinitionMap, result);
updateEnumDef(typeDefinitionMap, result);
updateStructDef(typeDefinitionMap, result);
}
private void updateEntityDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
for (AtlasEntityDef def: typeDefinitionMap.getEntityDefs()) {
AtlasEntityDef existing = typeRegistry.getEntityDefByName(def.getName());
if(existing != null && addAttributes(existing, def)) {
typeDefStore.updateEntityDefByName(existing.getName(), existing);
result.incrementMeticsCounter("typedef:entitydef:update");
}
}
}
private void updateClassificationDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
for (AtlasClassificationDef def: typeDefinitionMap.getClassificationDefs()) {
AtlasClassificationDef existing = typeRegistry.getClassificationDefByName(def.getName());
if(existing != null && addAttributes(existing, def)) {
typeDefStore.updateClassificationDefByName(existing.getName(), existing);
result.incrementMeticsCounter("typedef:classification:update");
}
}
}
private void updateEnumDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
for (AtlasEnumDef def: typeDefinitionMap.getEnumDefs()) {
AtlasEnumDef existing = typeRegistry.getEnumDefByName(def.getName());
if(existing != null && addElements(existing, def)) {
typeDefStore.updateEnumDefByName(existing.getName(), existing);
result.incrementMeticsCounter("typedef:enum:update");
}
}
}
private void updateStructDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException {
for (AtlasStructDef def: typeDefinitionMap.getStructDefs()) {
AtlasStructDef existing = typeRegistry.getStructDefByName(def.getName());
if(existing != null && addAttributes(existing, def)) {
typeDefStore.updateStructDefByName(existing.getName(), existing);
result.incrementMeticsCounter("typedef:struct:update");
}
}
}
private boolean addElements(AtlasEnumDef existing, AtlasEnumDef incoming) {
return addElements(existing, getElementsAbsentInExisting(existing, incoming));
}
private boolean addAttributes(AtlasStructDef existing, AtlasStructDef incoming) {
return addAttributes(existing, getElementsAbsentInExisting(existing, incoming));
}
private List<AtlasStructDef.AtlasAttributeDef> getElementsAbsentInExisting(AtlasStructDef existing, AtlasStructDef incoming) {
List<AtlasStructDef.AtlasAttributeDef> difference = new ArrayList<>();
for (AtlasStructDef.AtlasAttributeDef attr : incoming.getAttributeDefs()) {
if(existing.getAttribute(attr.getName()) == null) {
difference.add(attr);
}
}
return difference;
}
private List<AtlasEnumDef.AtlasEnumElementDef> getElementsAbsentInExisting(AtlasEnumDef existing, AtlasEnumDef incoming) {
List<AtlasEnumDef.AtlasEnumElementDef> difference = new ArrayList<>();
for (AtlasEnumDef.AtlasEnumElementDef ed : incoming.getElementDefs()) {
if(existing.getElement(ed.getValue()) == null) {
difference.add(ed);
}
}
return difference;
}
private boolean addAttributes(AtlasStructDef def, List<AtlasStructDef.AtlasAttributeDef> list) {
for (AtlasStructDef.AtlasAttributeDef ad : list) {
def.addAttribute(ad);
}
return list.size() > 0;
}
private boolean addElements(AtlasEnumDef def, List<AtlasEnumDef.AtlasEnumElementDef> list) {
for (AtlasEnumDef.AtlasEnumElementDef ad : list) {
def.addElement(ad);
}
return list.size() > 0;
}
}
repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
...@@ -40,6 +40,47 @@ public class AtlasImportRequestTest { ...@@ -40,6 +40,47 @@ public class AtlasImportRequestTest {
} }
@Test @Test
public void serializeOptions_VerifyAccessors() {
String guid = "\"abcd\"";
String pos = "\"1\"";
String trueVal = "\"true\"";
String jsonData = "{ \"options\": " +
" {" +
"\"startGuid\":" + guid + "," +
"\"startPosition\":" + pos + "," +
"\"updateTypeDefinition\":" + trueVal +
"}" +
"}";
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
assertNotNull(request);
assertNotNull(request.getStartGuid());
assertNotNull(request.getStartPosition());
assertNotNull(request.getUpdateTypeDefs());
assertEquals(request.getStartGuid(), guid.replace("\"", ""));
assertEquals(request.getStartPosition(), pos.replace("\"", ""));
assertEquals(request.getUpdateTypeDefs(), trueVal.replace("\"", ""));
}
@Test
public void optionsDefaultsTest() {
String jsonData = "{ \"options\": " +
" {" +
"}" +
"}";
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
assertNotNull(request);
assertNull(request.getStartGuid());
assertNull(request.getStartPosition());
assertNull(request.getUpdateTypeDefs());
}
@Test
public void serializeAtlasImportRequstFromJsonWithEmptyTransforms() { public void serializeAtlasImportRequstFromJsonWithEmptyTransforms() {
String jsonData = "{ \"options\": { \"transforms\": \"{ }\" } }"; String jsonData = "{ \"options\": { \"transforms\": \"{ }\" } }";
......
repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
...@@ -20,6 +20,7 @@ package org.apache.atlas.repository.impexp; ...@@ -20,6 +20,7 @@ package org.apache.atlas.repository.impexp;
import com.google.inject.Inject; import com.google.inject.Inject;
import org.apache.atlas.TestModules; import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
...@@ -31,10 +32,12 @@ import org.testng.annotations.Guice; ...@@ -31,10 +32,12 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource; import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.*;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; import static org.testng.Assert.assertEquals;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runAndVerifyQuickStart_v1_Import; import static org.testng.Assert.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class) @Guice(modules = TestModules.TestOnlyModule.class)
public class ImportServiceTest { public class ImportServiceTest {
...@@ -55,7 +58,7 @@ public class ImportServiceTest { ...@@ -55,7 +58,7 @@ public class ImportServiceTest {
} }
@Test(dataProvider = "sales") @Test(dataProvider = "sales")
public void importDB1_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException { public void importDB1(ZipSource zipSource) throws AtlasBaseException, IOException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry); loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource); runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
} }
...@@ -66,7 +69,7 @@ public class ImportServiceTest { ...@@ -66,7 +69,7 @@ public class ImportServiceTest {
} }
@Test(dataProvider = "reporting") @Test(dataProvider = "reporting")
public void importDB2_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException { public void importDB2(ZipSource zipSource) throws AtlasBaseException, IOException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry); loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource); runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
} }
...@@ -77,8 +80,57 @@ public class ImportServiceTest { ...@@ -77,8 +80,57 @@ public class ImportServiceTest {
} }
@Test(dataProvider = "logging") @Test(dataProvider = "logging")
public void importDB3_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException { public void importDB3(ZipSource zipSource) throws AtlasBaseException, IOException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry); loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource); runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
} }
@DataProvider(name = "salesNewTypeAttrs")
public static Object[][] getDataFromSalesNewTypeAttrs(ITestContext context) throws IOException {
return getZipSource("salesNewTypeAttrs.zip");
}
@Test(dataProvider = "salesNewTypeAttrs", dependsOnMethods = "importDB1")
public void importDB4(ZipSource zipSource) throws AtlasBaseException, IOException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), getDefaultImportRequest(), zipSource);
}
@DataProvider(name = "salesNewTypeAttrs-next")
public static Object[][] getDataFromSalesNewTypeAttrsNext(ITestContext context) throws IOException {
return getZipSource("salesNewTypeAttrs-next.zip");
}
@Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4")
public void importDB5(ZipSource zipSource) throws AtlasBaseException, IOException {
final String newEnumDefName = "database_action";
assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
AtlasImportRequest request = getDefaultImportRequest();
Map<String, String> options = new HashMap<>();
options.put("updateTypeDefinition", "false");
request.setOptions(options);
runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), request, zipSource);
assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 4);
}
@Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4")
public void importDB6(ZipSource zipSource) throws AtlasBaseException, IOException {
final String newEnumDefName = "database_action";
assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
AtlasImportRequest request = getDefaultImportRequest();
Map<String, String> options = new HashMap<>();
options.put("updateTypeDefinition", "true");
request.setOptions(options);
runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), request, zipSource);
assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 8);
}
} }
repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.impexp;
import com.google.common.collect.Sets;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.testng.Assert;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class ImportServiceTestUtils {
public static void verifyImportedEntities(List<String> creationOrder, List<String> processedEntities) {
Set<String> lhs = com.google.common.collect.Sets.newHashSet(creationOrder);
Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities);
Set<String> difference = Sets.difference(lhs, rhs);
Assert.assertNotNull(difference);
Assert.assertEquals(difference.size(), 0);
}
public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) {
Map<String, Integer> metricsForCompare = getImportMetricsForCompare(importResult);
for (Map.Entry<String, Integer> entry : exportResult.getMetrics().entrySet()) {
if(entry.getKey().startsWith("entity") == false ||
entry.getKey().contains("withExtInfo") ||
entry.getKey().contains("Column") ||
entry.getKey().contains("StorageDesc")) continue;
Assert.assertTrue(metricsForCompare.containsKey(entry.getKey()));
Assert.assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey()));
}
}
private static Map<String,Integer> getImportMetricsForCompare(AtlasImportResult result) {
Map<String, Integer> r = new HashMap<>();
for (Map.Entry<String, Integer> entry : result.getMetrics().entrySet()) {
r.put(entry.getKey().replace(":updated", "").replace(":created", ""), entry.getValue());
}
return r;
}
public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName);
createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
}
private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesFromJson, typeRegistry);
if (!typesToCreate.isEmpty()) {
typeDefStore.createTypesDef(typesToCreate);
}
}
private static AtlasTypesDef getAtlasTypesDefFromFile(String fileName) throws IOException {
String sampleTypes = ZipFileResourceTestUtils.getModelJson(fileName);
return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
}
public static AtlasImportRequest getDefaultImportRequest() {
return new AtlasImportRequest();
}
public static AtlasImportResult runImportWithParameters(ImportService importService, AtlasImportRequest request, ZipSource source) throws AtlasBaseException, IOException {
final String requestingIP = "1.0.0.0";
final String hostName = "localhost";
final String userName = "admin";
AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP);
Assert.assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS);
return result;
}
public static void runAndVerifyQuickStart_v1_Import(ImportService importService, ZipSource zipSource) throws AtlasBaseException, IOException {
AtlasExportResult exportResult = zipSource.getExportResult();
List<String> creationOrder = zipSource.getCreationOrder();
AtlasImportRequest request = getDefaultImportRequest();
AtlasImportResult result = runImportWithParameters(importService, request, zipSource);
Assert.assertNotNull(result);
verifyImportedMetrics(exportResult, result);
verifyImportedEntities(creationOrder, result.getProcessedEntities());
}
}
repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.impexp;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.powermock.reflect.Whitebox;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.List;
public class TypeAttributeDifferenceTest {
private TypeAttributeDifference typeAttributeDifference;
@BeforeClass
public void setup() {
typeAttributeDifference = new TypeAttributeDifference(null, null);
}
private List<AtlasEnumDef.AtlasEnumElementDef> getEnumElementDefs(int startIndex, String... names) {
int i = startIndex;
List<AtlasEnumDef.AtlasEnumElementDef> list = new ArrayList<>();
for (String s: names) {
list.add(new AtlasEnumDef.AtlasEnumElementDef(s, s, i++));
}
return list;
}
private List<AtlasStructDef.AtlasAttributeDef> getAtlasAttributeDefs(String... names) {
List<AtlasStructDef.AtlasAttributeDef> list = new ArrayList<>();
for (String s : names) {
list.add(new AtlasStructDef.AtlasAttributeDef(s, AtlasBaseTypeDef.ATLAS_TYPE_STRING));
}
return list;
}
private AtlasEntityDef getAtlasEntityDefWithAttributes(String... attributeNames) {
AtlasEntityDef e = new AtlasEntityDef();
for (AtlasStructDef.AtlasAttributeDef a : getAtlasAttributeDefs(attributeNames)) {
e.addAttribute(a);
}
return e;
}
@Test
public void entityDefWithNoAttributes() throws Exception {
AtlasEntityDef existing = new AtlasEntityDef();
AtlasEntityDef incoming = new AtlasEntityDef();
List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = new ArrayList<>();
List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
Assert.assertEquals(actualAttributes, expectedAttributes);
}
private List<AtlasStructDef.AtlasAttributeDef> invokeGetAttributesAbsentInExisting(AtlasStructDef existing, AtlasStructDef incoming) throws Exception {
return Whitebox.invokeMethod(typeAttributeDifference, "getElementsAbsentInExisting", existing, incoming);
}
private List<AtlasEnumDef.AtlasEnumElementDef> invokeGetAttributesAbsentInExisting(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception {
return Whitebox.invokeMethod(typeAttributeDifference, "getElementsAbsentInExisting", existing, incoming);
}
private AtlasEnumDef getAtlasEnumWithAttributes(String... elements) {
AtlasEnumDef enumDef = new AtlasEnumDef();
for (AtlasEnumDef.AtlasEnumElementDef ed : getEnumElementDefs(0, elements)) {
enumDef.addElement(ed);
}
return enumDef;
}
@Test
public void bothSame_DifferenceIsEmptyList() throws Exception {
AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name", "qualifiedName");
AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name", "qualifiedName");
List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs();
List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
Assert.assertEquals(actualAttributes, expectedAttributes);
}
@Test
public void different_ReturnsDifference() throws Exception {
AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name");
AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name", "qualifiedName");
List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs( "qualifiedName");
List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
Assert.assertEquals(actualAttributes, expectedAttributes);
}
@Test
public void differentSubset_ReturnsDifference() throws Exception {
AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name", "qualifiedName");
AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name");
List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs();
Assert.assertEquals(actualAttributes, expectedAttributes);
}
@Test
public void differentEnumDef_ReturnsDifference () throws Exception {
AtlasEnumDef existing = getAtlasEnumWithAttributes("Alpha", "Bravo");
AtlasEnumDef incoming = getAtlasEnumWithAttributes("Alpha", "Bravo", "Delta", "Echo");
List<AtlasEnumDef.AtlasEnumElementDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
List<AtlasEnumDef.AtlasEnumElementDef> expectedAttributes = getEnumElementDefs(2, "Delta", "Echo");
Assert.assertEquals(actualAttributes, expectedAttributes);
}
@Test
public void differentEnumDefs_ReturnsDifference () throws Exception {
AtlasEnumDef existing = getAtlasEnumWithAttributes("Alpha", "Bravo");
AtlasEnumDef incoming = getAtlasEnumWithAttributes("Alpha", "Bravo", "Delta", "Echo");
boolean ret = invokeUpdate(existing, incoming);
List<AtlasEnumDef.AtlasEnumElementDef> expectedAttributes = getEnumElementDefs(0, "Alpha", "Bravo", "Delta", "Echo");
Assert.assertTrue(ret, "Update took place");
Assert.assertEquals(existing.getElementDefs(), expectedAttributes);
}
private boolean invokeUpdate(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception {
return Whitebox.invokeMethod(typeAttributeDifference, "addElements", existing, incoming);
}
}
repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.impexp;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
public class UniqueListTest {
private final String firstElement = "firstElement";
private ExportService.UniqueList<String> uniqueList;
@BeforeClass
public void setup() {
uniqueList = new ExportService.UniqueList();
uniqueList.add(firstElement);
uniqueList.add("def");
uniqueList.add("firstElement");
uniqueList.add("ghi");
}
@Test
public void add3Elements_ListHas2() {
assertEquals(3, uniqueList.size());
}
@Test
public void addAllList_ListHas2() {
ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
uniqueList2.addAll(uniqueList);
assertEquals(3, uniqueList2.size());
}
@Test
public void attemptClear_SizeIsZero() {
ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
uniqueList2.addAll(uniqueList);
uniqueList2.clear();
assertEquals(0, uniqueList2.size());
}
@Test
public void attemptOneRemove_SizeIsReduced() {
ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
uniqueList2.addAll(uniqueList);
String removedElement = uniqueList2.remove(0);
assertEquals(2, uniqueList2.size());
assertEquals(firstElement, removedElement);
}
}
repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
...@@ -31,7 +31,6 @@ import org.apache.commons.io.FileUtils; ...@@ -31,7 +31,6 @@ import org.apache.commons.io.FileUtils;
import org.apache.solr.common.StringUtils; import org.apache.solr.common.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.testng.Assert;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
...@@ -42,6 +41,8 @@ import java.util.List; ...@@ -42,6 +41,8 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static org.testng.Assert.*;
public class ZipFileResourceTestUtils { public class ZipFileResourceTestUtils {
public static final Logger LOG = LoggerFactory.getLogger(ZipFileResourceTestUtils.class); public static final Logger LOG = LoggerFactory.getLogger(ZipFileResourceTestUtils.class);
...@@ -67,7 +68,7 @@ public class ZipFileResourceTestUtils { ...@@ -67,7 +68,7 @@ public class ZipFileResourceTestUtils {
String filePath = userDir + "/../addons/models/" + fileName; String filePath = userDir + "/../addons/models/" + fileName;
File f = new File(filePath); File f = new File(filePath);
String s = FileUtils.readFileToString(f); String s = FileUtils.readFileToString(f);
Assert.assertFalse(StringUtils.isEmpty(s), "Model file read correctly!"); assertFalse(StringUtils.isEmpty(s), "Model file read correctly!");
return s; return s;
} }
...@@ -84,8 +85,8 @@ public class ZipFileResourceTestUtils { ...@@ -84,8 +85,8 @@ public class ZipFileResourceTestUtils {
Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities); Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities);
Set<String> difference = Sets.difference(lhs, rhs); Set<String> difference = Sets.difference(lhs, rhs);
Assert.assertNotNull(difference); assertNotNull(difference);
Assert.assertEquals(difference.size(), 0); assertEquals(difference.size(), 0);
} }
public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) { public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) {
...@@ -96,8 +97,8 @@ public class ZipFileResourceTestUtils { ...@@ -96,8 +97,8 @@ public class ZipFileResourceTestUtils {
entry.getKey().contains("Column") || entry.getKey().contains("Column") ||
entry.getKey().contains("StorageDesc")) continue; entry.getKey().contains("StorageDesc")) continue;
Assert.assertTrue(metricsForCompare.containsKey(entry.getKey())); assertTrue(metricsForCompare.containsKey(entry.getKey()), entry.getKey());
Assert.assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey())); assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey()), entry.getKey());
} }
} }
...@@ -140,7 +141,7 @@ public class ZipFileResourceTestUtils { ...@@ -140,7 +141,7 @@ public class ZipFileResourceTestUtils {
final String userName = "admin"; final String userName = "admin";
AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP); AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP);
Assert.assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS); assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS);
return result; return result;
} }
...@@ -151,7 +152,7 @@ public class ZipFileResourceTestUtils { ...@@ -151,7 +152,7 @@ public class ZipFileResourceTestUtils {
AtlasImportRequest request = getDefaultImportRequest(); AtlasImportRequest request = getDefaultImportRequest();
AtlasImportResult result = runImportWithParameters(importService, request, zipSource); AtlasImportResult result = runImportWithParameters(importService, request, zipSource);
Assert.assertNotNull(result); assertNotNull(result);
verifyImportedMetrics(exportResult, result); verifyImportedMetrics(exportResult, result);
verifyImportedEntities(creationOrder, result.getProcessedEntities()); verifyImportedEntities(creationOrder, result.getProcessedEntities());
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment