Skip to content

A
atlas
Commit 801d40c3 by Ashutosh Mestry Committed by Nikhil Bonte
Options

ATLAS-4006, ATLAS-4007: Support business metadata in export and import operations.

Signed-off-by: 's avatarNikhil P Bonte <nbonte@apache.org>
parent e6fff748
Showing with 223 additions and 2 deletions
repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
......@@ -105,7 +105,7 @@ public class AtlasServerService {
AtlasObjectId objectId = getObjectId(server);
for (String guid : entityGuids) {
AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(guid, false, true);
AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(guid, false, false);
updateAttribute(entityWithExtInfo, attributeName, objectId);
}
}
......
repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
......@@ -163,6 +164,12 @@ public class ExportService {
typesDef.getRelationshipDefs().add(relationshipDef);
}
for (String bm : context.businessMetadataTypes) {
AtlasBusinessMetadataDef bmDef = typeRegistry.getBusinessMetadataDefByName(bm);
typesDef.getBusinessMetadataDefs().add(bmDef);
}
}
private AtlasExportResult.OperationStatus[] processItems(AtlasExportRequest request, ExportContext context) {
......
repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
......@@ -121,6 +121,7 @@ class ExportTypeProcessor {
addAttributeTypes(entityType, context);
addRelationshipTypes(entityType, context);
addBusinessMetadataType(entityType, context);
if (CollectionUtils.isNotEmpty(entityType.getAllSuperTypes())) {
for (String superType : entityType.getAllSuperTypes()) {
......@@ -180,6 +181,14 @@ class ExportTypeProcessor {
}
}
private void addBusinessMetadataType(AtlasEntityType entityType, ExportService.ExportContext context) {
for (String bmTypeName : entityType.getBusinessAttributes().keySet()) {
AtlasBusinessMetadataType bmType = typeRegistry.getBusinessMetadataTypeByName(bmTypeName);
addBusinessMetadataType(bmType, context);
}
}
private void addAttributeTypes(AtlasStructType structType, ExportService.ExportContext context) {
for (AtlasStructDef.AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) {
addType(attributeDef.getTypeName(), context);
......
repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
......@@ -72,6 +73,10 @@ public class ImportTypeDefProcessor {
for (AtlasRelationshipDef def : typesDef.getRelationshipDefs()) {
def.setGuid(null);
}
for (AtlasBusinessMetadataDef def : typesDef.getBusinessMetadataDefs()) {
def.setGuid(null);
}
}
private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) {
......@@ -80,5 +85,6 @@ public class ImportTypeDefProcessor {
result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size());
result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size());
result.incrementMeticsCounter("typedef:relationship", typeDefinitionMap.getRelationshipDefs().size());
result.incrementMeticsCounter("typedef:businessmetadata", typeDefinitionMap.getBusinessMetadataDefs().size());
}
}
repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
......@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
......@@ -54,6 +55,7 @@ public class TypeAttributeDifference {
updateClassificationDef(typeDefinitionMap.getClassificationDefs(), result);
updateEntityDef(typeDefinitionMap.getEntityDefs(), result);
updateRelationshipDefs(typeDefinitionMap.getRelationshipDefs(), result);
updateBusinessMetadataDefs(typeDefinitionMap.getBusinessMetadataDefs(), result);
}
private void updateEntityDef(List<AtlasEntityDef> entityDefs, AtlasImportResult result) throws AtlasBaseException {
......@@ -108,6 +110,16 @@ public class TypeAttributeDifference {
}
}
private void updateBusinessMetadataDefs(List<AtlasBusinessMetadataDef> businessMetadataDefs, AtlasImportResult result) throws AtlasBaseException {
for (AtlasBusinessMetadataDef def : businessMetadataDefs) {
AtlasBusinessMetadataDef existing = typeRegistry.getBusinessMetadataDefByName(def.getName());
if (existing != null && addAttributes(existing, def)) {
typeDefStore.updateStructDefByName(existing.getName(), existing);
result.incrementMeticsCounter("typedef:businessmetadatadef:update");
}
}
}
@VisibleForTesting
boolean addElements(AtlasEnumDef existing, AtlasEnumDef incoming) throws AtlasBaseException {
return addElements(existing, getElementsAbsentInExisting(existing, incoming));
......
repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
......@@ -17,22 +17,33 @@
*/
package org.apache.atlas.repository.impexp;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.TestLoadModelUtils;
import org.apache.atlas.utils.TestResourceFileUtils;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
public class TypeAttributeDifferenceTest {
private TypeAttributeDifference typeAttributeDifference;
......@@ -161,4 +172,15 @@ public class TypeAttributeDifferenceTest {
private boolean invokeUpdate(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception {
return typeAttributeDifference.addElements(existing, incoming);
}
@Test
public void t1() throws IOException, AtlasBaseException {
AtlasTypesDef typesDef = TestResourceFileUtils.readObjectFromJson(".", "typesDef-bm", AtlasTypesDef.class);
AtlasImportResult result = new AtlasImportResult();
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, this.typeRegistry);
typeDefStore.createTypesDef(typesToCreate);
typeAttributeDifference.updateTypes(typesDef, result);
assertNotNull(typesDef);
}
}
repository/src/test/resources/json/typesDef-bm.json 0 → 100644
{
"enumDefs": [],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
{
"category": "ENTITY",
"name": "rdbms_table",
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "name",
"typeName": "string",
"isOptional": false,
"cardinality": "SINGLE",
"valuesMinCount": -1,
"valuesMaxCount": -1,
"isUnique": false,
"isIndexable": false
}
],
"superTypes": []
},
{
"category": "ENTITY",
"name": "rdbms_storage",
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "name",
"typeName": "string",
"isOptional": false,
"cardinality": "SINGLE",
"valuesMinCount": -1,
"valuesMaxCount": -1,
"isUnique": false,
"isIndexable": false
}
],
"superTypes": []
},
{
"category": "ENTITY",
"name": "rdbms_db",
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "name",
"typeName": "string",
"isOptional": false,
"cardinality": "SINGLE",
"valuesMinCount": -1,
"valuesMaxCount": -1,
"isUnique": true,
"isIndexable": true
},
{
"name": "sd",
"typeName": "rdbms_storage",
"isOptional": true,
"cardinality": "SINGLE",
"valuesMinCount": -1,
"valuesMaxCount": -1,
"isUnique": false,
"isIndexable": false,
"constraints": [
{
"type": "ownedRef"
}
],
"options": {
"isSoftReference": "true"
}
},
{
"name": "tables",
"typeName": "array<rdbms_table>",
"isOptional": true,
"cardinality": "SINGLE",
"valuesMinCount": -1,
"valuesMaxCount": -1,
"isUnique": false,
"isIndexable": false,
"constraints": [
{
"type": "ownedRef"
}
],
"options": {
"isSoftReference": "true"
}
},
{
"name": "regions",
"typeName": "map<string,rdbms_table>",
"isOptional": true,
"cardinality": "SINGLE",
"valuesMinCount": -1,
"valuesMaxCount": -1,
"isUnique": false,
"isIndexable": false,
"constraints": [
{
"type": "ownedRef"
}
],
"options": {
"isSoftReference": "true"
}
}
],
"superTypes": [],
"businessAttributeDefs": {
"B1": [{
"name": "A1",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"valuesMinCount": 0,
"valuesMaxCount": 1,
"isUnique": false,
"isIndexable": true,
"includeInNotification": false,
"searchWeight": 5,
"options": {
"applicableEntityTypes": "[\"hive_db\"]",
"maxStrLength": "50"
}
}]
}
}
],
"businessMetadataDefs": [
{
"category": "BUSINESS_METADATA",
"guid": "a0d52995-9ef7-41e7-bfc0-652e0cf3587c",
"createdBy": "admin",
"updatedBy": "admin",
"createTime": 1603487492875,
"updateTime": 1603487515791,
"version": 2,
"name": "B1",
"description": "Test B1",
"typeVersion": "1.1",
"attributeDefs": [
{
"name": "A1",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"valuesMinCount": 0,
"valuesMaxCount": 1,
"isUnique": false,
"isIndexable": true,
"includeInNotification": false,
"searchWeight": 5,
"options": {
"applicableEntityTypes": "[\"rdbms_table\"]",
"maxStrLength": "50"
}
}
]
}
]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment