ATLAS-4006, ATLAS-4007: Support business metadata in export and import operations.

repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java

@@ -105,7 +105,7 @@ public class AtlasServerService {
 
         AtlasObjectId objectId = getObjectId(server);
         for (String guid : entityGuids) {
-            AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(guid, false, true);
+            AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(guid, false, false);
             updateAttribute(entityWithExtInfo, attributeName, objectId);
         }
     }

repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java

@@ -25,6 +25,7 @@ import org.apache.atlas.model.impexp.AtlasExportResult;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
 import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
 import org.apache.atlas.model.typedef.AtlasClassificationDef;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
@@ -163,6 +164,12 @@ public class ExportService {
 
             typesDef.getRelationshipDefs().add(relationshipDef);
         }
+
+        for (String bm : context.businessMetadataTypes) {
+            AtlasBusinessMetadataDef bmDef = typeRegistry.getBusinessMetadataDefByName(bm);
+
+            typesDef.getBusinessMetadataDefs().add(bmDef);
+        }
     }
 
     private AtlasExportResult.OperationStatus[] processItems(AtlasExportRequest request, ExportContext context) {

repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java

@@ -121,6 +121,7 @@ class ExportTypeProcessor {
 
             addAttributeTypes(entityType, context);
             addRelationshipTypes(entityType, context);
+            addBusinessMetadataType(entityType, context);
 
             if (CollectionUtils.isNotEmpty(entityType.getAllSuperTypes())) {
                 for (String superType : entityType.getAllSuperTypes()) {
@@ -180,6 +181,14 @@ class ExportTypeProcessor {
         }
     }
 
+    private void addBusinessMetadataType(AtlasEntityType entityType, ExportService.ExportContext context) {
+        for (String bmTypeName : entityType.getBusinessAttributes().keySet()) {
+            AtlasBusinessMetadataType bmType = typeRegistry.getBusinessMetadataTypeByName(bmTypeName);
+
+            addBusinessMetadataType(bmType, context);
+        }
+    }
+
     private void addAttributeTypes(AtlasStructType structType, ExportService.ExportContext context) {
         for (AtlasStructDef.AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) {
             addType(attributeDef.getTypeName(), context);

repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java

@@ -19,6 +19,7 @@ package org.apache.atlas.repository.impexp;
 
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
 import org.apache.atlas.model.typedef.AtlasClassificationDef;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
@@ -72,6 +73,10 @@ public class ImportTypeDefProcessor {
         for (AtlasRelationshipDef def : typesDef.getRelationshipDefs()) {
             def.setGuid(null);
         }
+
+        for (AtlasBusinessMetadataDef def : typesDef.getBusinessMetadataDefs()) {
+            def.setGuid(null);
+        }
     }
 
     private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) {
@@ -80,5 +85,6 @@ public class ImportTypeDefProcessor {
         result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size());
         result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size());
         result.incrementMeticsCounter("typedef:relationship", typeDefinitionMap.getRelationshipDefs().size());
+        result.incrementMeticsCounter("typedef:businessmetadata", typeDefinitionMap.getBusinessMetadataDefs().size());
     }
 }

repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java

@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
 import org.apache.atlas.model.typedef.AtlasClassificationDef;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
@@ -54,6 +55,7 @@ public class TypeAttributeDifference {
         updateClassificationDef(typeDefinitionMap.getClassificationDefs(), result);
         updateEntityDef(typeDefinitionMap.getEntityDefs(), result);
         updateRelationshipDefs(typeDefinitionMap.getRelationshipDefs(), result);
+        updateBusinessMetadataDefs(typeDefinitionMap.getBusinessMetadataDefs(), result);
     }
 
     private void updateEntityDef(List<AtlasEntityDef> entityDefs, AtlasImportResult result) throws AtlasBaseException {
@@ -108,6 +110,16 @@ public class TypeAttributeDifference {
         }
     }
 
+    private void updateBusinessMetadataDefs(List<AtlasBusinessMetadataDef> businessMetadataDefs, AtlasImportResult result) throws AtlasBaseException {
+        for (AtlasBusinessMetadataDef def : businessMetadataDefs) {
+            AtlasBusinessMetadataDef existing = typeRegistry.getBusinessMetadataDefByName(def.getName());
+            if (existing != null && addAttributes(existing, def)) {
+                typeDefStore.updateStructDefByName(existing.getName(), existing);
+                result.incrementMeticsCounter("typedef:businessmetadatadef:update");
+            }
+        }
+    }
+
     @VisibleForTesting
     boolean addElements(AtlasEnumDef existing, AtlasEnumDef incoming) throws AtlasBaseException {
         return addElements(existing, getElementsAbsentInExisting(existing, incoming));

repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java

@@ -17,22 +17,33 @@
  */
 package org.apache.atlas.repository.impexp;
 
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportResult;
 import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
 import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
 import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestLoadModelUtils;
+import org.apache.atlas.utils.TestResourceFileUtils;
 import org.mockito.Mock;
-import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
 public class TypeAttributeDifferenceTest {
     private TypeAttributeDifference typeAttributeDifference;
 
@@ -161,4 +172,15 @@ public class TypeAttributeDifferenceTest {
     private boolean invokeUpdate(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception {
         return typeAttributeDifference.addElements(existing, incoming);
     }
+
+    @Test
+    public void t1() throws IOException, AtlasBaseException {
+        AtlasTypesDef typesDef = TestResourceFileUtils.readObjectFromJson(".", "typesDef-bm", AtlasTypesDef.class);
+
+        AtlasImportResult result = new AtlasImportResult();
+        AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, this.typeRegistry);
+        typeDefStore.createTypesDef(typesToCreate);
+        typeAttributeDifference.updateTypes(typesDef, result);
+        assertNotNull(typesDef);
+    }
 }

repository/src/test/resources/json/typesDef-bm.json

+{
+  "enumDefs": [],
+  "structDefs": [],
+  "classificationDefs": [],
+  "entityDefs": [
+    {
+      "category": "ENTITY",
+      "name": "rdbms_table",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": -1,
+          "valuesMaxCount": -1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "name": "rdbms_storage",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": -1,
+          "valuesMaxCount": -1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "name": "rdbms_db",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": -1,
+          "valuesMaxCount": -1,
+          "isUnique": true,
+          "isIndexable": true
+        },
+        {
+          "name": "sd",
+          "typeName": "rdbms_storage",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": -1,
+          "valuesMaxCount": -1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ],
+          "options": {
+            "isSoftReference": "true"
+          }
+        },
+        {
+          "name": "tables",
+          "typeName": "array<rdbms_table>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": -1,
+          "valuesMaxCount": -1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ],
+          "options": {
+            "isSoftReference": "true"
+          }
+        },
+        {
+          "name": "regions",
+          "typeName": "map<string,rdbms_table>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": -1,
+          "valuesMaxCount": -1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ],
+          "options": {
+            "isSoftReference": "true"
+          }
+        }
+      ],
+      "superTypes": [],
+      "businessAttributeDefs": {
+        "B1": [{
+          "name": "A1",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true,
+          "includeInNotification": false,
+          "searchWeight": 5,
+          "options": {
+            "applicableEntityTypes": "[\"hive_db\"]",
+            "maxStrLength": "50"
+          }
+        }]
+      }
+    }
+  ],
+  "businessMetadataDefs": [
+    {
+      "category": "BUSINESS_METADATA",
+      "guid": "a0d52995-9ef7-41e7-bfc0-652e0cf3587c",
+      "createdBy": "admin",
+      "updatedBy": "admin",
+      "createTime": 1603487492875,
+      "updateTime": 1603487515791,
+      "version": 2,
+      "name": "B1",
+      "description": "Test B1",
+      "typeVersion": "1.1",
+      "attributeDefs": [
+        {
+          "name": "A1",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true,
+          "includeInNotification": false,
+          "searchWeight": 5,
+          "options": {
+            "applicableEntityTypes": "[\"rdbms_table\"]",
+            "maxStrLength": "50"
+          }
+        }
+      ]
+    }
+  ]
+}