ATLAS-3037: fixed entity-updated detection to handle object-id attributes correctly

authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizationUtils.java

@@ -22,6 +22,7 @@ package org.apache.atlas.authorize;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.RequestContext;
 import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -87,6 +88,8 @@ public class AtlasAuthorizationUtils {
     }
 
     public static boolean isAccessAllowed(AtlasAdminAccessRequest request) {
+        MetricRecorder metric = RequestContext.get().startMetricRecord("isAccessAllowed");
+
         boolean ret      = false;
         String  userName = getCurrentUserName();
 
@@ -104,10 +107,14 @@ public class AtlasAuthorizationUtils {
             ret = true;
         }
 
+        RequestContext.get().endMetricRecord(metric);
+
         return ret;
     }
 
     public static boolean isAccessAllowed(AtlasEntityAccessRequest request) {
+        MetricRecorder metric = RequestContext.get().startMetricRecord("isAccessAllowed");
+
         boolean ret      = false;
         String  userName = getCurrentUserName();
 
@@ -125,10 +132,14 @@ public class AtlasAuthorizationUtils {
             ret = true;
         }
 
+        RequestContext.get().endMetricRecord(metric);
+
         return ret;
     }
 
     public static boolean isAccessAllowed(AtlasTypeAccessRequest request) {
+        MetricRecorder metric = RequestContext.get().startMetricRecord("isAccessAllowed");
+
         boolean ret      = false;
         String  userName = getCurrentUserName();
 
@@ -146,10 +157,14 @@ public class AtlasAuthorizationUtils {
             ret = true;
         }
 
+        RequestContext.get().endMetricRecord(metric);
+
         return ret;
     }
 
     public static boolean isAccessAllowed(AtlasRelationshipAccessRequest request) {
+        MetricRecorder metric = RequestContext.get().startMetricRecord("isAccessAllowed");
+
         boolean ret      = false;
         String  userName = getCurrentUserName();
 
@@ -167,6 +182,8 @@ public class AtlasAuthorizationUtils {
             ret = true;
         }
 
+        RequestContext.get().endMetricRecord(metric);
+
         return ret;
     }
 

intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java

@@ -28,6 +28,7 @@ import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.SERVICE_TYPE_ATLAS_CORE;
@@ -149,7 +150,7 @@ public class AtlasArrayType extends AtlasType {
     }
 
     @Override
-    public boolean areEqualValues(Object val1, Object val2) {
+    public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
         boolean ret = true;
 
         if (val1 == null) {
@@ -164,7 +165,7 @@ public class AtlasArrayType extends AtlasType {
                     ret = false;
                 } else {
                     for (int i = 0; i < len; i++) {
-                        if (!elementType.areEqualValues(Array.get(val1, i), Array.get(val2, i))) {
+                        if (!elementType.areEqualValues(Array.get(val1, i), Array.get(val2, i), guidAssignments)) {
                             ret = false;
 
                             break;
@@ -182,7 +183,7 @@ public class AtlasArrayType extends AtlasType {
                         boolean foundInSet2 = false;
 
                         for (Object elem2 : set2) {
-                            if (elementType.areEqualValues(elem1, elem2)) {
+                            if (elementType.areEqualValues(elem1, elem2, guidAssignments)) {
                                 foundInSet2 = true;
 
                                 break;
@@ -213,7 +214,7 @@ public class AtlasArrayType extends AtlasType {
                             ret = false;
                         } else {
                             for (int i = 0; i < len; i++) {
-                                if (!elementType.areEqualValues(list1.get(i), list2.get(i))) {
+                                if (!elementType.areEqualValues(list1.get(i), list2.get(i), guidAssignments)) {
                                     ret = false;
 
                                     break;

intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java

@@ -29,6 +29,7 @@ import java.math.BigInteger;
 import java.text.ParseException;
 import java.util.Date;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.SERVICE_TYPE_ATLAS_CORE;
 
@@ -277,7 +278,8 @@ public class AtlasBuiltInTypes {
             return getNormalizedValue(obj) != null;
         }
 
-        public boolean areEqualValues(Object val1, Object val2) {
+        @Override
+        public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
             final boolean ret;
 
             if (val1 == null) {
@@ -353,7 +355,8 @@ public class AtlasBuiltInTypes {
             return getNormalizedValue(obj) != null;
         }
 
-        public boolean areEqualValues(Object val1, Object val2) {
+        @Override
+        public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
             final boolean ret;
 
             if (val1 == null) {
@@ -626,6 +629,48 @@ public class AtlasBuiltInTypes {
         }
 
         @Override
+        public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
+            boolean ret = true;
+
+            if (val1 == null) {
+                ret = val2 == null;
+            } else if (val2 == null) {
+                ret = false;
+            } else {
+                AtlasObjectId v1 = getNormalizedValue(val1);
+                AtlasObjectId v2 = getNormalizedValue(val2);
+
+                if (v1 == null || v2 == null) {
+                    ret = false;
+                } else {
+                    String guid1 = v1.getGuid();
+                    String guid2 = v2.getGuid();
+
+                    if (guidAssignments != null ) {
+                        if (guidAssignments.containsKey(guid1)) {
+                            guid1 = guidAssignments.get(guid1);
+                        }
+
+                        if (guidAssignments.containsKey(guid2)) {
+                            guid2 = guidAssignments.get(guid2);
+                        }
+                    }
+
+                    boolean isV1AssignedGuid = AtlasTypeUtil.isAssignedGuid(guid1);
+                    boolean isV2AssignedGuid = AtlasTypeUtil.isAssignedGuid(guid2);
+
+                    if (isV1AssignedGuid == isV2AssignedGuid) { // if both have assigned/unassigned guids, compare guids
+                        ret = Objects.equals(guid1, guid2);
+                    } else { // if one has assigned and other unassigned guid, compare typeName and unique-attribute
+                        ret = Objects.equals(v1.getTypeName(), v2.getTypeName()) && Objects.equals(v1.getUniqueAttributes(), v2.getUniqueAttributes());
+                    }
+                }
+            }
+
+            return ret;
+        }
+
+        @Override
         public AtlasObjectId getNormalizedValue(Object obj) {
             if (obj != null) {
                 if (obj instanceof AtlasObjectId) {

intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java

@@ -315,14 +315,14 @@ public class AtlasClassificationType extends AtlasStructType {
     }
 
     @Override
-    public boolean areEqualValues(Object val1, Object val2) {
+    public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
         for (AtlasClassificationType superType : superTypes) {
-            if (!superType.areEqualValues(val1, val2)) {
+            if (!superType.areEqualValues(val1, val2, guidAssignments)) {
                 return false;
             }
         }
 
-        return super.areEqualValues(val1, val2);
+        return super.areEqualValues(val1, val2, guidAssignments);
     }
 
     @Override

intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java

@@ -362,14 +362,14 @@ public class AtlasEntityType extends AtlasStructType {
     }
 
     @Override
-    public boolean areEqualValues(Object val1, Object val2) {
+    public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
         for (AtlasEntityType superType : superTypes) {
-            if (!superType.areEqualValues(val1, val2)) {
+            if (!superType.areEqualValues(val1, val2, guidAssignments)) {
                 return false;
             }
         }
 
-        return super.areEqualValues(val1, val2);
+        return super.areEqualValues(val1, val2, guidAssignments);
     }
 
     @Override

intg/src/main/java/org/apache/atlas/type/AtlasMapType.java

@@ -119,7 +119,7 @@ public class AtlasMapType extends AtlasType {
     }
 
     @Override
-    public boolean areEqualValues(Object val1, Object val2) {
+    public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
         boolean ret = true;
 
         if (val1 == null) {
@@ -143,7 +143,7 @@ public class AtlasMapType extends AtlasType {
                         ret = false;
                     } else {
                         for (Object key : map1.keySet()) {
-                            if (!valueType.areEqualValues(map1.get(key), map2.get(key))) {
+                            if (!valueType.areEqualValues(map1.get(key), map2.get(key), guidAssignments)) {
                                 ret = false;
 
                                 break;

intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java

@@ -172,7 +172,7 @@ public class AtlasRelationshipType extends AtlasStructType {
     }
 
     @Override
-    public boolean areEqualValues(Object val1, Object val2) {
+    public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
         final boolean ret;
 
         if (val1 == null) {
@@ -189,7 +189,7 @@ public class AtlasRelationshipType extends AtlasStructType {
 
                 if (rel2 == null) {
                     ret = false;
-                } else if (!super.areEqualValues(rel1, rel2)) {
+                } else if (!super.areEqualValues(rel1, rel2, guidAssignments)) {
                     ret = false;
                 } else {
                     ret = Objects.equals(rel1.getGuid(), rel2.getGuid()) &&

intg/src/main/java/org/apache/atlas/type/AtlasStructType.java

@@ -244,7 +244,7 @@ public class AtlasStructType extends AtlasType {
     }
 
     @Override
-    public boolean areEqualValues(Object val1, Object val2) {
+    public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
         boolean ret = true;
 
         if (val1 == null) {
@@ -264,17 +264,11 @@ public class AtlasStructType extends AtlasType {
                 } else if (!StringUtils.equalsIgnoreCase(structVal1.getTypeName(), structVal2.getTypeName())) {
                     ret = false;
                 } else {
-                    for (Map.Entry<String, Object> entry : structVal1.getAttributes().entrySet()) {
-                        String         attrName  = entry.getKey();
-                        AtlasAttribute attribute = getAttribute(attrName);
+                    for (AtlasAttribute attribute : getAllAttributes().values()) {
+                        Object attrValue1 = structVal1.getAttribute(attribute.getName());
+                        Object attrValue2 = structVal2.getAttribute(attribute.getName());
 
-                        if (attribute == null) { // ignore unknown attribute
-                            continue;
-                        } else {
-                            Object attrValue1 = entry.getValue();
-                            Object attrValue2 = structVal2.getAttribute(attrName);
-
-                            if (!attribute.getAttributeType().areEqualValues(attrValue1, attrValue2)) {
+                        if (!attribute.getAttributeType().areEqualValues(attrValue1, attrValue2, guidAssignments)) {
                             ret = false;
 
                             break;
@@ -283,7 +277,6 @@ public class AtlasStructType extends AtlasType {
                 }
             }
         }
-        }
 
         return ret;
     }

intg/src/main/java/org/apache/atlas/type/AtlasType.java

@@ -27,6 +27,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 
@@ -79,7 +80,7 @@ public abstract class AtlasType {
 
     public abstract boolean isValidValue(Object obj);
 
-    public boolean areEqualValues(Object val1, Object val2) {
+    public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
         final boolean ret;
 
         if (val1 == null) {

intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java

@@ -43,38 +43,6 @@ public class AtlasEntityUtil {
     private static final int    SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
     private static final int    SOFT_REFERENCE_FORMAT_INDEX_GUID      = 1;
 
-    public static boolean hasAnyAttributeUpdate(AtlasEntityType entityType, AtlasEntity currEntity, AtlasEntity newEntity) {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("==> hasAnyAttributeUpdate(guid={}, typeName={})", currEntity.getGuid(), currEntity.getTypeName());
-        }
-
-        boolean ret = false;
-
-        for (AtlasAttribute attribute : entityType.getAllAttributes().values()) {
-            String    attrName  = attribute.getName();
-            AtlasType attrType  = attribute.getAttributeType();
-            Object    currValue = currEntity.getAttribute(attrName);
-            Object    newValue  = newEntity.getAttribute(attrName);
-
-            if (!attrType.areEqualValues(currEntity.getAttribute(attrName), newEntity.getAttribute(attrName))) {
-                ret = true;
-
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("hasAnyAttributeUpdate(guid={}, typeName={}): attribute '{}' is found updated - currentValue={}, newValue={}",
-                            currEntity.getGuid(), currEntity.getTypeName(), attrName, currValue, newValue);
-                }
-
-                break;
-            }
-        }
-
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("<== hasAnyAttributeUpdate(guid={}, typeName={}): ret={}", currEntity.getGuid(), currEntity.getTypeName(), ret);
-        }
-
-        return ret;
-    }
-
 
     public static String formatSoftRefValue(String typeName, String guid) {
         return String.format(SOFT_REF_FORMAT, typeName, guid);

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java

@@ -40,7 +40,6 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.type.AtlasTypeUtil;
-import org.apache.atlas.utils.AtlasEntityUtil;
 import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.commons.collections.CollectionUtils;
@@ -694,25 +693,69 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
 
             // for existing entities, skip update if incoming entity doesn't have any change
             if (CollectionUtils.isNotEmpty(context.getUpdatedEntities())) {
+                MetricRecorder checkForUnchangedEntities = RequestContext.get().startMetricRecord("checkForUnchangedEntities");
+
                 List<AtlasEntity> entitiesToSkipUpdate = null;
 
                 for (AtlasEntity entity : context.getUpdatedEntities()) {
                     String          guid       = entity.getGuid();
                     AtlasVertex     vertex     = context.getVertex(guid);
-                    AtlasEntity     entityInStore = entityRetriever.toAtlasEntity(vertex);
                     AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
+                    boolean         hasUpdates = false;
+
+                    if (MapUtils.isNotEmpty(entity.getRelationshipAttributes())) {
+                        hasUpdates = true; // if relationship attributes are provided, assume there is an update
+                    }
+
+                    if (!hasUpdates) {
+                        hasUpdates = entity.getStatus() == AtlasEntity.Status.DELETED; // entity status could be updated during import
+                    }
+
+                    if (!hasUpdates) {
+                        for (AtlasAttribute attribute : entityType.getAllAttributes().values()) {
+                            if (!entity.getAttributes().containsKey(attribute.getName())) {  // if value is not provided, current value will not be updated
+                                continue;
+                            }
+
+                            Object newVal  = entity.getAttribute(attribute.getName());
+                            Object currVal = entityRetriever.getEntityAttribute(vertex, attribute);
+
+                            if (!attribute.getAttributeType().areEqualValues(currVal, newVal, context.getGuidAssignments())) {
+                                hasUpdates = true;
+
+                                if (LOG.isDebugEnabled()) {
+                                    LOG.debug("found attribute update: entity(guid={}, typeName={}), attrName={}, currValue={}, newValue={}", guid, entity.getTypeName(), attribute.getName(), currVal, newVal);
+                                }
+
+                                break;
+                            }
+                        }
+                    }
+
+                    // if classifications are to be replaced, then skip updates only when no change in classifications
+                    if (!hasUpdates && replaceClassifications) {
+                        List<AtlasClassification> newVal  = entity.getClassifications();
+                        List<AtlasClassification> currVal = entityRetriever.getAllClassifications(vertex);
+
+                        if (!Objects.equals(currVal, newVal)) {
+                            hasUpdates = true;
+
+                            if (LOG.isDebugEnabled()) {
+                                LOG.debug("found classifications update: entity(guid={}, typeName={}), currValue={}, newValue={}", guid, entity.getTypeName(), currVal, newVal);
+                            }
+                        }
+                    }
 
-                    if (!AtlasEntityUtil.hasAnyAttributeUpdate(entityType, entity, entityInStore)) {
-                        // if classifications are to be replaced as well, then skip updates only when no change in classifications as well
-                        if (!replaceClassifications || Objects.equals(entity.getClassifications(), entityInStore.getClassifications())) {
+                    if (!hasUpdates) {
                         if (entitiesToSkipUpdate == null) {
                             entitiesToSkipUpdate = new ArrayList<>();
                         }
 
+                        LOG.info("skipping unchanged entity: {}", entity);
+
                         entitiesToSkipUpdate.add(entity);
                     }
                 }
-                }
 
                 if (entitiesToSkipUpdate != null) {
                     context.getUpdatedEntities().removeAll(entitiesToSkipUpdate);
@@ -725,6 +768,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
                                                              "update entity: type=", entity.getTypeName());
                     }
                 }
+
+                RequestContext.get().endMetricRecord(checkForUnchangedEntities);
             }
 
             EntityMutationResponse ret = entityGraphMapper.mapAttributesAndClassifications(context, isPartialUpdate, replaceClassifications);

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java

@@ -197,6 +197,18 @@ public class EntityGraphRetriever {
         return ret;
     }
 
+    public Object getEntityAttribute(AtlasVertex entityVertex, AtlasAttribute attribute) {
+        Object ret = null;
+
+        try {
+            ret = getVertexAttribute(entityVertex, attribute);
+        } catch (AtlasBaseException excp) {
+            // ignore
+        }
+
+        return ret;
+    }
+
     public AtlasEntityHeader toAtlasEntityHeader(AtlasEntity entity) {
         AtlasEntityHeader ret        = null;
         String            typeName   = entity.getTypeName();

repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java

@@ -388,6 +388,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
         //Drop the first key and change the class type as well to col0
         columnsMap.clear();
         columnsMap.put("col0", AtlasTypeUtil.getAtlasObjectId(col0));
+        tableEntity.setAttribute(TestUtilsV2.COLUMNS_MAP, columnsMap);
         init();
 
         response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
@@ -420,6 +421,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
 
         //Remove an entry
         paramsMap.remove("key1");
+        tableEntity.setAttribute("parametersMap", paramsMap);
         init();
         response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
         validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 1);
@@ -446,21 +448,24 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
 
         //add a new element to array of struct
         partitions.add(new AtlasStruct(TestUtilsV2.PARTITION_STRUCT_TYPE, TestUtilsV2.NAME, "part3"));
+        tableEntity.setAttribute("partitions", partitions);
         init();
         response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
         updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
         validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
 
         //remove one of the struct values
-        init();
         partitions.remove(1);
+        tableEntity.setAttribute("partitions", partitions);
+        init();
         response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
         updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
         validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
 
         //Update struct value within array of struct
-        init();
         partitions.get(0).setAttribute(TestUtilsV2.NAME, "part4");
+        tableEntity.setAttribute("partitions", partitions);
+        init();
         response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
         updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
         validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
@@ -468,6 +473,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
 
         //add a repeated element to array of struct
         partitions.add(new AtlasStruct(TestUtilsV2.PARTITION_STRUCT_TYPE, TestUtilsV2.NAME, "part4"));
+        tableEntity.setAttribute("partitions", partitions);
         init();
         response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
         updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
@@ -475,6 +481,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
 
         // Remove all elements. Should set array attribute to null
         partitions.clear();
+        tableEntity.setAttribute("partitions", partitions);
         init();
         response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
         updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);

webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java

@@ -51,7 +51,7 @@ public class AdminExportImportTestIT extends BaseResourceIT {
     static final String IMPORT_TRANSFORM_CLEAR_ATTRS =
             "{ \"Asset\": { \"*\":[ \"clearAttrValue:replicatedTo,replicatedFrom\" ] } }";
     static final String IMPORT_TRANSFORM_SET_DELETED =
-            "{ \"Asset\": { \"*\":[ \"setDeleted\" ] } }";
+            "{ \"Referenceable\": { \"*\":[ \"setDeleted\" ] } }";
 
     @Test
     public void isActive() throws AtlasServiceException {