ATLAS-3035: updated entity-get/delete to retrieve/delete soft-ref entities

intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java

@@ -19,24 +19,30 @@ package org.apache.atlas.utils;
 
 
 import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.model.instance.AtlasObjectId;
 import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasMapType;
-import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
 import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
 
 public class AtlasEntityUtil {
     private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityUtil.class);
 
+    private static final String SOFT_REFERENCE_FORMAT_SEPERATOR       = ":";
+    private static final String SOFT_REF_FORMAT                       = "%s" + SOFT_REFERENCE_FORMAT_SEPERATOR + "%s";
+    private static final int    SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
+    private static final int    SOFT_REFERENCE_FORMAT_INDEX_GUID      = 1;
+
     public static boolean hasAnyAttributeUpdate(AtlasEntityType entityType, AtlasEntity currEntity, AtlasEntity newEntity) {
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> hasAnyAttributeUpdate(guid={}, typeName={})", currEntity.getGuid(), currEntity.getTypeName());
@@ -68,4 +74,85 @@ public class AtlasEntityUtil {
 
         return ret;
     }
+
+
+    public static String formatSoftRefValue(String typeName, String guid) {
+        return String.format(SOFT_REF_FORMAT, typeName, guid);
+    }
+
+    public static String formatSoftRefValue(AtlasObjectId objectId) {
+        return formatSoftRefValue(objectId.getTypeName(), objectId.getGuid());
+    }
+
+    public static List<String> formatSoftRefValue(List<AtlasObjectId> objIds) {
+        List<String> ret = new ArrayList<>();
+
+        for (AtlasObjectId objId : objIds) {
+            ret.add(formatSoftRefValue(objId));
+        }
+
+        return ret;
+    }
+
+    public static Map<String, String> formatSoftRefValue(Map<String, AtlasObjectId> objIdMap) {
+        Map<String, String> ret = new HashMap<>();
+
+        for (Map.Entry<String, AtlasObjectId> entry : objIdMap.entrySet()) {
+            ret.put(entry.getKey(), formatSoftRefValue(entry.getValue()));
+        }
+
+        return ret;
+    }
+
+    public static AtlasObjectId parseSoftRefValue(String softRefValue) {
+        AtlasObjectId ret = null;
+
+        if (StringUtils.isNotEmpty(softRefValue)) {
+            String[] objectIdParts = StringUtils.split(softRefValue, SOFT_REFERENCE_FORMAT_SEPERATOR);
+
+            if(objectIdParts.length >= 2) {
+                ret = new AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID], objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]);
+            } else {
+                LOG.warn("Invalid soft-ref value: {}", softRefValue);
+            }
+        }
+
+        return ret;
+    }
+
+    public static List<AtlasObjectId> parseSoftRefValue(List<String> softRefValue) {
+        List<AtlasObjectId> ret = null;
+
+        if (CollectionUtils.isNotEmpty(softRefValue)) {
+            ret = new ArrayList<>();
+
+            for (String elemValue : softRefValue) {
+                AtlasObjectId objId = parseSoftRefValue(elemValue);
+
+                if (objId != null) {
+                    ret.add(objId);
+                }
+            }
+        }
+
+        return ret;
+    }
+
+    public static Map<String, AtlasObjectId> parseSoftRefValue(Map<String, String> softRefValue) {
+        Map<String, AtlasObjectId> ret = null;
+
+        if (MapUtils.isNotEmpty(softRefValue)) {
+            ret = new HashMap<>();
+
+            for (Map.Entry<String, String> entry : softRefValue.entrySet()) {
+                AtlasObjectId objId = parseSoftRefValue(entry.getValue());
+
+                if (objId != null) {
+                    ret.put(entry.getKey(), objId);
+                }
+            }
+        }
+
+        return ret;
+    }
 }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java

@@ -45,6 +45,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
 import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.AtlasEntityUtil;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
@@ -210,14 +211,23 @@ public abstract class DeleteHandlerV1 {
                 TypeCategory typeCategory = attrType.getTypeCategory();
 
                 if (typeCategory == OBJECT_ID_TYPE) {
-                    AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
+                    if (attributeInfo.getAttributeDef().isSoftReferenced()) {
+                        String        softRefVal = vertex.getProperty(attributeInfo.getVertexPropertyName(), String.class);
+                        AtlasObjectId refObjId   = AtlasEntityUtil.parseSoftRefValue(softRefVal);
+                        AtlasVertex   refVertex  = refObjId != null ? AtlasGraphUtilsV2.findByGuid(refObjId.getGuid()) : null;
 
-                    if (edge == null || getState(edge) == DELETED) {
-                        continue;
-                    }
+                        if (refVertex != null) {
+                            vertices.push(refVertex);
+                        }
+                    } else {
+                        AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
 
-                    vertices.push(edge.getInVertex());
+                        if (edge == null || getState(edge) == DELETED) {
+                            continue;
+                        }
 
+                        vertices.push(edge.getInVertex());
+                    }
                 } else if (typeCategory == ARRAY || typeCategory == MAP) {
                     TypeCategory elementType = null;
 
@@ -231,15 +241,46 @@ public abstract class DeleteHandlerV1 {
                         continue;
                     }
 
-                    List<AtlasEdge> edges = getCollectionElementsUsingRelationship(vertex, attributeInfo);
+                    if (attributeInfo.getAttributeDef().isSoftReferenced()) {
+                        if (typeCategory == ARRAY) {
+                            List                softRefVal = vertex.getListProperty(attributeInfo.getVertexPropertyName(), List.class);
+                            List<AtlasObjectId> refObjIds  = AtlasEntityUtil.parseSoftRefValue(softRefVal);
+
+                            if (CollectionUtils.isNotEmpty(refObjIds)) {
+                                for (AtlasObjectId refObjId : refObjIds) {
+                                    AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(refObjId.getGuid());
+
+                                    if (refVertex != null) {
+                                        vertices.push(refVertex);
+                                    }
+                                }
+                            }
+                        } else if (typeCategory == MAP) {
+                            Map                        softRefVal = vertex.getProperty(attributeInfo.getVertexPropertyName(), Map.class);
+                            Map<String, AtlasObjectId> refObjIds  = AtlasEntityUtil.parseSoftRefValue(softRefVal);
+
+                            if (MapUtils.isNotEmpty(refObjIds)) {
+                                for (AtlasObjectId refObjId : refObjIds.values()) {
+                                    AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(refObjId.getGuid());
 
-                    if (CollectionUtils.isNotEmpty(edges)) {
-                        for (AtlasEdge edge : edges) {
-                            if (edge == null || getState(edge) == DELETED) {
-                                continue;
+                                    if (refVertex != null) {
+                                        vertices.push(refVertex);
+                                    }
+                                }
                             }
+                        }
+
+                    } else {
+                        List<AtlasEdge> edges = getCollectionElementsUsingRelationship(vertex, attributeInfo);
 
-                            vertices.push(edge.getInVertex());
+                        if (CollectionUtils.isNotEmpty(edges)) {
+                            for (AtlasEdge edge : edges) {
+                                if (edge == null || getState(edge) == DELETED) {
+                                    continue;
+                                }
+
+                                vertices.push(edge.getInVertex());
+                            }
                         }
                     }
                 }

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java

@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdg
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.utils.AtlasEntityUtil;
 import org.apache.atlas.utils.AtlasJson;
 import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -496,28 +497,27 @@ public class EntityGraphMapper {
         }
     }
 
-    private Object mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) {
-        if(ctx.getValue() != null && !(ctx.getValue() instanceof AtlasObjectId)) {
-            LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found: {}", ctx.getValue().getClass());
-            return null;
-        }
+    private String mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) {
+        String ret = null;
 
-        String softRefValue = null;
-        if(ctx.getValue() != null) {
+        if (ctx.getValue() instanceof AtlasObjectId) {
             AtlasObjectId objectId = (AtlasObjectId) ctx.getValue();
-            String resolvedGuid = AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid())
-                    ? context.getGuidAssignments().get(objectId.getGuid())
-                    : objectId.getGuid();
+            String        typeName = objectId.getTypeName();
+            String        guid     = AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid()) ? context.getGuidAssignments().get(objectId.getGuid()) : objectId.getGuid();
 
-            softRefValue = String.format(SOFT_REF_FORMAT, objectId.getTypeName(), resolvedGuid);
+            ret = AtlasEntityUtil.formatSoftRefValue(typeName, guid);
+        } else {
+            if (ctx.getValue() != null) {
+                LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found: {}", ctx.getValue().getClass());
+            }
         }
 
-        return softRefValue;
+        return ret;
     }
 
     private Object mapSoftRefValueWithUpdate(AttributeMutationContext ctx, EntityMutationContext context) {
+        String softRefValue = mapSoftRefValue(ctx, context);
 
-        String softRefValue = (String) mapSoftRefValue(ctx, context);
         AtlasGraphUtilsV2.setProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), softRefValue);
 
         return softRefValue;

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java

@@ -52,6 +52,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.utils.AtlasEntityUtil;
 import org.apache.atlas.utils.AtlasJson;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
@@ -71,6 +72,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -106,9 +108,6 @@ public class EntityGraphRetriever {
     public static final String DESCRIPTION    = "description";
     public static final String OWNER          = "owner";
     public static final String CREATE_TIME    = "createTime";
-    private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":";
-    private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
-    private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1;
     public static final String QUALIFIED_NAME = "qualifiedName";
 
     private static final TypeReference<List<TimeBoundary>> TIME_BOUNDARIES_LIST_TYPE = new TypeReference<List<TimeBoundary>>() {};
@@ -704,21 +703,21 @@ public class EntityGraphRetriever {
                 break;
             case OBJECT_ID_TYPE:
                 if(attribute.getAttributeDef().isSoftReferenced()) {
-                    ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute.getVertexPropertyName());
+                    ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
                 } else {
                 	ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
 				}
                 break;
             case ARRAY:
                 if(attribute.getAttributeDef().isSoftReferenced()) {
-                    ret = mapVertexToArrayForSoftRef(entityVertex, attribute.getVertexPropertyName());
+                    ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
                 } else {
                 	ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
 				}
                 break;
             case MAP:
                 if(attribute.getAttributeDef().isSoftReferenced()) {
-                    ret = mapVertexToMapForSoftRef(entityVertex, attribute.getVertexPropertyName());
+                    ret = mapVertexToMapForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
                 } else {
                 	ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
 				}
@@ -731,74 +730,78 @@ public class EntityGraphRetriever {
         return ret;
     }
 
-    private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex entityVertex, String propertyName) {
-        Map map = entityVertex.getProperty(propertyName, Map.class);
-        if (MapUtils.isEmpty(map)) {
-            return null;
-        }
+    private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex entityVertex,  AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) {
+        Map<String, AtlasObjectId> ret        = null;
+        Map                        softRefVal = entityVertex.getProperty(attribute.getVertexPropertyName(), Map.class);
 
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("Mapping map attribute {} for vertex {}", propertyName, entityVertex);
-        }
+        if (MapUtils.isNotEmpty(softRefVal)) {
+            ret = new HashMap<>();
 
-        Map ret = new HashMap();
-        for (Object mapKey : map.keySet()) {
-            String softRefRaw = (String) map.get(mapKey);
-            AtlasObjectId mapValue = getAtlasObjectIdFromSoftRefFormat(softRefRaw);
-            if (mapValue != null) {
-                ret.put(mapKey, mapValue);
+            for (Object mapKey : softRefVal.keySet()) {
+                AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat(Objects.toString(softRefVal.get(mapKey)), attribute, entityExtInfo, isMinExtInfo);
+
+                if (objectId != null) {
+                    ret.put(Objects.toString(mapKey), objectId);
+                }
             }
         }
 
         return ret;
     }
 
-    private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex entityVertex, String propertyName) {
-        List rawValue = entityVertex.getListProperty(propertyName, List.class);
-        if (CollectionUtils.isEmpty(rawValue)) {
-            return null;
-        }
+    private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) {
+        List<AtlasObjectId> ret        = null;
+        List                softRefVal = entityVertex.getListProperty(attribute.getVertexPropertyName(), List.class);
 
-        List list = (List) rawValue;
-        List<AtlasObjectId> objectIds = new ArrayList<>();
-        for (Object o : list) {
-            if (!(o instanceof String)) {
-                continue;
-            }
+        if (CollectionUtils.isNotEmpty(softRefVal)) {
+            ret = new ArrayList<>();
 
-            AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat((String) o);
-            if(objectId == null) {
-                continue;
-            }
+            for (Object o : softRefVal) {
+                AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat(Objects.toString(o), attribute, entityExtInfo, isMinExtInfo);
 
-            objectIds.add(objectId);
+                if(objectId != null) {
+                    ret.add(objectId);
+                }
+            }
         }
 
-        return objectIds;
+        return ret;
     }
 
-    private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, String vertexPropertyName) {
-        String rawValue = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, vertexPropertyName, String.class);
-        if(StringUtils.isEmpty(rawValue)) {
-            return null;
-        }
+    private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) {
+        String softRefVal = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, attribute.getVertexPropertyName(), String.class);
 
-        return getAtlasObjectIdFromSoftRefFormat(rawValue);
+        return StringUtils.isNotEmpty(softRefVal) ? getAtlasObjectIdFromSoftRefFormat(softRefVal, attribute, entityExtInfo, isMinExtInfo) : null;
     }
 
-    private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String rawValue) {
-        if(StringUtils.isEmpty(rawValue)) {
-            return null;
-        }
+    private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String softRefVal, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) {
+        AtlasObjectId ret = AtlasEntityUtil.parseSoftRefValue(softRefVal);
 
-        String[] objectIdParts = StringUtils.split(rawValue, SOFT_REFERENCE_FORMAT_SEPERATOR);
-        if(objectIdParts.length < 2) {
-            LOG.warn("Expecting value to be formatted for softRef. Instead found: {}", rawValue);
-            return null;
+        if(ret != null) {
+            if (entityExtInfo != null && attribute.isOwnedRef()) {
+                try {
+                    AtlasVertex referenceVertex = getEntityVertex(ret.getGuid());
+
+                    if (referenceVertex != null) {
+                        final AtlasEntity entity;
+
+                        if (isMinExtInfo) {
+                            entity = mapVertexToAtlasEntityMin(referenceVertex, entityExtInfo);
+                        } else {
+                            entity = mapVertexToAtlasEntity(referenceVertex, entityExtInfo);
+                        }
+
+                        if (entity != null) {
+                            ret = toAtlasObjectId(entity);
+                        }
+                    }
+                } catch (AtlasBaseException excp) {
+                    LOG.info("failed to retrieve soft-referenced entity(typeName={}, guid={}); errorCode={}. Ignoring", ret.getTypeName(), ret.getGuid(), excp.getAtlasErrorCode());
+                }
+            }
         }
 
-        return new AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID],
-                objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]);
+        return ret;
     }
 
     private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
@@ -986,7 +989,7 @@ public class EntityGraphRetriever {
                         ret = AtlasTypeUtil.getAtlasObjectId(entity);
                     }
                 } else {
-                    ret = new AtlasObjectId(getGuid(referenceVertex), getTypeName(referenceVertex));
+                    ret = toAtlasObjectId(referenceVertex);
                 }
             }
         }