Skip to content

A
atlas
Commit 2341d58a by Madhan Neethiraj
Options

ATLAS-3035: updated entity-get/delete to retrieve/delete soft-ref entities

parent 4985b9f2
Showing with 214 additions and 83 deletions
intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
......@@ -19,24 +19,30 @@ package org.apache.atlas.utils;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public class AtlasEntityUtil {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityUtil.class);
private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":";
private static final String SOFT_REF_FORMAT = "%s" + SOFT_REFERENCE_FORMAT_SEPERATOR + "%s";
private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1;
public static boolean hasAnyAttributeUpdate(AtlasEntityType entityType, AtlasEntity currEntity, AtlasEntity newEntity) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> hasAnyAttributeUpdate(guid={}, typeName={})", currEntity.getGuid(), currEntity.getTypeName());
......@@ -68,4 +74,85 @@ public class AtlasEntityUtil {
return ret;
}
public static String formatSoftRefValue(String typeName, String guid) {
return String.format(SOFT_REF_FORMAT, typeName, guid);
}
public static String formatSoftRefValue(AtlasObjectId objectId) {
return formatSoftRefValue(objectId.getTypeName(), objectId.getGuid());
}
public static List<String> formatSoftRefValue(List<AtlasObjectId> objIds) {
List<String> ret = new ArrayList<>();
for (AtlasObjectId objId : objIds) {
ret.add(formatSoftRefValue(objId));
}
return ret;
}
public static Map<String, String> formatSoftRefValue(Map<String, AtlasObjectId> objIdMap) {
Map<String, String> ret = new HashMap<>();
for (Map.Entry<String, AtlasObjectId> entry : objIdMap.entrySet()) {
ret.put(entry.getKey(), formatSoftRefValue(entry.getValue()));
}
return ret;
}
public static AtlasObjectId parseSoftRefValue(String softRefValue) {
AtlasObjectId ret = null;
if (StringUtils.isNotEmpty(softRefValue)) {
String[] objectIdParts = StringUtils.split(softRefValue, SOFT_REFERENCE_FORMAT_SEPERATOR);
if(objectIdParts.length >= 2) {
ret = new AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID], objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]);
} else {
LOG.warn("Invalid soft-ref value: {}", softRefValue);
}
}
return ret;
}
public static List<AtlasObjectId> parseSoftRefValue(List<String> softRefValue) {
List<AtlasObjectId> ret = null;
if (CollectionUtils.isNotEmpty(softRefValue)) {
ret = new ArrayList<>();
for (String elemValue : softRefValue) {
AtlasObjectId objId = parseSoftRefValue(elemValue);
if (objId != null) {
ret.add(objId);
}
}
}
return ret;
}
public static Map<String, AtlasObjectId> parseSoftRefValue(Map<String, String> softRefValue) {
Map<String, AtlasObjectId> ret = null;
if (MapUtils.isNotEmpty(softRefValue)) {
ret = new HashMap<>();
for (Map.Entry<String, String> entry : softRefValue.entrySet()) {
AtlasObjectId objId = parseSoftRefValue(entry.getValue());
if (objId != null) {
ret.put(entry.getKey(), objId);
}
}
}
return ret;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -45,6 +45,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
......@@ -210,14 +211,23 @@ public abstract class DeleteHandlerV1 {
TypeCategory typeCategory = attrType.getTypeCategory();
if (typeCategory == OBJECT_ID_TYPE) {
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
if (attributeInfo.getAttributeDef().isSoftReferenced()) {
String softRefVal = vertex.getProperty(attributeInfo.getVertexPropertyName(), String.class);
AtlasObjectId refObjId = AtlasEntityUtil.parseSoftRefValue(softRefVal);
AtlasVertex refVertex = refObjId != null ? AtlasGraphUtilsV2.findByGuid(refObjId.getGuid()) : null;
if (edge == null || getState(edge) == DELETED) {
continue;
}
if (refVertex != null) {
vertices.push(refVertex);
}
} else {
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
vertices.push(edge.getInVertex());
if (edge == null || getState(edge) == DELETED) {
continue;
}
vertices.push(edge.getInVertex());
}
} else if (typeCategory == ARRAY || typeCategory == MAP) {
TypeCategory elementType = null;
......@@ -231,15 +241,46 @@ public abstract class DeleteHandlerV1 {
continue;
}
List<AtlasEdge> edges = getCollectionElementsUsingRelationship(vertex, attributeInfo);
if (attributeInfo.getAttributeDef().isSoftReferenced()) {
if (typeCategory == ARRAY) {
List softRefVal = vertex.getListProperty(attributeInfo.getVertexPropertyName(), List.class);
List<AtlasObjectId> refObjIds = AtlasEntityUtil.parseSoftRefValue(softRefVal);
if (CollectionUtils.isNotEmpty(refObjIds)) {
for (AtlasObjectId refObjId : refObjIds) {
AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(refObjId.getGuid());
if (refVertex != null) {
vertices.push(refVertex);
}
}
}
} else if (typeCategory == MAP) {
Map softRefVal = vertex.getProperty(attributeInfo.getVertexPropertyName(), Map.class);
Map<String, AtlasObjectId> refObjIds = AtlasEntityUtil.parseSoftRefValue(softRefVal);
if (MapUtils.isNotEmpty(refObjIds)) {
for (AtlasObjectId refObjId : refObjIds.values()) {
AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(refObjId.getGuid());
if (CollectionUtils.isNotEmpty(edges)) {
for (AtlasEdge edge : edges) {
if (edge == null || getState(edge) == DELETED) {
continue;
if (refVertex != null) {
vertices.push(refVertex);
}
}
}
}
} else {
List<AtlasEdge> edges = getCollectionElementsUsingRelationship(vertex, attributeInfo);
vertices.push(edge.getInVertex());
if (CollectionUtils.isNotEmpty(edges)) {
for (AtlasEdge edge : edges) {
if (edge == null || getState(edge) == DELETED) {
continue;
}
vertices.push(edge.getInVertex());
}
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
......@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdg
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.utils.AtlasJson;
import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
import org.apache.commons.codec.digest.DigestUtils;
......@@ -496,28 +497,27 @@ public class EntityGraphMapper {
}
}
private Object mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) {
if(ctx.getValue() != null && !(ctx.getValue() instanceof AtlasObjectId)) {
LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found: {}", ctx.getValue().getClass());
return null;
}
private String mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) {
String ret = null;
String softRefValue = null;
if(ctx.getValue() != null) {
if (ctx.getValue() instanceof AtlasObjectId) {
AtlasObjectId objectId = (AtlasObjectId) ctx.getValue();
String resolvedGuid = AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid())
? context.getGuidAssignments().get(objectId.getGuid())
: objectId.getGuid();
String typeName = objectId.getTypeName();
String guid = AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid()) ? context.getGuidAssignments().get(objectId.getGuid()) : objectId.getGuid();
softRefValue = String.format(SOFT_REF_FORMAT, objectId.getTypeName(), resolvedGuid);
ret = AtlasEntityUtil.formatSoftRefValue(typeName, guid);
} else {
if (ctx.getValue() != null) {
LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found: {}", ctx.getValue().getClass());
}
}
return softRefValue;
return ret;
}
private Object mapSoftRefValueWithUpdate(AttributeMutationContext ctx, EntityMutationContext context) {
String softRefValue = mapSoftRefValue(ctx, context);
String softRefValue = (String) mapSoftRefValue(ctx, context);
AtlasGraphUtilsV2.setProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), softRefValue);
return softRefValue;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
......@@ -52,6 +52,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.utils.AtlasJson;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
......@@ -71,6 +72,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
......@@ -106,9 +108,6 @@ public class EntityGraphRetriever {
public static final String DESCRIPTION = "description";
public static final String OWNER = "owner";
public static final String CREATE_TIME = "createTime";
private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":";
private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1;
public static final String QUALIFIED_NAME = "qualifiedName";
private static final TypeReference<List<TimeBoundary>> TIME_BOUNDARIES_LIST_TYPE = new TypeReference<List<TimeBoundary>>() {};
......@@ -704,21 +703,21 @@ public class EntityGraphRetriever {
break;
case OBJECT_ID_TYPE:
if(attribute.getAttributeDef().isSoftReferenced()) {
ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute.getVertexPropertyName());
ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
}
break;
case ARRAY:
if(attribute.getAttributeDef().isSoftReferenced()) {
ret = mapVertexToArrayForSoftRef(entityVertex, attribute.getVertexPropertyName());
ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
}
break;
case MAP:
if(attribute.getAttributeDef().isSoftReferenced()) {
ret = mapVertexToMapForSoftRef(entityVertex, attribute.getVertexPropertyName());
ret = mapVertexToMapForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
}
......@@ -731,74 +730,78 @@ public class EntityGraphRetriever {
return ret;
}
private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex entityVertex, String propertyName) {
Map map = entityVertex.getProperty(propertyName, Map.class);
if (MapUtils.isEmpty(map)) {
return null;
}
private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) {
Map<String, AtlasObjectId> ret = null;
Map softRefVal = entityVertex.getProperty(attribute.getVertexPropertyName(), Map.class);
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping map attribute {} for vertex {}", propertyName, entityVertex);
}
if (MapUtils.isNotEmpty(softRefVal)) {
ret = new HashMap<>();
Map ret = new HashMap();
for (Object mapKey : map.keySet()) {
String softRefRaw = (String) map.get(mapKey);
AtlasObjectId mapValue = getAtlasObjectIdFromSoftRefFormat(softRefRaw);
if (mapValue != null) {
ret.put(mapKey, mapValue);
for (Object mapKey : softRefVal.keySet()) {
AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat(Objects.toString(softRefVal.get(mapKey)), attribute, entityExtInfo, isMinExtInfo);
if (objectId != null) {
ret.put(Objects.toString(mapKey), objectId);
}
}
}
return ret;
}
private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex entityVertex, String propertyName) {
List rawValue = entityVertex.getListProperty(propertyName, List.class);
if (CollectionUtils.isEmpty(rawValue)) {
return null;
}
private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) {
List<AtlasObjectId> ret = null;
List softRefVal = entityVertex.getListProperty(attribute.getVertexPropertyName(), List.class);
List list = (List) rawValue;
List<AtlasObjectId> objectIds = new ArrayList<>();
for (Object o : list) {
if (!(o instanceof String)) {
continue;
}
if (CollectionUtils.isNotEmpty(softRefVal)) {
ret = new ArrayList<>();
AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat((String) o);
if(objectId == null) {
continue;
}
for (Object o : softRefVal) {
AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat(Objects.toString(o), attribute, entityExtInfo, isMinExtInfo);
objectIds.add(objectId);
if(objectId != null) {
ret.add(objectId);
}
}
}
return objectIds;
return ret;
}
private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, String vertexPropertyName) {
String rawValue = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, vertexPropertyName, String.class);
if(StringUtils.isEmpty(rawValue)) {
return null;
}
private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) {
String softRefVal = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, attribute.getVertexPropertyName(), String.class);
return getAtlasObjectIdFromSoftRefFormat(rawValue);
return StringUtils.isNotEmpty(softRefVal) ? getAtlasObjectIdFromSoftRefFormat(softRefVal, attribute, entityExtInfo, isMinExtInfo) : null;
}
private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String rawValue) {
if(StringUtils.isEmpty(rawValue)) {
return null;
}
private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String softRefVal, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) {
AtlasObjectId ret = AtlasEntityUtil.parseSoftRefValue(softRefVal);
String[] objectIdParts = StringUtils.split(rawValue, SOFT_REFERENCE_FORMAT_SEPERATOR);
if(objectIdParts.length < 2) {
LOG.warn("Expecting value to be formatted for softRef. Instead found: {}", rawValue);
return null;
if(ret != null) {
if (entityExtInfo != null && attribute.isOwnedRef()) {
try {
AtlasVertex referenceVertex = getEntityVertex(ret.getGuid());
if (referenceVertex != null) {
final AtlasEntity entity;
if (isMinExtInfo) {
entity = mapVertexToAtlasEntityMin(referenceVertex, entityExtInfo);
} else {
entity = mapVertexToAtlasEntity(referenceVertex, entityExtInfo);
}
if (entity != null) {
ret = toAtlasObjectId(entity);
}
}
} catch (AtlasBaseException excp) {
LOG.info("failed to retrieve soft-referenced entity(typeName={}, guid={}); errorCode={}. Ignoring", ret.getTypeName(), ret.getGuid(), excp.getAtlasErrorCode());
}
}
}
return new AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID],
objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]);
return ret;
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
......@@ -986,7 +989,7 @@ public class EntityGraphRetriever {
ret = AtlasTypeUtil.getAtlasObjectId(entity);
}
} else {
ret = new AtlasObjectId(getGuid(referenceVertex), getTypeName(referenceVertex));
ret = toAtlasObjectId(referenceVertex);
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment