Skip to content

A
atlas
Commit 6e7aa6ed by Sarath Subramanian
Options

ATLAS-2662: Remove complex array and map attribute's edge information from entity vertex

parent a8fab3e7
Showing with 393 additions and 59 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -126,6 +126,8 @@ public final class Constants {
public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY = STATE_PROPERTY_KEY;
public static final String CLASSIFICATION_LABEL = "classifiedAs";
public static final String TERM_ASSIGNMENT_LABEL = "r:AtlasGlossarySemanticAssignment";
public static final String ATTRIBUTE_INDEX_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "index";
public static final String ATTRIBUTE_KEY_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "key";
public static final String VERTEX_ID_IN_IMPORT_KEY = "__vIdInImport";
public static final String EDGE_ID_IN_IMPORT_KEY = "__eIdInImport";
......
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -142,6 +142,7 @@ public enum AtlasErrorCode {
RELATIONSHIP_END_IS_NULL(400, "ATLAS-400-00-07D", "Relationship end is invalid. Expected {0} but is NULL"),
INVALID_TERM_RELATION_TO_SELF(400, "ATLAS-400-00-07E", "Invalid Term relationship: Term can't have a relationship with self"),
INVALID_CHILD_CATEGORY_DIFFERENT_GLOSSARY(400, "ATLAS-400-00-07F", "Invalid child category relationship: Child category (guid = {0}) belongs to different glossary"),
ATTRIBUTE_TYPE_INVALID(400, "ATLAS-400-00-080", "{0}.{1}: invalid attribute type. Attribute cannot be of type classification"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java
......@@ -34,6 +34,8 @@ import javax.xml.bind.annotation.XmlSeeAlso;
import org.apache.atlas.model.PList;
import org.apache.atlas.model.SearchFilter.SortType;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.commons.lang.StringUtils;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
......@@ -163,13 +165,12 @@ public class AtlasObjectId implements Serializable {
AtlasObjectId that = (AtlasObjectId) o;
// if guid is null, equality should be based on typeName/uniqueAttributes
if (guid != null && Objects.equals(guid, that.guid)) {
return true;
// if guid is empty/null, equality should be based on typeName/uniqueAttributes
if (StringUtils.isEmpty(guid) && StringUtils.isEmpty(that.guid)) {
return Objects.equals(typeName, that.typeName) && Objects.equals(uniqueAttributes, that.uniqueAttributes);
} else {
return Objects.equals(guid, that.guid);
}
return Objects.equals(typeName, that.typeName) &&
Objects.equals(uniqueAttributes, that.uniqueAttributes);
}
@Override
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -98,6 +98,17 @@ public class AtlasStructType extends AtlasType {
arrayType.setMaxCount(attributeDef.getValuesMaxCount());
}
//check if attribute type is not classification
if (attrType instanceof AtlasArrayType) {
attrType = ((AtlasArrayType) attrType).getElementType();
} else if (attrType instanceof AtlasMapType) {
attrType = ((AtlasMapType) attrType).getValueType();
}
if (attrType instanceof AtlasClassificationType) {
throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_TYPE_INVALID, getTypeName(), attributeDef.getName());
}
a.put(attributeDef.getName(), attribute);
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -26,12 +26,13 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.v1.model.instance.Id;
import org.apache.atlas.v1.model.instance.Referenceable;
......@@ -54,6 +55,7 @@ import org.apache.atlas.exception.EntityNotFoundException;
import org.apache.atlas.util.AttributeValueMap;
import org.apache.atlas.util.IndexedInstance;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -75,6 +77,8 @@ import java.util.Set;
import java.util.UUID;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
......@@ -82,6 +86,7 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PRO
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_NAME_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
......@@ -1170,6 +1175,11 @@ public final class GraphHelper {
return AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_ENTITY_GUID, String.class);
}
public static Integer getIndexValue(AtlasEdge edge) {
Integer index = edge.getProperty(ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.class);
return (index == null) ? 0: index;
}
public static boolean isPropagatedClassificationEdge(AtlasEdge edge) {
boolean ret = false;
......@@ -1557,44 +1567,100 @@ public final class GraphHelper {
return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX);
}
public static void setArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(AtlasGraphUtilsV1.isReference(elementType)) {
setListPropertyFromElementIds(instanceVertex, actualPropertyName, (List)values);
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (!AtlasGraphUtilsV1.isReference(elementType)) {
return instanceVertex.getProperty(vertexPropertyName, Object.class);
}
else {
setProperty(instanceVertex, actualPropertyName, values);
return null;
}
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (isReference(elementType)) {
return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
} else {
return (List) instanceVertex.getListProperty(encodedPropertyName);
}
}
public static void setMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, Object value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(AtlasGraphUtilsV1.isReference(elementType)) {
instanceVertex.setPropertyFromElementId(actualPropertyName, (AtlasEdge)value);
public static Map<String, Object> getMapElementsProperty(AtlasMapType mapType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
AtlasType mapValueType = mapType.getValueType();
if (isReference(mapValueType)) {
return getReferenceMap(instanceVertex, attribute);
} else {
return getPrimitiveMap(instanceVertex, propertyName, mapValueType);
}
else {
instanceVertex.setProperty(actualPropertyName, value);
}
// map elements for reference types - AtlasObjectId, AtlasStruct
public static Map<String, Object> getReferenceMap(AtlasVertex instanceVertex, AtlasAttribute attribute) {
Map<String, Object> ret = new HashMap<>();
List<AtlasEdge> referenceEdges = getCollectionElementsUsingRelationship(instanceVertex, attribute);
for (AtlasEdge edge : referenceEdges) {
String key = edge.getProperty(ATTRIBUTE_KEY_PROPERTY_KEY, String.class);
if (StringUtils.isNotEmpty(key)) {
ret.put(key, edge);
}
}
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
return ret;
}
if (AtlasGraphUtilsV1.isReference(elementType)) {
return instanceVertex.getProperty(vertexPropertyName, AtlasEdge.class);
} else {
return instanceVertex.getProperty(vertexPropertyName, Object.class);
public static List<AtlasEdge> getMapValuesUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute) {
String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
Iterator<AtlasEdge> edgesForLabel = getEdgesForLabel(vertex, edgeLabel, edgeDirection);
return (List<AtlasEdge>) IteratorUtils.toList(edgesForLabel);
}
// map elements for primitive types
public static Map<String,Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName, AtlasType mapValueType) {
String encodedPropertyName = encodePropertyKey(propertyName);
List<String> currentKeys = getListProperty(instanceVertex, encodedPropertyName);
Map<String, Object> ret = new HashMap<>();
if (CollectionUtils.isNotEmpty(currentKeys)) {
for (String key : currentKeys) {
String propertyNameForKey = getQualifiedNameForMapKey(encodedPropertyName, encodePropertyKey(key));
Object propertyValueForKey = getMapValueProperty(mapValueType, instanceVertex, propertyNameForKey);
ret.put(key, propertyValueForKey);
}
}
// newly added
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(AtlasGraphUtilsV1.isReference(elementType)) {
return (List)instanceVertex.getListProperty(encodedPropertyName, AtlasEdge.class);
return ret;
}
else {
return (List)instanceVertex.getListProperty(encodedPropertyName);
public static List<AtlasEdge> getCollectionElementsUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute) {
List<AtlasEdge> ret;
String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
Iterator<AtlasEdge> edgesForLabel = getEdgesForLabel(vertex, edgeLabel, edgeDirection);
ret = IteratorUtils.toList(edgesForLabel);
sortCollectionElements(attribute, ret);
return ret;
}
private static void sortCollectionElements(AtlasAttribute attribute, List<AtlasEdge> edges) {
// sort array elements based on edge index
if (attribute.getAttributeType() instanceof AtlasArrayType && CollectionUtils.isNotEmpty(edges)) {
Collections.sort(edges, (e1, e2) -> {
Integer e1Index = getIndexValue(e1);
Integer e2Index = getIndexValue(e2);
return e1Index.compareTo(e2Index);
});
}
}
......@@ -1730,7 +1796,7 @@ public final class GraphHelper {
}
public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) throws AtlasException {
public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
instanceVertex.setListProperty(actualPropertyName, value);
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
......@@ -147,7 +147,6 @@ public class AtlasGraphUtilsV1 {
public static boolean isReference(TypeCategory typeCategory) {
return typeCategory == TypeCategory.STRUCT ||
typeCategory == TypeCategory.ENTITY ||
typeCategory == TypeCategory.CLASSIFICATION ||
typeCategory == TypeCategory.OBJECT_ID_TYPE;
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
......@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.utils.AtlasJson;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -97,15 +98,19 @@ import static org.apache.atlas.repository.graph.GraphHelper.getAdjacentEdgesByLa
import static org.apache.atlas.repository.graph.GraphHelper.getAllClassificationEdges;
import static org.apache.atlas.repository.graph.GraphHelper.getAllTraitNames;
import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds;
import static org.apache.atlas.repository.graph.GraphHelper.getArrayElementsProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
import static org.apache.atlas.repository.graph.GraphHelper.getGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPrimitiveMap;
import static org.apache.atlas.repository.graph.GraphHelper.getReferenceMap;
import static org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
import static org.apache.atlas.repository.graph.GraphHelper.getRelationshipGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
......@@ -614,10 +619,10 @@ public final class EntityGraphRetriever {
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection);
break;
case ARRAY:
ret = mapVertexToArray(entityVertex, (AtlasArrayType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, edgeDirection);
ret = mapVertexToArray(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
break;
case MAP:
ret = mapVertexToMap(entityVertex, (AtlasMapType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, edgeDirection);
ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
break;
case CLASSIFICATION:
// do nothing
......@@ -627,45 +632,50 @@ public final class EntityGraphRetriever {
return ret;
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, final String propertyName, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName);
if (CollectionUtils.isEmpty(mapKeys)) {
return null;
}
Map<String, Object> ret = null;
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
AtlasType mapValueType = mapType.getValueType();
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping map attribute {} for vertex {}", atlasMapType.getTypeName(), entityVertex);
LOG.debug("Mapping map attribute {} for vertex {}", mapType.getTypeName(), entityVertex);
}
Map<String, Object> ret = new HashMap<>(mapKeys.size());
AtlasType mapValueType = atlasMapType.getValueType();
for (String mapKey : mapKeys) {
final String keyPropertyName = propertyName + "." + mapKey;
final String edgeLabel = EDGE_LABEL_PREFIX + keyPropertyName;
final Object keyValue = GraphHelper.getMapValueProperty(mapValueType, entityVertex, keyPropertyName);
if (isReference(mapValueType)) {
Map<String, Object> currentMap = getReferenceMap(entityVertex, attribute);
Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, edgeLabel,
entityExtInfo, isOwnedAttribute, edgeDirection);
if (MapUtils.isNotEmpty(currentMap)) {
ret = new HashMap<>();
for (Map.Entry<String, Object> entry : currentMap.entrySet()) {
String mapKey = entry.getKey();
Object keyValue = entry.getValue();
Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, attribute.getRelationshipEdgeLabel(),
entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection());
if (mapValue != null) {
ret.put(mapKey, mapValue);
}
}
}
} else {
ret = getPrimitiveMap(entityVertex, propertyName, mapValueType);
}
if (MapUtils.isEmpty(ret)) {
ret = null;
}
return ret;
}
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasArrayType arrayType, String propertyName,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
private List<Object> mapVertexToArray(AtlasVertex entityVertex, String propertyName, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType();
List<Object> arrayElements = GraphHelper.getArrayElementsProperty(arrayElementType, entityVertex, propertyName);
List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, propertyName, attribute);
if (CollectionUtils.isEmpty(arrayElements)) {
return null;
......@@ -676,7 +686,8 @@ public final class EntityGraphRetriever {
}
List arrValues = new ArrayList(arrayElements.size());
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
for (Object element : arrayElements) {
// When internal types are deleted, sometimes the collection type attribute will contain a null value
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment