Commit 8be1be67 by Madhan Neethiraj

ATLAS-1522: entity type attributes (like hive_table.sd, hive_table.columns)…

ATLAS-1522: entity type attributes (like hive_table.sd, hive_table.columns) should use AtlasObjectId as value instead of entire entity contents
parent 169ab553
......@@ -48,7 +48,7 @@ public enum AtlasErrorCode {
INVALID_ATTRIBUTE_TYPE_FOR_CARDINALITY(400, "ATLAS40018E", "Cardinality of attribute {0}.{1} requires a list or set type"),
ATTRIBUTE_UNIQUE_INVALID(400, "ATLAS40019E", "Type {0} with unique attribute {1} does not exist"),
TYPE_NAME_INVALID(400, "ATLAS40020E", "Type {0} with name {1} does not exist"),
TYPE_CATEGORY_INVALID(400, "ATLAS40021E", "Type Category {0} does not match {1} or is invalid"),
TYPE_CATEGORY_INVALID(400, "ATLAS40021E", "Type Category {0} is invalid"),
PATCH_NOT_APPLICABLE_FOR_TYPE(400, "ATLAS40022E", "{0} - invalid patch for type {1}"),
PATCH_FOR_UNKNOWN_TYPE(400, "ATLAS40023E", "{0} - patch references unknown type {1}"),
PATCH_INVALID_DATA(400, "ATLAS40024E", "{0} - patch data is invalid for type {1}"),
......
......@@ -264,6 +264,23 @@ public class AtlasArrayType extends AtlasType {
return ret;
}
@Override
public AtlasType getTypeForAttribute() {
AtlasType elementAttributeType = elementType.getTypeForAttribute();
if (elementAttributeType == elementType) {
return this;
} else {
AtlasType attributeType = new AtlasArrayType(elementAttributeType, minCount, maxCount);
if (LOG.isDebugEnabled()) {
LOG.debug("getTypeForAttribute(): {} ==> {}", getTypeName(), attributeType.getTypeName());
}
return attributeType;
}
}
private boolean isValidElementCount(int count) {
if (minCount != COUNT_NOT_SET) {
if (count < minCount) {
......
......@@ -513,13 +513,27 @@ public class AtlasBuiltInTypes {
* class that implements behaviour of Atlas object-id type.
*/
public static class AtlasObjectIdType extends AtlasType {
public static final String DEFAULT_UNASSIGNED_GUID = "-1";
private final String objectType;
public AtlasObjectIdType() {
super(AtlasBaseTypeDef.ATLAS_TYPE_OBJECT_ID, TypeCategory.OBJECT_ID_TYPE);
objectType = AtlasBaseTypeDef.ATLAS_TYPE_ASSET;
}
public AtlasObjectIdType(String objectType) {
super(AtlasBaseTypeDef.ATLAS_TYPE_OBJECT_ID, TypeCategory.OBJECT_ID_TYPE);
this.objectType = objectType;
}
public String getObjectType() { return objectType; }
@Override
public AtlasObjectId createDefaultValue() {
return new AtlasObjectId("-1", AtlasBaseTypeDef.ATLAS_TYPE_ASSET);
return new AtlasObjectId(DEFAULT_UNASSIGNED_GUID, objectType);
}
@Override
......
......@@ -24,6 +24,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
......@@ -216,6 +217,17 @@ public class AtlasEntityType extends AtlasStructType {
return ret;
}
@Override
public AtlasType getTypeForAttribute() {
AtlasType attributeType = new AtlasObjectIdType(getTypeName());
if (LOG.isDebugEnabled()) {
LOG.debug("getTypeForAttribute(): {} ==> {}", getTypeName(), attributeType.getTypeName());
}
return attributeType;
}
public void normalizeAttributeValues(AtlasEntity ent) {
if (ent != null) {
for (AtlasEntityType superType : superTypes) {
......
......@@ -192,4 +192,22 @@ public class AtlasMapType extends AtlasType {
return ret;
}
@Override
public AtlasType getTypeForAttribute() {
AtlasType keyAttributeType = keyType.getTypeForAttribute();
AtlasType valueAttributeType = valueType.getTypeForAttribute();
if (keyAttributeType == keyType && valueAttributeType == valueType) {
return this;
} else {
AtlasType attributeType = new AtlasMapType(keyAttributeType, valueAttributeType);
if (LOG.isDebugEnabled()) {
LOG.debug("getTypeForAttribute(): {} ==> {}", getTypeName(), attributeType.getTypeName());
}
return attributeType;
}
}
}
......@@ -415,7 +415,7 @@ public class AtlasStructType extends AtlasType {
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
this.definedInType = definedInType;
this.attributeDef = attrDef;
this.attributeType = attributeType;
this.attributeType = attributeType.getTypeForAttribute();
this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName());
this.vertexPropertyName = encodePropertyKey(this.qualifiedName);
......
......@@ -77,6 +77,14 @@ public abstract class AtlasType {
return ret;
}
/* for attribute of entity-type, the value would be of AtlasObjectId
* when an attribute instance is created i.e. AtlasAttribute, this method
* will be called to get AtlasEntityType replaced with AtlasObjectType
*/
public AtlasType getTypeForAttribute() {
return this;
}
public static String toJson(Object obj) {
return GSON.toJson(obj);
......
......@@ -33,6 +33,7 @@ import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
......@@ -981,12 +982,6 @@ public final class GraphHelper {
}
public static boolean isReference(AtlasType type) {
return ((type.getTypeCategory() == org.apache.atlas.model.TypeCategory.STRUCT) ||
(type.getTypeCategory() == org.apache.atlas.model.TypeCategory.ENTITY));
}
public static void setArrayElementsProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
......@@ -1020,7 +1015,7 @@ public final class GraphHelper {
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (GraphHelper.isReference(elementType)) {
if (AtlasGraphUtilsV1.isReference(elementType)) {
return instanceVertex.getProperty(vertexPropertyName, AtlasEdge.class);
} else {
return instanceVertex.getProperty(vertexPropertyName, Object.class).toString();
......@@ -1030,7 +1025,7 @@ public final class GraphHelper {
// newly added
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
if(AtlasGraphUtilsV1.isReference(elementType)) {
return (List)instanceVertex.getListProperty(encodedPropertyName, AtlasEdge.class);
}
else {
......
......@@ -34,13 +34,8 @@ import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.repository.store.graph.EntityResolver;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -128,7 +123,7 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
}
}
private void visitReference(AtlasEntityType type, Object val) throws AtlasBaseException {
private void visitReference(AtlasObjectIdType type, Object val) throws AtlasBaseException {
if (type == null || val == null) {
return;
}
......@@ -149,8 +144,6 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
}
recordObjectReference(objId);
} else if (val instanceof AtlasEntity) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "found AtlasEntity");
} else {
throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, val.toString());
}
......@@ -161,23 +154,37 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
return;
}
if (isPrimitive(attrType.getTypeCategory()) ) {
return;
}
if (attrType.getTypeCategory() == TypeCategory.ARRAY) {
AtlasArrayType arrayType = (AtlasArrayType) attrType;
AtlasType elemType = arrayType.getElementType();
visitCollectionReferences(elemType, val);
} else if (attrType.getTypeCategory() == TypeCategory.MAP) {
AtlasType keyType = ((AtlasMapType) attrType).getKeyType();
AtlasType valueType = ((AtlasMapType) attrType).getValueType();
visitMapReferences(keyType, valueType, val);
} else if (attrType.getTypeCategory() == TypeCategory.STRUCT) {
visitStruct((AtlasStructType)attrType, val);
} else if (attrType.getTypeCategory() == TypeCategory.ENTITY) {
visitReference((AtlasEntityType) attrType, val);
switch (attrType.getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return;
case ARRAY: {
AtlasArrayType arrayType = (AtlasArrayType) attrType;
AtlasType elemType = arrayType.getElementType();
visitCollectionReferences(elemType, val);
}
break;
case MAP: {
AtlasType keyType = ((AtlasMapType) attrType).getKeyType();
AtlasType valueType = ((AtlasMapType) attrType).getValueType();
visitMapReferences(keyType, valueType, val);
}
break;
case STRUCT:
visitStruct((AtlasStructType)attrType, val);
break;
case OBJECT_ID_TYPE:
visitReference((AtlasObjectIdType) attrType, val);
break;
default:
throw new AtlasBaseException(AtlasErrorCode.TYPE_CATEGORY_INVALID, attrType.getTypeCategory().name());
}
}
......
......@@ -115,7 +115,8 @@ public class AtlasGraphUtilsV1 {
public static boolean isReference(TypeCategory typeCategory) {
return typeCategory == TypeCategory.STRUCT ||
typeCategory == TypeCategory.ENTITY ||
typeCategory == TypeCategory.CLASSIFICATION;
typeCategory == TypeCategory.CLASSIFICATION ||
typeCategory == TypeCategory.OBJECT_ID_TYPE;
}
public static String encodePropertyKey(String key) {
......
......@@ -145,7 +145,7 @@ public abstract class DeleteHandlerV1 {
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(entityType, attributeInfo.getName());
AtlasType attrType = typeRegistry.getType(attributeInfo.getTypeName());
switch (attrType.getTypeCategory()) {
case ENTITY:
case OBJECT_ID_TYPE:
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
if (edge != null && AtlasGraphUtilsV1.getState(edge) == AtlasEntity.Status.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex();
......@@ -154,7 +154,7 @@ public abstract class DeleteHandlerV1 {
break;
case ARRAY:
AtlasArrayType arrType = (AtlasArrayType) attrType;
if (arrType.getElementType().getTypeCategory() != TypeCategory.ENTITY) {
if (arrType.getElementType().getTypeCategory() != TypeCategory.OBJECT_ID_TYPE) {
continue;
}
Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel);
......@@ -171,7 +171,7 @@ public abstract class DeleteHandlerV1 {
case MAP:
AtlasMapType mapType = (AtlasMapType) attrType;
TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory();
if (valueTypeCategory != TypeCategory.ENTITY) {
if (valueTypeCategory != TypeCategory.OBJECT_ID_TYPE) {
continue;
}
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(entityType, attributeInfo.getName());
......@@ -209,8 +209,9 @@ public abstract class DeleteHandlerV1 {
LOG.debug("Deleting {}", string(edge));
boolean forceDelete =
(typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait;
if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION
|| (typeCategory == TypeCategory.ENTITY && isComposite)) {
|| (typeCategory == TypeCategory.OBJECT_ID_TYPE && isComposite)) {
//If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities.
//If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled
//through this delete, hence delete the edge and the reference vertex.
......@@ -258,6 +259,7 @@ public abstract class DeleteHandlerV1 {
break;
case ENTITY:
case OBJECT_ID_TYPE:
deleteEntities(Collections.singletonList(instanceVertex));
break;
......@@ -282,7 +284,7 @@ public abstract class DeleteHandlerV1 {
AtlasStructType structType = (AtlasStructType) parentType;
boolean isEntityType = (parentType instanceof AtlasEntityType);
for (AtlasStructType.AtlasAttribute attributeInfo : getAttributes(structType)) {
for (AtlasStructType.AtlasAttribute attributeInfo : structType.getAllAttributes().values()) {
LOG.debug("Deleting attribute {} for {}", attributeInfo.getName(), string(instanceVertex));
boolean isComposite = isEntityType && attributeInfo.isOwnedRef();
......@@ -291,14 +293,14 @@ public abstract class DeleteHandlerV1 {
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(structType, attributeInfo.getName());
switch (attrType.getTypeCategory()) {
case ENTITY:
case OBJECT_ID_TYPE:
//If its class attribute, delete the reference
deleteEdgeReference(instanceVertex, edgeLabel, TypeCategory.ENTITY, isComposite);
deleteEdgeReference(instanceVertex, edgeLabel, attrType.getTypeCategory(), isComposite);
break;
case STRUCT:
//If its struct attribute, delete the reference
deleteEdgeReference(instanceVertex, edgeLabel, TypeCategory.STRUCT, false);
deleteEdgeReference(instanceVertex, edgeLabel, attrType.getTypeCategory(), false);
break;
case ARRAY:
......@@ -403,7 +405,7 @@ public abstract class DeleteHandlerV1 {
AtlasType attrType = typeRegistry.getType(attrDef.getTypeName());
switch (attrType.getTypeCategory()) {
case ENTITY:
case OBJECT_ID_TYPE:
//If its class attribute, its the only edge between two vertices
if (attrDef.getIsOptional()) {
edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
......@@ -532,21 +534,4 @@ public abstract class DeleteHandlerV1 {
}
_deleteVertex(instanceVertex, force);
}
private Collection<AtlasStructType.AtlasAttribute> getAttributes(AtlasStructType structType) {
Collection<AtlasStructType.AtlasAttribute> ret = null;
if (structType.getTypeCategory() == TypeCategory.STRUCT) {
ret = structType.getAllAttributes().values();
} else if (structType.getTypeCategory() == TypeCategory.CLASSIFICATION) {
ret = ((AtlasClassificationType)structType).getAllAttributes().values();
} else if (structType.getTypeCategory() == TypeCategory.ENTITY) {
ret = ((AtlasEntityType)structType).getAllAttributes().values();
} else {
ret = Collections.emptyList();
}
return ret;
}
}
......@@ -243,7 +243,7 @@ public class EntityGraphMapper {
return newEdge;
}
case ENTITY: {
case OBJECT_ID_TYPE: {
String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexProperty());
AtlasEdge currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel);
AtlasEntityType instanceType = getInstanceType(ctx.getValue());
......@@ -252,7 +252,7 @@ public class EntityGraphMapper {
ctx.setElementType(instanceType);
ctx.setExistingEdge(edge);
AtlasEdge newEdge = mapEntityValue(ctx, context);
AtlasEdge newEdge = mapObjectIdValue(ctx, context);
if (currentEdge != null && !currentEdge.equals(newEdge)) {
deleteHandler.deleteEdgeReference(currentEdge, ctx.getAttrType().getTypeCategory(), ctx.getAttribute().isOwnedRef(), true);
......@@ -308,9 +308,9 @@ public class EntityGraphMapper {
return ret;
}
private AtlasEdge mapEntityValue(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException {
private AtlasEdge mapObjectIdValue(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> mapEntityValue({})", ctx);
LOG.debug("==> mapObjectIdValue({})", ctx);
}
AtlasEdge ret = null;
......@@ -342,7 +342,7 @@ public class EntityGraphMapper {
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== mapEntityValue({})", ctx);
LOG.debug("<== mapObjectIdValue({})", ctx);
}
return ret;
......@@ -494,10 +494,10 @@ public class EntityGraphMapper {
case STRUCT:
return mapStructValue(ctx, context);
case ENTITY:
case OBJECT_ID_TYPE:
AtlasEntityType instanceType = getInstanceType(ctx.getValue());
ctx.setElementType(instanceType);
return mapEntityValue(ctx, context);
return mapObjectIdValue(ctx, context);
case MAP:
case ARRAY:
......@@ -510,8 +510,6 @@ public class EntityGraphMapper {
if (val != null) {
if ( val instanceof AtlasObjectId) {
return ((AtlasObjectId) val);
} else if (val instanceof AtlasEntity) {
return ((AtlasEntity) val).getAtlasObjectId();
} else if (val instanceof Map) {
AtlasObjectId ret = new AtlasObjectId((Map)val);
......@@ -530,8 +528,6 @@ public class EntityGraphMapper {
if (val != null) {
if ( val instanceof AtlasObjectId) {
return ((AtlasObjectId) val).getGuid();
} else if (val instanceof AtlasEntity) {
return ((AtlasEntity) val).getGuid();
} else if (val instanceof Map) {
Object guidVal = ((Map)val).get(AtlasObjectId.KEY_GUID);
......@@ -550,8 +546,6 @@ public class EntityGraphMapper {
if (val instanceof AtlasObjectId) {
typeName = ((AtlasObjectId)val).getTypeName();
} else if (val instanceof AtlasEntity) {
typeName = ((AtlasEntity)val).getTypeName();
} else if (val instanceof Map) {
Object typeNameVal = ((Map)val).get(AtlasObjectId.KEY_TYPENAME);
......
......@@ -251,7 +251,7 @@ public final class EntityGraphRetriever {
case STRUCT:
ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo);
break;
case ENTITY:
case OBJECT_ID_TYPE:
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute);
break;
case ARRAY:
......@@ -344,7 +344,7 @@ public final class EntityGraphRetriever {
ret = mapVertexToStruct(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo);
break;
case ENTITY:
case OBJECT_ID_TYPE:
ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute);
break;
......
......@@ -660,7 +660,7 @@ public class AtlasEntityStoreV1Test {
private void validateAttribute(AtlasEntityExtInfo entityExtInfo, Object actual, Object expected, AtlasType attributeType, String attrName) throws AtlasBaseException, AtlasException {
switch(attributeType.getTypeCategory()) {
case ENTITY:
case OBJECT_ID_TYPE:
Assert.assertTrue(actual instanceof AtlasObjectId);
String guid = ((AtlasObjectId) actual).getGuid();
Assert.assertTrue(AtlasEntity.isAssigned(guid), "expected assigned guid. found " + guid);
......
......@@ -418,24 +418,4 @@ public final class RestUtils {
return ret.toArray(new AttributeDefinition[ret.size()]);
}
private static HierarchicalTypeDefinition<ClassType> findClassType(List<HierarchicalTypeDefinition<ClassType>> classDefs,
String typeName) {
HierarchicalTypeDefinition<ClassType> ret = null;
if (CollectionUtils.isNotEmpty(classDefs)) {
for (HierarchicalTypeDefinition<ClassType> classType : classDefs) {
if (classType.typeName.equalsIgnoreCase(typeName)) {
ret = classType;
}
}
}
return ret;
}
private static boolean isEntity(AtlasType type) {
return type.getTypeCategory() == TypeCategory.ENTITY;
}
}
......@@ -45,6 +45,10 @@ public class AtlasFormatConverters {
private void registerConverter(AtlasFormatConverter converter) {
registry.put(converter.getTypeCategory(), converter);
if (converter.getTypeCategory() == TypeCategory.ENTITY) {
registry.put(TypeCategory.OBJECT_ID_TYPE, converter);
}
}
public AtlasFormatConverter getConverter(TypeCategory typeCategory) throws AtlasBaseException {
......
......@@ -136,7 +136,7 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
Object v1Value = null;
AtlasFormatConverter attrConverter = null;
if (attrType.getTypeCategory() == TypeCategory.ENTITY && !attr.isOwnedRef()) {
if (attrType.getTypeCategory() == TypeCategory.OBJECT_ID_TYPE && !attr.isOwnedRef()) {
attrConverter = new AtlasObjectIdConverter(converterRegistry, typeRegistry);
v1Value = attrConverter.fromV2ToV1(v2Value, attrType, context);
} else {
......@@ -174,18 +174,4 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
return ret;
}
private Collection<AtlasStructType.AtlasAttribute> getAttributes(AtlasStructType structType) {
Collection<AtlasStructType.AtlasAttribute> ret = null;
if (structType.getTypeCategory() == TypeCategory.STRUCT
|| structType.getTypeCategory() == TypeCategory.CLASSIFICATION
|| structType.getTypeCategory() == TypeCategory.ENTITY) {
ret = structType.getAllAttributes().values();
} else {
ret = Collections.emptyList();
}
return ret;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment