Skip to content

A
atlas
Commit 66c2964e by Sarath Subramanian Committed by Madhan Neethiraj
Options

ATLAS-3036: Improve FullTextMapper performance during entity retrieval

(cherry picked from commit a08a5a39a83fcfee0965471a3a1b1c29279feea3) (cherry picked from commit 7ff396af047c8ff829dbe5b8a4094177fb3e9686) (cherry picked from commit 2fdbc851db06f7ebe96cc3fde4d8507d088e7ec6)
parent 9f71b529
Showing with 71 additions and 19 deletions
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
......@@ -131,7 +131,7 @@ public class FullTextMapperV2 {
entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
entityExtInfo = entityWithExtInfo;
} else {
entity = getAndCacheEntity(guid);
entity = getAndCacheEntity(guid, false);
entityExtInfo = null;
}
......@@ -272,11 +272,15 @@ public class FullTextMapperV2 {
}
private AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException {
return getAndCacheEntity(guid, true);
}
private AtlasEntity getAndCacheEntity(String guid, boolean includeReferences) throws AtlasBaseException {
RequestContext context = RequestContext.get();
AtlasEntity entity = context.getEntity(guid);
if (entity == null) {
entity = entityGraphRetriever.toAtlasEntity(guid);
entity = entityGraphRetriever.toAtlasEntity(guid, includeReferences);
if (entity != null) {
context.cache(entity);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
......@@ -44,6 +44,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
......@@ -155,6 +156,10 @@ public class EntityGraphRetriever {
this.ignoreRelationshipAttr = ignoreRelationshipAttr;
}
public AtlasEntity toAtlasEntity(String guid, boolean includeReferences) throws AtlasBaseException {
return mapVertexToAtlasEntity(getEntityVertex(guid), null, false, includeReferences);
}
public AtlasEntity toAtlasEntity(String guid) throws AtlasBaseException {
return toAtlasEntity(getEntityVertex(guid));
}
......@@ -429,6 +434,10 @@ public class EntityGraphRetriever {
}
private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
return mapVertexToAtlasEntity(entityVertex, entityExtInfo, isMinExtInfo, true);
}
private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
String guid = GraphHelper.getGuid(entityVertex);
AtlasEntity entity = entityExtInfo != null ? entityExtInfo.getEntity(guid) : null;
......@@ -445,7 +454,7 @@ public class EntityGraphRetriever {
mapSystemAttributes(entityVertex, entity);
mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo, includeReferences);
if (!ignoreRelationshipAttr) { // only map when really needed
mapRelationshipAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
......@@ -595,6 +604,10 @@ public class EntityGraphRetriever {
}
private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
mapAttributes(entityVertex, struct, entityExtInfo, isMinExtInfo, true);
}
private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(struct.getTypeName());
if (!(objType instanceof AtlasStructType)) {
......@@ -604,7 +617,7 @@ public class EntityGraphRetriever {
AtlasStructType structType = (AtlasStructType) objType;
for (AtlasAttribute attribute : structType.getAllAttributes().values()) {
Object attrValue = mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo);
Object attrValue = mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo, includeReferences);
struct.setAttribute(attribute.getName(), attrValue);
}
......@@ -721,6 +734,10 @@ public class EntityGraphRetriever {
}
private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) throws AtlasBaseException {
return mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo, true);
}
private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Object ret = null;
AtlasType attrType = attribute.getAttributeType();
String edgeLabel = attribute.getRelationshipEdgeLabel();
......@@ -742,24 +759,55 @@ public class EntityGraphRetriever {
ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo, isMinExtInfo);
break;
case OBJECT_ID_TYPE:
if(attribute.getAttributeDef().isSoftReferenced()) {
ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
if (includeReferences) {
ret = attribute.getAttributeDef().isSoftReferenced() ? mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo) :
mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
} else {
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
ret = null;
}
break;
case ARRAY:
if(attribute.getAttributeDef().isSoftReferenced()) {
case ARRAY: {
final boolean skipAttribute;
if (!includeReferences) {
AtlasType elementType = ((AtlasArrayType) attrType).getElementType();
skipAttribute = (elementType instanceof AtlasObjectIdType || elementType instanceof AtlasEntityType);
} else {
skipAttribute = false;
}
if (skipAttribute) {
ret = null;
} else {
if (attribute.getAttributeDef().isSoftReferenced()) {
ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
}
}
}
break;
case MAP:
if(attribute.getAttributeDef().isSoftReferenced()) {
case MAP: {
final boolean skipAttribute;
if (!includeReferences) {
AtlasType valueType = ((AtlasMapType) attrType).getValueType();
skipAttribute = (valueType instanceof AtlasObjectIdType || valueType instanceof AtlasEntityType);
} else {
skipAttribute = false;
}
if (skipAttribute) {
ret = null;
} else {
if (attribute.getAttributeDef().isSoftReferenced()) {
ret = mapVertexToMapForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
}
}
}
break;
case CLASSIFICATION:
......@@ -848,7 +896,7 @@ public class EntityGraphRetriever {
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException {
boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Map<String, Object> ret = null;
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
......@@ -868,7 +916,7 @@ public class EntityGraphRetriever {
String mapKey = entry.getKey();
Object keyValue = entry.getValue();
Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, attribute.getRelationshipEdgeLabel(),
entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(), isMinExtInfo);
entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(), isMinExtInfo, includeReferences);
if (mapValue != null) {
ret.put(mapKey, mapValue);
}
......@@ -882,7 +930,7 @@ public class EntityGraphRetriever {
}
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException {
boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType();
......@@ -909,7 +957,7 @@ public class EntityGraphRetriever {
}
Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel,
entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo, includeReferences);
if (arrValue != null) {
arrValues.add(arrValue);
......@@ -921,7 +969,7 @@ public class EntityGraphRetriever {
private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value,
String edgeLabel, AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
AtlasRelationshipEdgeDirection edgeDirection, final boolean isMinExtInfo) throws AtlasBaseException {
AtlasRelationshipEdgeDirection edgeDirection, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Object ret = null;
switch (arrayElement.getTypeCategory()) {
......@@ -940,7 +988,7 @@ public class EntityGraphRetriever {
break;
case OBJECT_ID_TYPE:
ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
ret = includeReferences ? mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo) : null;
break;
default:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment