Skip to content

A
atlas
Commit 66c2964e by Sarath Subramanian Committed by Madhan Neethiraj
Options

ATLAS-3036: Improve FullTextMapper performance during entity retrieval

(cherry picked from commit a08a5a39a83fcfee0965471a3a1b1c29279feea3) (cherry picked from commit 7ff396af047c8ff829dbe5b8a4094177fb3e9686) (cherry picked from commit 2fdbc851db06f7ebe96cc3fde4d8507d088e7ec6)
parent 9f71b529
Showing with 76 additions and 24 deletions
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
...@@ -131,7 +131,7 @@ public class FullTextMapperV2 { ...@@ -131,7 +131,7 @@ public class FullTextMapperV2 {
entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null; entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
entityExtInfo = entityWithExtInfo; entityExtInfo = entityWithExtInfo;
} else { } else {
entity = getAndCacheEntity(guid); entity = getAndCacheEntity(guid, false);
entityExtInfo = null; entityExtInfo = null;
} }
...@@ -272,11 +272,15 @@ public class FullTextMapperV2 { ...@@ -272,11 +272,15 @@ public class FullTextMapperV2 {
} }
private AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException { private AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException {
return getAndCacheEntity(guid, true);
}
private AtlasEntity getAndCacheEntity(String guid, boolean includeReferences) throws AtlasBaseException {
RequestContext context = RequestContext.get(); RequestContext context = RequestContext.get();
AtlasEntity entity = context.getEntity(guid); AtlasEntity entity = context.getEntity(guid);
if (entity == null) { if (entity == null) {
entity = entityGraphRetriever.toAtlasEntity(guid); entity = entityGraphRetriever.toAtlasEntity(guid, includeReferences);
if (entity != null) { if (entity != null) {
context.cache(entity); context.cache(entity);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
...@@ -44,6 +44,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; ...@@ -44,6 +44,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType; import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasRelationshipType;
...@@ -155,6 +156,10 @@ public class EntityGraphRetriever { ...@@ -155,6 +156,10 @@ public class EntityGraphRetriever {
this.ignoreRelationshipAttr = ignoreRelationshipAttr; this.ignoreRelationshipAttr = ignoreRelationshipAttr;
} }
public AtlasEntity toAtlasEntity(String guid, boolean includeReferences) throws AtlasBaseException {
return mapVertexToAtlasEntity(getEntityVertex(guid), null, false, includeReferences);
}
public AtlasEntity toAtlasEntity(String guid) throws AtlasBaseException { public AtlasEntity toAtlasEntity(String guid) throws AtlasBaseException {
return toAtlasEntity(getEntityVertex(guid)); return toAtlasEntity(getEntityVertex(guid));
} }
...@@ -429,6 +434,10 @@ public class EntityGraphRetriever { ...@@ -429,6 +434,10 @@ public class EntityGraphRetriever {
} }
private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException { private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
return mapVertexToAtlasEntity(entityVertex, entityExtInfo, isMinExtInfo, true);
}
private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
String guid = GraphHelper.getGuid(entityVertex); String guid = GraphHelper.getGuid(entityVertex);
AtlasEntity entity = entityExtInfo != null ? entityExtInfo.getEntity(guid) : null; AtlasEntity entity = entityExtInfo != null ? entityExtInfo.getEntity(guid) : null;
...@@ -445,7 +454,7 @@ public class EntityGraphRetriever { ...@@ -445,7 +454,7 @@ public class EntityGraphRetriever {
mapSystemAttributes(entityVertex, entity); mapSystemAttributes(entityVertex, entity);
mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo); mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo, includeReferences);
if (!ignoreRelationshipAttr) { // only map when really needed if (!ignoreRelationshipAttr) { // only map when really needed
mapRelationshipAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo); mapRelationshipAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
...@@ -595,6 +604,10 @@ public class EntityGraphRetriever { ...@@ -595,6 +604,10 @@ public class EntityGraphRetriever {
} }
private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException { private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
mapAttributes(entityVertex, struct, entityExtInfo, isMinExtInfo, true);
}
private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(struct.getTypeName()); AtlasType objType = typeRegistry.getType(struct.getTypeName());
if (!(objType instanceof AtlasStructType)) { if (!(objType instanceof AtlasStructType)) {
...@@ -604,7 +617,7 @@ public class EntityGraphRetriever { ...@@ -604,7 +617,7 @@ public class EntityGraphRetriever {
AtlasStructType structType = (AtlasStructType) objType; AtlasStructType structType = (AtlasStructType) objType;
for (AtlasAttribute attribute : structType.getAllAttributes().values()) { for (AtlasAttribute attribute : structType.getAllAttributes().values()) {
Object attrValue = mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo); Object attrValue = mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo, includeReferences);
struct.setAttribute(attribute.getName(), attrValue); struct.setAttribute(attribute.getName(), attrValue);
} }
...@@ -721,6 +734,10 @@ public class EntityGraphRetriever { ...@@ -721,6 +734,10 @@ public class EntityGraphRetriever {
} }
private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) throws AtlasBaseException { private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) throws AtlasBaseException {
return mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo, true);
}
private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Object ret = null; Object ret = null;
AtlasType attrType = attribute.getAttributeType(); AtlasType attrType = attribute.getAttributeType();
String edgeLabel = attribute.getRelationshipEdgeLabel(); String edgeLabel = attribute.getRelationshipEdgeLabel();
...@@ -742,25 +759,56 @@ public class EntityGraphRetriever { ...@@ -742,25 +759,56 @@ public class EntityGraphRetriever {
ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo, isMinExtInfo); ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo, isMinExtInfo);
break; break;
case OBJECT_ID_TYPE: case OBJECT_ID_TYPE:
if(attribute.getAttributeDef().isSoftReferenced()) { if (includeReferences) {
ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo); ret = attribute.getAttributeDef().isSoftReferenced() ? mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo) :
mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
} else { } else {
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo); ret = null;
} }
break; break;
case ARRAY: case ARRAY: {
if(attribute.getAttributeDef().isSoftReferenced()) { final boolean skipAttribute;
ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
if (!includeReferences) {
AtlasType elementType = ((AtlasArrayType) attrType).getElementType();
skipAttribute = (elementType instanceof AtlasObjectIdType || elementType instanceof AtlasEntityType);
} else {
skipAttribute = false;
}
if (skipAttribute) {
ret = null;
} else { } else {
ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo); if (attribute.getAttributeDef().isSoftReferenced()) {
} ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
}
}
}
break; break;
case MAP: case MAP: {
if(attribute.getAttributeDef().isSoftReferenced()) { final boolean skipAttribute;
ret = mapVertexToMapForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
if (!includeReferences) {
AtlasType valueType = ((AtlasMapType) attrType).getValueType();
skipAttribute = (valueType instanceof AtlasObjectIdType || valueType instanceof AtlasEntityType);
} else { } else {
ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo); skipAttribute = false;
} }
if (skipAttribute) {
ret = null;
} else {
if (attribute.getAttributeDef().isSoftReferenced()) {
ret = mapVertexToMapForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
}
}
}
break; break;
case CLASSIFICATION: case CLASSIFICATION:
// do nothing // do nothing
...@@ -848,7 +896,7 @@ public class EntityGraphRetriever { ...@@ -848,7 +896,7 @@ public class EntityGraphRetriever {
} }
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException { boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Map<String, Object> ret = null; Map<String, Object> ret = null;
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType(); AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
...@@ -868,7 +916,7 @@ public class EntityGraphRetriever { ...@@ -868,7 +916,7 @@ public class EntityGraphRetriever {
String mapKey = entry.getKey(); String mapKey = entry.getKey();
Object keyValue = entry.getValue(); Object keyValue = entry.getValue();
Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, attribute.getRelationshipEdgeLabel(), Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, attribute.getRelationshipEdgeLabel(),
entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(), isMinExtInfo); entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(), isMinExtInfo, includeReferences);
if (mapValue != null) { if (mapValue != null) {
ret.put(mapKey, mapValue); ret.put(mapKey, mapValue);
} }
...@@ -882,7 +930,7 @@ public class EntityGraphRetriever { ...@@ -882,7 +930,7 @@ public class EntityGraphRetriever {
} }
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException { boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType(); AtlasType arrayElementType = arrayType.getElementType();
...@@ -909,7 +957,7 @@ public class EntityGraphRetriever { ...@@ -909,7 +957,7 @@ public class EntityGraphRetriever {
} }
Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel, Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel,
entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo); entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo, includeReferences);
if (arrValue != null) { if (arrValue != null) {
arrValues.add(arrValue); arrValues.add(arrValue);
...@@ -921,7 +969,7 @@ public class EntityGraphRetriever { ...@@ -921,7 +969,7 @@ public class EntityGraphRetriever {
private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value, private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value,
String edgeLabel, AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute, String edgeLabel, AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
AtlasRelationshipEdgeDirection edgeDirection, final boolean isMinExtInfo) throws AtlasBaseException { AtlasRelationshipEdgeDirection edgeDirection, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Object ret = null; Object ret = null;
switch (arrayElement.getTypeCategory()) { switch (arrayElement.getTypeCategory()) {
...@@ -940,7 +988,7 @@ public class EntityGraphRetriever { ...@@ -940,7 +988,7 @@ public class EntityGraphRetriever {
break; break;
case OBJECT_ID_TYPE: case OBJECT_ID_TYPE:
ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo); ret = includeReferences ? mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo) : null;
break; break;
default: default:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment