Skip to content

A
atlas
Commit b32e547e by apoorvnaik Committed by Madhan Neethiraj
Options

ATLAS-2489: Lineage info to include relationship guid

Change-Id: Ie26d1ad07b6ec66beb42830ad154a9dd81e7933f Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 5ebd7070
Showing with 73 additions and 43 deletions
intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageInfo.java
...@@ -145,12 +145,14 @@ public class AtlasLineageInfo implements Serializable { ...@@ -145,12 +145,14 @@ public class AtlasLineageInfo implements Serializable {
public static class LineageRelation { public static class LineageRelation {
private String fromEntityId; private String fromEntityId;
private String toEntityId; private String toEntityId;
private String relationshipId;
public LineageRelation() { } public LineageRelation() { }
public LineageRelation(String fromEntityId, String toEntityId) { public LineageRelation(String fromEntityId, String toEntityId, final String relationshipId) {
this.fromEntityId = fromEntityId; this.fromEntityId = fromEntityId;
this.toEntityId = toEntityId; this.toEntityId = toEntityId;
this.relationshipId = relationshipId;
} }
public String getFromEntityId() { public String getFromEntityId() {
...@@ -169,18 +171,27 @@ public class AtlasLineageInfo implements Serializable { ...@@ -169,18 +171,27 @@ public class AtlasLineageInfo implements Serializable {
this.toEntityId = toEntityId; this.toEntityId = toEntityId;
} }
public String getRelationshipId() {
return relationshipId;
}
public void setRelationshipId(final String relationshipId) {
this.relationshipId = relationshipId;
}
@Override @Override
public boolean equals(Object o) { public boolean equals(Object o) {
if (this == o) return true; if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false; if (o == null || getClass() != o.getClass()) return false;
LineageRelation that = (LineageRelation) o; LineageRelation that = (LineageRelation) o;
return Objects.equals(fromEntityId, that.fromEntityId) && return Objects.equals(fromEntityId, that.fromEntityId) &&
Objects.equals(toEntityId, that.toEntityId); Objects.equals(toEntityId, that.toEntityId) &&
Objects.equals(relationshipId, that.relationshipId);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(fromEntityId, toEntityId); return Objects.hash(fromEntityId, toEntityId, relationshipId);
} }
@Override @Override
...@@ -188,6 +199,7 @@ public class AtlasLineageInfo implements Serializable { ...@@ -188,6 +199,7 @@ public class AtlasLineageInfo implements Serializable {
return "LineageRelation{" + return "LineageRelation{" +
"fromEntityId='" + fromEntityId + '\'' + "fromEntityId='" + fromEntityId + '\'' +
", toEntityId='" + toEntityId + '\'' + ", toEntityId='" + toEntityId + '\'' +
", relationshipId='" + relationshipId + '\'' +
'}'; '}';
} }
} }
......
repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
...@@ -33,6 +33,8 @@ import org.apache.atlas.model.instance.AtlasObjectId; ...@@ -33,6 +33,8 @@ import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.lineage.AtlasLineageInfo; import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
...@@ -46,13 +48,14 @@ import org.apache.atlas.v1.model.lineage.SchemaResponse.SchemaDetails; ...@@ -46,13 +48,14 @@ import org.apache.atlas.v1.model.lineage.SchemaResponse.SchemaDetails;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.inject.Inject; import javax.inject.Inject;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
...@@ -60,9 +63,11 @@ import java.util.stream.Collectors; ...@@ -60,9 +63,11 @@ import java.util.stream.Collectors;
@Service @Service
public class EntityLineageService implements AtlasLineageService { public class EntityLineageService implements AtlasLineageService {
private static final String INPUT_PROCESS_EDGE = "__Process.inputs"; private static final Logger LOG = LoggerFactory.getLogger(EntityLineageService.class);
private static final String OUTPUT_PROCESS_EDGE = "__Process.outputs";
private static final String COLUMNS = "columns"; private static final String PROCESS_INPUTS_EDGE = "__Process.inputs";
private static final String PROCESS_OUTPUTS_EDGE = "__Process.outputs";
private static final String COLUMNS = "columns";
private final AtlasGraph graph; private final AtlasGraph graph;
private final AtlasGremlinQueryProvider gremlinQueryProvider; private final AtlasGremlinQueryProvider gremlinQueryProvider;
...@@ -162,7 +167,7 @@ public class EntityLineageService implements AtlasLineageService { ...@@ -162,7 +167,7 @@ public class EntityLineageService implements AtlasLineageService {
List<String> ret = new ArrayList<>(); List<String> ret = new ArrayList<>();
Object columnObjs = entity.getAttribute(COLUMNS); Object columnObjs = entity.getAttribute(COLUMNS);
if (columnObjs != null && columnObjs instanceof List) { if (columnObjs instanceof List) {
for (Object pkObj : (List) columnObjs) { for (Object pkObj : (List) columnObjs) {
if (pkObj instanceof AtlasObjectId) { if (pkObj instanceof AtlasObjectId) {
ret.add(((AtlasObjectId) pkObj).getGuid()); ret.add(((AtlasObjectId) pkObj).getGuid());
...@@ -182,35 +187,27 @@ public class EntityLineageService implements AtlasLineageService { ...@@ -182,35 +187,27 @@ public class EntityLineageService implements AtlasLineageService {
Set<LineageRelation> relations = new HashSet<>(); Set<LineageRelation> relations = new HashSet<>();
String lineageQuery = getLineageQuery(guid, direction, depth); String lineageQuery = getLineageQuery(guid, direction, depth);
List paths = (List) graph.executeGremlinScript(lineageQuery, true); List edgeMapList = (List) graph.executeGremlinScript(lineageQuery, false);
if (CollectionUtils.isNotEmpty(paths)) { if (CollectionUtils.isNotEmpty(edgeMapList)) {
for (Object path : paths) { for (Object edgeMap : edgeMapList) {
if (path instanceof List) { if (edgeMap instanceof Map) {
List vertices = (List) path; for (final Object o : ((Map) edgeMap).entrySet()) {
final Map.Entry entry = (Map.Entry) o;
if (CollectionUtils.isNotEmpty(vertices)) { Object value = entry.getValue();
AtlasEntityHeader prev = null;
if (value instanceof List) {
for (Object vertex : vertices) { for (Object elem : (List) value) {
if (!(vertex instanceof AtlasVertex)) { if (elem instanceof AtlasEdge) {
continue; processEdge((AtlasEdge) elem, entities, relations);
} } else {
LOG.warn("Invalid value of type {} found, ignoring", (elem != null ? elem.getClass().getSimpleName() : "null"));
AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader((AtlasVertex) vertex);
if (!entities.containsKey(entity.getGuid())) {
entities.put(entity.getGuid(), entity);
}
if (prev != null) {
if (direction.equals(LineageDirection.INPUT)) {
relations.add(new LineageRelation(entity.getGuid(), prev.getGuid()));
} else if (direction.equals(LineageDirection.OUTPUT)) {
relations.add(new LineageRelation(prev.getGuid(), entity.getGuid()));
} }
} }
prev = entity; } else if (value instanceof AtlasEdge) {
processEdge((AtlasEdge) value, entities, relations);
} else {
LOG.warn("Invalid value of type {} found, ignoring", (value != null ? value.getClass().getSimpleName() : "null"));
} }
} }
} }
...@@ -220,6 +217,31 @@ public class EntityLineageService implements AtlasLineageService { ...@@ -220,6 +217,31 @@ public class EntityLineageService implements AtlasLineageService {
return new AtlasLineageInfo(guid, entities, relations, direction, depth); return new AtlasLineageInfo(guid, entities, relations, direction, depth);
} }
private void processEdge(final AtlasEdge edge, final Map<String, AtlasEntityHeader> entities, final Set<LineageRelation> relations) throws AtlasBaseException {
AtlasVertex inVertex = edge.getInVertex();
AtlasVertex outVertex = edge.getOutVertex();
String inGuid = AtlasGraphUtilsV1.getIdFromVertex(inVertex);
String outGuid = AtlasGraphUtilsV1.getIdFromVertex(outVertex);
String relationGuid = AtlasGraphUtilsV1.getProperty(edge, Constants.RELATIONSHIP_GUID_PROPERTY_KEY, String.class);
boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE);
if (!entities.containsKey(inGuid)) {
AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(inVertex);
entities.put(inGuid, entityHeader);
}
if (!entities.containsKey(outGuid)) {
AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(outVertex);
entities.put(outGuid, entityHeader);
}
if (isInputEdge) {
relations.add(new LineageRelation(inGuid, outGuid, relationGuid));
} else {
relations.add(new LineageRelation(outGuid, inGuid, relationGuid));
}
}
private AtlasLineageInfo getBothLineageInfo(String guid, int depth) throws AtlasBaseException { private AtlasLineageInfo getBothLineageInfo(String guid, int depth) throws AtlasBaseException {
AtlasLineageInfo inputLineage = getLineageInfo(guid, LineageDirection.INPUT, depth); AtlasLineageInfo inputLineage = getLineageInfo(guid, LineageDirection.INPUT, depth);
AtlasLineageInfo outputLineage = getLineageInfo(guid, LineageDirection.OUTPUT, depth); AtlasLineageInfo outputLineage = getLineageInfo(guid, LineageDirection.OUTPUT, depth);
...@@ -236,10 +258,10 @@ public class EntityLineageService implements AtlasLineageService { ...@@ -236,10 +258,10 @@ public class EntityLineageService implements AtlasLineageService {
String lineageQuery = null; String lineageQuery = null;
if (direction.equals(LineageDirection.INPUT)) { if (direction.equals(LineageDirection.INPUT)) {
lineageQuery = generateLineageQuery(entityGuid, depth, OUTPUT_PROCESS_EDGE, INPUT_PROCESS_EDGE); lineageQuery = generateLineageQuery(entityGuid, depth, PROCESS_OUTPUTS_EDGE, PROCESS_INPUTS_EDGE);
} else if (direction.equals(LineageDirection.OUTPUT)) { } else if (direction.equals(LineageDirection.OUTPUT)) {
lineageQuery = generateLineageQuery(entityGuid, depth, INPUT_PROCESS_EDGE, OUTPUT_PROCESS_EDGE); lineageQuery = generateLineageQuery(entityGuid, depth, PROCESS_INPUTS_EDGE, PROCESS_OUTPUTS_EDGE);
} }
return lineageQuery; return lineageQuery;
......
repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
...@@ -44,13 +44,9 @@ public class AtlasGremlin3QueryProvider extends AtlasGremlin2QueryProvider { ...@@ -44,13 +44,9 @@ public class AtlasGremlin3QueryProvider extends AtlasGremlin2QueryProvider {
case EXPORT_BY_GUID_CONNECTED_OUT_EDGE: case EXPORT_BY_GUID_CONNECTED_OUT_EDGE:
return "g.V().has('__guid', startGuid).outE().inV().has('__guid').project('__guid', 'isProcess').by('__guid').by(map {it.get().values('__superTypeNames').toSet().contains('Process')}).dedup().toList()"; return "g.V().has('__guid', startGuid).outE().inV().has('__guid').project('__guid', 'isProcess').by('__guid').by(map {it.get().values('__superTypeNames').toSet().contains('Process')}).dedup().toList()";
case FULL_LINEAGE: case FULL_LINEAGE:
return "g.V().has('__guid', '%s').repeat(__.in('%s').out('%s'))." + return "g.V().has('__guid', '%s').repeat(__.inE('%s').as('e1').outV().outE('%s').as('e2').inV()).emit().select('e1', 'e2').toList()";
"emit(has('__superTypeNames').and().properties('__superTypeNames').hasValue('DataSet'))." +
"path().toList()";
case PARTIAL_LINEAGE: case PARTIAL_LINEAGE:
return "g.V().has('__guid', '%s').repeat(__.in('%s').out('%s')).times(%s)." + return "g.V().has('__guid', '%s').repeat(__.inE('%s').as('e1').outV().outE('%s').as('e2').inV()).times(%s).emit().select('e1', 'e2').toList()";
"emit(has('__superTypeNames').and().properties('__superTypeNames').hasValue('DataSet'))." +
"path().toList()";
case TO_RANGE_LIST: case TO_RANGE_LIST:
return ".range(startIdx, endIdx).toList()"; return ".range(startIdx, endIdx).toList()";
case RELATIONSHIP_SEARCH: case RELATIONSHIP_SEARCH:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment