diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index e3eb52a..21ce14d 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -39,5 +39,5 @@ public interface AtlasDiscoveryService { * @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0. * @return AtlasSearchResult */ - AtlasSearchResult searchUsingFullTextQuery(String query, int limit, int offset); + AtlasSearchResult searchUsingFullTextQuery(String query, int limit, int offset) throws AtlasBaseException; } diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index da06230..5628ed8 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -17,7 +17,6 @@ */ package org.apache.atlas.discovery; -import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult; import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType; @@ -25,9 +24,7 @@ import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult; import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; -import org.apache.atlas.model.instance.AtlasEntity.Status; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.query.Expressions.AliasExpression; import org.apache.atlas.query.Expressions.Expression; import org.apache.atlas.query.Expressions.SelectExpression; @@ -44,6 +41,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.slf4j.Logger; @@ -55,7 +54,6 @@ import scala.util.parsing.combinator.Parsers.NoSuccess; import javax.inject.Inject; import javax.script.ScriptException; import java.util.ArrayList; -import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -63,19 +61,17 @@ import java.util.Map; import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED; public class EntityDiscoveryService implements AtlasDiscoveryService { - - private final AtlasGraph graph; - private final DefaultGraphPersistenceStrategy graphPersistenceStrategy; private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class); - private final static String PROPERTY_KEY_NAME = AtlasBaseTypeDef.ATLAS_TYPE_ASSET + "." + AtlasClient.NAME; - private final static String PROPERTY_KEY_DESCRIPTION = AtlasBaseTypeDef.ATLAS_TYPE_ASSET + "." + AtlasClient.DESCRIPTION; - private final static String PROPERTY_KEY_OWNER = AtlasBaseTypeDef.ATLAS_TYPE_ASSET + "." + AtlasClient.OWNER; + private final AtlasGraph graph; + private final DefaultGraphPersistenceStrategy graphPersistenceStrategy; + private final EntityGraphRetriever entityRetriever; @Inject - EntityDiscoveryService(MetadataRepository metadataRepository) { - this.graph = AtlasGraphProvider.getGraphInstance(); + EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry) { + this.graph = AtlasGraphProvider.getGraphInstance(); this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository); + this.entityRetriever = new EntityGraphRetriever(typeRegistry); } @Override @@ -97,7 +93,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { if (firstElement instanceof AtlasVertex) { for (Object element : queryResult) { if (element instanceof AtlasVertex) { - ret.addEntity(toAtlasEntityHeader((AtlasVertex)element)); + ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex)element)); } else { LOG.warn("searchUsingDslQuery({}): expected an AtlasVertex; found unexpected entry in result {}", dslQuery, element); } @@ -115,7 +111,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { Object entry = ((List)value).get(0); if (entry instanceof AtlasVertex) { - ret.addEntity(toAtlasEntityHeader((AtlasVertex)entry)); + ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex)entry)); } } } @@ -136,7 +132,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } @Override - public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, int limit, int offset) { + public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, int limit, int offset) throws AtlasBaseException { AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT); QueryParams params = validateSearchParams(limit, offset); AtlasIndexQuery idxQuery = toAtlasIndexQuery(fullTextQuery); @@ -149,7 +145,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return ret; } - private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams params) { + private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams params) throws AtlasBaseException { List<AtlasFullTextResult> ret = new ArrayList<>(); Iterator<Result> iter = query.vertices(); @@ -159,7 +155,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { String guid = vertex != null ? vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class) : null; if (guid != null) { - AtlasEntityHeader entity = toAtlasEntityHeader(vertex); + AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(vertex); Double score = idxQueryResult.getScore(); ret.add(new AtlasFullTextResult(entity, score)); } @@ -204,37 +200,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return new QueryParams(limit, offset); } - private AtlasEntityHeader toAtlasEntityHeader(AtlasVertex vertex) { - if (vertex == null) { - return null; - } - - AtlasEntityHeader ret = new AtlasEntityHeader(); - - String typeName = vertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); - - ret.setTypeName(typeName); - ret.setGuid(vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class)); - ret.setDisplayText(vertex.getProperty(Constants.QUALIFIED_NAME, String.class)); - ret.setAttribute(AtlasClient.NAME, vertex.getProperty(PROPERTY_KEY_NAME, String.class)); - ret.setAttribute(AtlasClient.DESCRIPTION, vertex.getProperty(PROPERTY_KEY_DESCRIPTION, String.class)); - ret.setAttribute(AtlasClient.OWNER, vertex.getProperty(PROPERTY_KEY_OWNER, String.class)); - - Collection<String> classificationNames = vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class); - - if (CollectionUtils.isNotEmpty(classificationNames)) { - ret.setClassificationNames(new ArrayList<>(classificationNames)); - } - - String state = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class); - if (state != null) { - Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.ACTIVE : Status.DELETED); - ret.setStatus(status); - } - - return ret; - } - private AtlasIndexQuery toAtlasIndexQuery(String fullTextQuery) { String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, fullTextQuery); return graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery); diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 4a98920..3035d16 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -22,7 +22,6 @@ package org.apache.atlas.discovery; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasEntity.Status; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.lineage.AtlasLineageInfo; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; @@ -31,6 +30,9 @@ import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; +import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.AtlasGremlinQueryProvider; import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; import org.apache.commons.collections.CollectionUtils; @@ -48,13 +50,15 @@ public class EntityLineageService implements AtlasLineageService { private static final String INPUT_PROCESS_EDGE = "__Process.inputs"; private static final String OUTPUT_PROCESS_EDGE = "__Process.outputs"; - private final AtlasGraph graph; + private final AtlasGraph graph; private final AtlasGremlinQueryProvider gremlinQueryProvider; + private final EntityGraphRetriever entityRetriever; @Inject - EntityLineageService() throws DiscoveryException { - this.graph = AtlasGraphProvider.getGraphInstance(); + EntityLineageService(AtlasTypeRegistry typeRegistry) throws DiscoveryException { + this.graph = AtlasGraphProvider.getGraphInstance(); this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; + this.entityRetriever = new EntityGraphRetriever(typeRegistry); } @Override @@ -99,7 +103,11 @@ public class EntityLineageService implements AtlasLineageService { AtlasEntityHeader prev = null; for (Object vertex : vertices) { - AtlasEntityHeader entity = toAtlasEntityHeader(vertex); + if (!(vertex instanceof AtlasVertex)) { + continue; + } + + AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader((AtlasVertex)vertex); if (!entities.containsKey(entity.getGuid())) { entities.put(entity.getGuid(), entity); @@ -163,23 +171,6 @@ public class EntityLineageService implements AtlasLineageService { return lineageQuery; } - private AtlasEntityHeader toAtlasEntityHeader(Object vertexObj) { - AtlasEntityHeader ret = new AtlasEntityHeader(); - - if (vertexObj instanceof AtlasVertex) { - AtlasVertex vertex = (AtlasVertex) vertexObj; - ret.setTypeName(vertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class)); - ret.setGuid(vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class)); - ret.setDisplayText(vertex.getProperty(Constants.QUALIFIED_NAME, String.class)); - - String state = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class); - Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.ACTIVE : Status.DELETED); - ret.setStatus(status); - } - - return ret; - } - private boolean entityExists(String guid) { boolean ret = false; Iterator<AtlasVertex> results = graph.query() diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index 44ca854..ea5bf93 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -17,8 +17,8 @@ */ package org.apache.atlas.repository.store.graph.v1; -import com.google.common.base.Optional; import com.sun.istack.Nullable; +import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.exception.AtlasBaseException; @@ -27,9 +27,11 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; @@ -120,6 +122,10 @@ public final class EntityGraphRetriever { return ret; } + public AtlasEntityHeader toAtlasEntityHeader(AtlasVertex entityVertex) throws AtlasBaseException { + return entityVertex != null ? mapVertexToAtlasEntityHeader(entityVertex) : null; + } + private AtlasVertex getEntityVertex(String guid) throws AtlasBaseException { try { return graphHelper.getVertexForGUID(guid); @@ -171,6 +177,41 @@ public final class EntityGraphRetriever { return entity; } + private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) throws AtlasBaseException { + AtlasEntityHeader ret = new AtlasEntityHeader(); + + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); + String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); + + ret.setTypeName(typeName); + ret.setGuid(guid); + ret.setStatus(GraphHelper.getStatus(entityVertex)); + ret.setClassificationNames(GraphHelper.getTraitNames(entityVertex)); + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + + if (entityType != null) { + Object name = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.NAME)); + Object description = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.DESCRIPTION)); + Object owner = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.OWNER)); + Object displayText = name; + + if (displayText == null) { + displayText = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.QUALIFIED_NAME)); + } + + ret.setAttribute(AtlasClient.NAME, name); + ret.setAttribute(AtlasClient.DESCRIPTION, description); + ret.setAttribute(AtlasClient.OWNER, owner); + + if (displayText != null) { + ret.setDisplayText(displayText.toString()); + } + } + + return ret; + } + private AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity entity) { if (LOG.isDebugEnabled()) { LOG.debug("Mapping system attributes for type {}", entity.getTypeName()); @@ -495,4 +536,8 @@ public final class EntityGraphRetriever { return ret; } + + private Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException { + return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null) : null; + } } \ No newline at end of file