Skip to content

A
atlas
Commit 9d4380e0 by Ayush Nigam Committed by Madhan Neethiraj
Options

ATLAS-3110: added REST API to get multiple entities given their unique attribute values

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent f070d924
Showing with 220 additions and 26 deletions
client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
......@@ -27,7 +27,6 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.instance.EntityMutationResponse;
......@@ -56,6 +55,7 @@ public class AtlasClientV2 extends AtlasBaseClient {
// Entity APIs
public static final String ENTITY_API = BASE_URI + "v2/entity/";
private static final String PREFIX_ATTR = "attr:";
private static final String PREFIX_ATTR_ = "attr_";
private static final String TYPEDEFS_API = TYPES_API + "typedefs/";
private static final String TYPEDEF_BY_NAME = TYPES_API + "typedef/name/";
private static final String TYPEDEF_BY_GUID = TYPES_API + "typedef/guid/";
......@@ -271,12 +271,12 @@ public class AtlasClientV2 extends AtlasBaseClient {
return callAPI(API_V2.GET_ENTITY_BY_GUID, AtlasEntityWithExtInfo.class, queryParams, guid);
}
public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map<String, String> attributes) throws AtlasServiceException {
return getEntityByAttribute(type, attributes, false, false);
public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map<String, String> uniqAttributes) throws AtlasServiceException {
return getEntityByAttribute(type, uniqAttributes, false, false);
}
public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map<String, String> attributes, boolean minExtInfo, boolean ignoreRelationship) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(attributes);
public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map<String, String> uniqAttributes, boolean minExtInfo, boolean ignoreRelationship) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributes);
queryParams.add("minExtInfo", String.valueOf(minExtInfo));
queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationship));
......@@ -284,17 +284,31 @@ public class AtlasClientV2 extends AtlasBaseClient {
return callAPI(API_V2.GET_ENTITY_BY_ATTRIBUTE, AtlasEntityWithExtInfo.class, queryParams, type);
}
public EntityMutationResponse updateEntityByAttribute(String type, Map<String, String> attributes, AtlasEntityWithExtInfo entityInfo)
public AtlasEntitiesWithExtInfo getEntitiesByAttribute(String type, List<Map<String,String>> uniqAttributesList) throws AtlasServiceException {
return getEntitiesByAttribute(type, uniqAttributesList, false, false);
}
public AtlasEntitiesWithExtInfo getEntitiesByAttribute(String type, List<Map<String, String>> uniqAttributesList, boolean minExtInfo, boolean ignoreRelationship) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributesList, null);
queryParams.add("minExtInfo", String.valueOf(minExtInfo));
queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationship));
return callAPI(API_V2.GET_ENTITIES_BY_ATTRIBUTES, AtlasEntitiesWithExtInfo.class, queryParams, type);
}
public EntityMutationResponse updateEntityByAttribute(String type, Map<String, String> uniqAttributes, AtlasEntityWithExtInfo entityInfo)
throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(attributes);
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributes);
return callAPI(API_V2.UPDATE_ENTITY_BY_ATTRIBUTE, EntityMutationResponse.class, entityInfo, queryParams, type);
}
/* Lineage Calls */
public EntityMutationResponse deleteEntityByAttribute(String type, Map<String, String> attributes) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(attributes);
public EntityMutationResponse deleteEntityByAttribute(String type, Map<String, String> uniqAttributes) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributes);
return callAPI(API_V2.DELETE_ENTITY_BY_ATTRIBUTE, EntityMutationResponse.class, queryParams, type);
}
......@@ -462,6 +476,23 @@ public class AtlasClientV2 extends AtlasBaseClient {
return queryParams;
}
private MultivaluedMap<String, String> attributesToQueryParams(List<Map<String, String>> attributesList,
MultivaluedMap<String, String> queryParams) {
if (queryParams == null) {
queryParams = new MultivaluedMapImpl();
}
for (int i = 0; i < attributesList.size(); i++) {
Map<String, String> attributes = attributesList.get(i);
for (Map.Entry<String, String> entry : attributes.entrySet()) {
queryParams.putSingle(PREFIX_ATTR_ + i + ":" + entry.getKey(), entry.getValue());
}
}
return queryParams;
}
private <T> T getTypeDefByName(final String name, Class<T> typeDefClass) throws AtlasServiceException {
String atlasPath = getAtlasPath(typeDefClass);
API api = new API(String.format(GET_BY_NAME_TEMPLATE, atlasPath, name), HttpMethod.GET, Response.Status.OK);
......@@ -502,6 +533,7 @@ public class AtlasClientV2 extends AtlasBaseClient {
public static final API_V2 UPDATE_ENTITY_BY_ATTRIBUTE = new API_V2(ENTITY_API + "uniqueAttribute/type/", HttpMethod.PUT, Response.Status.OK);
public static final API_V2 DELETE_ENTITY_BY_GUID = new API_V2(ENTITY_API + "guid/", HttpMethod.DELETE, Response.Status.OK);
public static final API_V2 DELETE_ENTITY_BY_ATTRIBUTE = new API_V2(ENTITY_API + "uniqueAttribute/type/", HttpMethod.DELETE, Response.Status.OK);
public static final API_V2 GET_ENTITIES_BY_ATTRIBUTES = new API_V2(ENTITY_BULK_API + "uniqueAttribute/type/", HttpMethod.GET, Response.Status.OK);
public static final API_V2 GET_ENTITIES_BY_GUIDS = new API_V2(ENTITY_BULK_API, HttpMethod.GET, Response.Status.OK);
public static final API_V2 CREATE_ENTITIES = new API_V2(ENTITY_BULK_API, HttpMethod.POST, Response.Status.OK);
public static final API_V2 UPDATE_ENTITIES = new API_V2(ENTITY_BULK_API, HttpMethod.POST, Response.Status.OK);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
......@@ -89,6 +89,17 @@ public interface AtlasEntityStore {
AtlasEntitiesWithExtInfo getByIds(List<String> guid, boolean isMinExtInfo, boolean ignoreRelationships) throws AtlasBaseException;
/**
* Batch GET to retrieve entities by their uniqueIds
* @param entityType
* @param uniqueAttributes
* @param isMinExtInfo
* @param ignoreRelationships
* @return
* @throws AtlasBaseException
*/
AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(AtlasEntityType entityType, List<Map<String, Object>> uniqueAttributes, boolean isMinExtInfo, boolean ignoreRelationships) throws AtlasBaseException;
/**
*
* Get an eneity by its unique attribute
* @param entityType type of the entity
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
......@@ -189,6 +189,30 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
@Override
@GraphTransaction
public AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(AtlasEntityType entityType, List<Map<String, Object>> uniqueAttributes , boolean isMinExtInfo, boolean ignoreRelationships) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getEntitiesByUniqueAttributes({}, {})", entityType.getTypeName(), uniqueAttributes);
}
EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry, ignoreRelationships);
AtlasEntitiesWithExtInfo ret = entityRetriever.getEntitiesByUniqueAttributes(entityType.getTypeName(), uniqueAttributes, isMinExtInfo);
if (ret != null && ret.getEntities() != null) {
for (AtlasEntity entity : ret.getEntities()) {
AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(entity)), "read entity: typeName=", entityType.getTypeName(), ", guid=", entity.getGuid());
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== getEntitiesByUniqueAttributes({}, {}): {}", entityType.getTypeName(), uniqueAttributes, ret);
}
return ret;
}
@Override
@GraphTransaction
public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
throws AtlasBaseException {
return getByUniqueAttributes(entityType, uniqAttributes, false, false);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
......@@ -35,7 +35,6 @@ import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.AtlasRelationship.AtlasRelationshipWithExtInfo;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
......@@ -45,7 +44,6 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
......@@ -119,8 +117,6 @@ import static org.apache.atlas.repository.graph.GraphHelper.getPrimitiveMap;
import static org.apache.atlas.repository.graph.GraphHelper.getReferenceMap;
import static org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagationEnabledClassificationVertices;
import static org.apache.atlas.repository.graph.GraphHelper.getRelationshipGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getRemovePropagations;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
......@@ -375,6 +371,33 @@ public class EntityGraphRetriever {
return ret;
}
public AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(String typeName, List<Map<String, Object>> uniqueAttributesList, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEntitiesWithExtInfo ret = new AtlasEntitiesWithExtInfo();
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType != null) {
for (Map<String, Object> uniqAttributes : uniqueAttributesList) {
try {
AtlasVertex vertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(entityType, uniqAttributes);
if (vertex != null) {
AtlasEntity entity = mapVertexToAtlasEntity(vertex, ret, isMinExtInfo);
ret.addEntity(entity);
}
} catch(AtlasBaseException e) {
if (e.getAtlasErrorCode() != AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND) {
throw e;
}
}
}
}
ret.compact();
return ret;
}
private AtlasVertex getEntityVertex(AtlasObjectId objId) throws AtlasBaseException {
AtlasVertex ret = null;
......
webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java
......@@ -81,7 +81,9 @@ public class EntityREST {
private static final Logger LOG = LoggerFactory.getLogger(EntityREST.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.EntityREST");
public static final String PREFIX_ATTR = "attr:";
public static final String PREFIX_ATTR = "attr:";
public static final String PREFIX_ATTR_ = "attr_";
private final AtlasTypeRegistry typeRegistry;
private final AtlasEntityStore entitiesStore;
......@@ -159,6 +161,8 @@ public class EntityREST {
* GET /v2/entity/uniqueAttribute/type/aType?attr:aTypeAttribute=someValue
*
* @param typeName
* @param minExtInfo
* @param ignoreRelationships
* @return AtlasEntityWithExtInfo
* @throws AtlasBaseException
*/
......@@ -582,6 +586,54 @@ public class EntityREST {
/******************************************************************/
/**
* Bulk API to retrieve list of entities identified by its unique attributes.
*
* In addition to the typeName path parameter, attribute key-value pair(s) can be provided in the following format
*
* typeName=<typeName>&attr_1:<attrName>=<attrValue>&attr_2:<attrName>=<attrValue>&attr_3:<attrName>=<attrValue>
*
* NOTE: The attrName should be an unique attribute for the given entity-type
*
* The REST request would look something like this
*
* GET /v2/entity/bulk/uniqueAttribute/type/hive_db?attrs_0:qualifiedName=db1@cl1&attrs_2:qualifiedName=db2@cl1
*
* @param typeName
* @param minExtInfo
* @param ignoreRelationships
* @return AtlasEntitiesWithExtInfo
* @throws AtlasBaseException
*/
@GET
@Path("/bulk/uniqueAttribute/type/{typeName}")
public AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(@PathParam("typeName") String typeName,
@QueryParam("minExtInfo") @DefaultValue("false") boolean minExtInfo,
@QueryParam("ignoreRelationships") @DefaultValue("false") boolean ignoreRelationships,
@Context HttpServletRequest servletRequest) throws AtlasBaseException {
Servlets.validateQueryParamLength("typeName", typeName);
AtlasPerfTracer perf = null;
try {
List<Map<String, Object>> uniqAttributesList = getAttributesList(servletRequest);
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getEntitiesByUniqueAttributes(" + typeName + "," + uniqAttributesList + ")");
}
AtlasEntityType entityType = ensureEntityType(typeName);
for (Map<String, Object> uniqAttributes : uniqAttributesList) {
validateUniqueAttribute(entityType, uniqAttributes);
}
return entitiesStore.getEntitiesByUniqueAttributes(entityType, uniqAttributesList, minExtInfo, ignoreRelationships);
} finally {
AtlasPerfTracer.log(perf);
}
}
/**
* Bulk API to retrieve list of entities identified by its GUIDs.
*/
@GET
......@@ -781,6 +833,7 @@ public class EntityREST {
return ret;
}
// attr:qualifiedName=db1@cl1 ==> { qualifiedName:db1@cl1 }
private Map<String, Object> getAttributes(HttpServletRequest request) {
Map<String, Object> attributes = new HashMap<>();
......@@ -800,6 +853,43 @@ public class EntityREST {
return attributes;
}
// attrs_1:qualifiedName=db1@cl1&attrs_2:qualifiedName=db2@cl1 ==> [ { qualifiedName:db1@cl1 }, { qualifiedName:db2@cl1 } ]
private List<Map<String, Object>> getAttributesList(HttpServletRequest request) {
Map<String, Map<String, Object>> ret = new HashMap<>();
if (MapUtils.isNotEmpty(request.getParameterMap())) {
for (Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
String key = entry.getKey();
if (key == null || !key.startsWith(PREFIX_ATTR_)) {
continue;
}
int sepPos = key.indexOf(':', PREFIX_ATTR_.length());
String[] values = entry.getValue();
String value = values != null && values.length > 0 ? values[0] : null;
if (sepPos == -1 || value == null) {
continue;
}
String attrName = key.substring(sepPos + 1);
String listIdx = key.substring(PREFIX_ATTR_.length(), sepPos);
Map<String, Object> attributes = ret.get(listIdx);
if (attributes == null) {
attributes = new HashMap<>();
ret.put(listIdx, attributes);
}
attributes.put(attrName, value);
}
}
return new ArrayList<>(ret.values());
}
/**
* Validate that each attribute given is an unique attribute
* @param entityType the entity type
......
webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java
......@@ -23,6 +23,7 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
......@@ -31,24 +32,14 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static org.apache.atlas.AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME;
import static org.apache.atlas.examples.QuickStartV2.CLUSTER_SUFFIX;
import static org.apache.atlas.examples.QuickStartV2.LOAD_PROCESS_TYPE;
import static org.apache.atlas.examples.QuickStartV2.LOAD_SALES_DAILY_PROCESS;
import static org.apache.atlas.examples.QuickStartV2.LOAD_SALES_MONTHLY_PROCESS;
import static org.apache.atlas.examples.QuickStartV2.PRODUCT_DIM_TABLE;
import static org.apache.atlas.examples.QuickStartV2.PRODUCT_DIM_VIEW;
import static org.apache.atlas.examples.QuickStartV2.SALES_DB;
import static org.apache.atlas.examples.QuickStartV2.SALES_FACT_DAILY_MV_TABLE;
import static org.apache.atlas.examples.QuickStartV2.SALES_FACT_MONTHLY_MV_TABLE;
import static org.apache.atlas.examples.QuickStartV2.SALES_FACT_TABLE;
import static org.apache.atlas.examples.QuickStartV2.TIME_DIM_TABLE;
import static org.apache.atlas.examples.QuickStartV2.VIEW_TYPE;
import static org.apache.atlas.examples.QuickStartV2.*;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.AssertJUnit.assertTrue;
......@@ -91,6 +82,19 @@ public class QuickStartV2IT extends BaseResourceIT {
verifyTrait(table);
}
@Test
public void testTablesAreAdded2() throws AtlasServiceException {
List<String> tableNames = Arrays.asList(SALES_FACT_TABLE, PRODUCT_DIM_TABLE, CUSTOMER_DIM_TABLE,
TIME_DIM_TABLE, SALES_FACT_DAILY_MV_TABLE, SALES_FACT_MONTHLY_MV_TABLE,
LOG_FACT_DAILY_MV_TABLE, LOG_FACT_MONTHLY_MV_TABLE);
AtlasEntitiesWithExtInfo entities = getTables(tableNames);
assertNotNull(entities);
assertNotNull(entities.getEntities());
assertEquals(entities.getEntities().size(), tableNames.size());
}
private AtlasEntity getTable(String tableName) throws AtlasServiceException {
Map<String, String> attributes = Collections.singletonMap(REFERENCEABLE_ATTRIBUTE_NAME, tableName + CLUSTER_SUFFIX);
AtlasEntity tableEntity = atlasClientV2.getEntityByAttribute(QuickStartV2.TABLE_TYPE, attributes).getEntity();
......@@ -98,6 +102,16 @@ public class QuickStartV2IT extends BaseResourceIT {
return tableEntity;
}
private AtlasEntitiesWithExtInfo getTables(List<String> tableNames) throws AtlasServiceException {
List<Map<String, String>> attributesList = new ArrayList<>();
for (String tableName : tableNames) {
attributesList.add(Collections.singletonMap(REFERENCEABLE_ATTRIBUTE_NAME, tableName + CLUSTER_SUFFIX));
}
return atlasClientV2.getEntitiesByAttribute(QuickStartV2.TABLE_TYPE, attributesList);
}
private AtlasEntity getProcess(String processName) throws AtlasServiceException {
Map<String, String> attributes = Collections.singletonMap(REFERENCEABLE_ATTRIBUTE_NAME, processName + CLUSTER_SUFFIX);
AtlasEntity processEntity = atlasClientV2.getEntityByAttribute(LOAD_PROCESS_TYPE, attributes).getEntity();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment