Skip to content

A
atlas
Commit fbb244c2 by Abhishek Kadam Committed by apoorvnaik
Options

ATLAS-2815: UI Slow loading enhancements

1. Updated REST APIs that read entities with an option to include only minimum attributes for referred entities 2. UI changes for minExtInfo Change-Id: I633b46cc79de8ec807236719d8b5828aef537208 Signed-off-by: 's avatarapoorvnaik <apoorvnaik@apache.org>
parent 651ecd9e
Showing with 345 additions and 157 deletions
dashboardv2/public/js/models/VEntity.js
......@@ -43,7 +43,17 @@ define(['require',
*************************/
getEntity: function(token, options) {
var url = UrlLinks.entitiesApiUrl(token);
var url = UrlLinks.entitiesApiUrl({ guid: token });
options = _.extend({
contentType: 'application/json',
dataType: 'json'
}, options);
return this.constructor.nonCrudOperation.call(this, url, 'GET', options);
},
getEntityHeader: function(token, options) {
var url = UrlLinks.entityHeaderApiUrl(token);
options = _.extend({
contentType: 'application/json',
......
dashboardv2/public/js/models/VSearch.js
......@@ -41,7 +41,7 @@ define(['require',
* Non - CRUD operations
*************************/
getEntity: function(id, options) {
var url = UrlLinks.entitiesApiUrl(id);
var url = UrlLinks.entitiesApiUrl({guid: id});
options = _.extend({
contentType: 'application/json',
......
dashboardv2/public/js/models/VTag.js
......@@ -41,7 +41,7 @@ define(['require',
* Non - CRUD operations
*************************/
deleteAssociation: function(guid, name, options) {
var url = UrlLinks.entitiesApiUrl(guid, name);
var url = UrlLinks.entitiesApiUrl({ guid: guid, name: name });
options = _.extend({
contentType: 'application/json',
dataType: 'json'
......
dashboardv2/public/js/router/Router.js
......@@ -125,7 +125,7 @@ define([
'id': id,
'value': paramObj
}, that.preFetchedCollectionLists, that.sharedObj)));
this.entityCollection.url = UrlLinks.entitiesApiUrl(id);
this.entityCollection.url = UrlLinks.entitiesApiUrl({ guid: id, minExtInfo: true });
this.entityCollection.fetch({ reset: true });
});
}
......
dashboardv2/public/js/utils/CommonViewFunction.js
......@@ -81,11 +81,11 @@ define(['require', 'utils/Utils', 'modules/Modal', 'utils/Messages', 'utils/Enum
var table = "",
fetchInputOutputValue = function(id) {
var that = this;
scope.entityModel.getEntity(id, {
scope.entityModel.getEntityHeader(id, {
success: function(serverData) {
var value = "",
deleteButton = "",
data = serverData.entity;
data = serverData;
value = Utils.getName(data);
var id = "";
if (data.guid) {
......
dashboardv2/public/js/utils/UrlLinks.js
......@@ -48,16 +48,28 @@ define(['require', 'utils/Enums', 'utils/Utils', 'underscore'], function(require
return defApiUrl.defs + '?excludeInternalTypesAndReferences=true&type=' + type;
}
},
entitiesApiUrl: function(guid, name) {
entitiesApiUrl: function(options) {
var entitiesUrl = this.baseUrlV2 + '/entity';
if (options) {
var guid = options.guid,
name = options.name,
minExtInfo = options.minExtInfo;
if (guid && name) {
return entitiesUrl + '/guid/' + guid + '/classification/' + name;
entitiesUrl += '/guid/' + guid + '/classification/' + name;
} else if (guid && !name) {
return entitiesUrl + '/guid/' + guid;
} else {
entitiesUrl += '/guid/' + guid;
}
}
if (!minExtInfo) {
return entitiesUrl;
} else {
return entitiesUrl += '?minExtInfo=' + (minExtInfo);
}
},
entityHeaderApiUrl: function(guid) {
return this.entitiesApiUrl({ guid: guid }) + "/header"
},
entitiesTraitsApiUrl: function(token) {
if (token) {
return this.baseUrlV2 + '/entity/guid/' + token + '/classifications';
......@@ -69,6 +81,9 @@ define(['require', 'utils/Enums', 'utils/Utils', 'underscore'], function(require
entityCollectionaudit: function(guid) {
return this.baseUrlV2 + '/entity/' + guid + '/audit';
},
replicationCollectionaudit: function(name) {
return this.baseUrl + '/admin/cluster/audit/' + name;
},
classicationApiUrl: function(name, guid) {
var typeUrl = this.typedefsUrl();
if (name) {
......
dashboardv2/public/js/views/entity/CreateEntityLayoutView.js
......@@ -184,7 +184,7 @@ define(['require',
},
fetchCollections: function() {
if (this.guid) {
this.collection.url = UrlLinks.entitiesApiUrl(this.guid);
this.collection.url = UrlLinks.entitiesApiUrl({ guid: this.guid });
this.collection.fetch({ reset: true });
} else {
this.entityCollectionList();
......@@ -212,7 +212,7 @@ define(['require',
_.each(attrObj, function(obj) {
if (obj.guid && !referredEntities[obj.guid]) {
++that.asyncReferEntityCounter;
that.collection.url = UrlLinks.entitiesApiUrl(obj.guid);
that.collection.url = UrlLinks.entitiesApiUrl({ guid: obj.guid });
that.collection.fetch({
success: function(data, response) {
referredEntities[obj.guid] = response.entity;
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......@@ -47,6 +47,14 @@ import java.util.Set;
public class AtlasEntityType extends AtlasStructType {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityType.class);
private static final String NAME = "name";
private static final String DESCRIPTION = "description";
private static final String OWNER = "owner";
private static final String CREATE_TIME = "createTime";
private static final String[] ENTITY_HEADER_ATTRIBUTES = new String[]{NAME, DESCRIPTION, OWNER, CREATE_TIME};
private static final String OPTION_SCHEMA_ATTRIBUTES = "schemaAttributes";
private final AtlasEntityDef entityDef;
private final String typeQryStr;
......@@ -62,6 +70,8 @@ public class AtlasEntityType extends AtlasStructType {
private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
private String typeAndAllSubTypesQryStr = "";
private boolean isInternalType = false;
private Map<String, AtlasAttribute> headerAttributes = Collections.emptyMap();
private Map<String, AtlasAttribute> minInfoAttributes = Collections.emptyMap();
public AtlasEntityType(AtlasEntityDef entityDef) {
......@@ -80,7 +90,9 @@ public class AtlasEntityType extends AtlasStructType {
resolveReferences(typeRegistry);
}
public AtlasEntityDef getEntityDef() { return entityDef; }
public AtlasEntityDef getEntityDef() {
return entityDef;
}
@Override
void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
......@@ -96,7 +108,7 @@ public class AtlasEntityType extends AtlasStructType {
AtlasType superType = typeRegistry.getType(superTypeName);
if (superType instanceof AtlasEntityType) {
s.add((AtlasEntityType)superType);
s.add((AtlasEntityType) superType);
} else {
throw new AtlasBaseException(AtlasErrorCode.INCOMPATIBLE_SUPERTYPE, superTypeName, entityDef.getName());
}
......@@ -117,6 +129,34 @@ public class AtlasEntityType extends AtlasStructType {
this.typeAndAllSuperTypes = new HashSet<>(this.allSuperTypes);
this.typeAndAllSuperTypes.add(this.getTypeName());
this.typeAndAllSuperTypes = Collections.unmodifiableSet(this.typeAndAllSuperTypes);
// headerAttributes includes uniqAttributes & ENTITY_HEADER_ATTRIBUTES
this.headerAttributes = new HashMap<>(this.uniqAttributes);
for (String headerAttributeName : ENTITY_HEADER_ATTRIBUTES) {
AtlasAttribute headerAttribute = getAttribute(headerAttributeName);
if (headerAttribute != null) {
this.headerAttributes.put(headerAttributeName, headerAttribute);
}
}
// minInfoAttributes includes all headerAttributes & schema-attributes
this.minInfoAttributes = new HashMap<>(this.headerAttributes);
Map<String, String> typeDefOptions = entityDef.getOptions();
String jsonList = typeDefOptions != null ? typeDefOptions.get(OPTION_SCHEMA_ATTRIBUTES) : null;
List<String> schemaAttributeNames = StringUtils.isNotEmpty(jsonList) ? AtlasType.fromJson(jsonList, List.class) : null;
if (CollectionUtils.isNotEmpty(schemaAttributeNames)) {
for (String schemaAttributeName : schemaAttributeNames) {
AtlasAttribute schemaAttribute = getAttribute(schemaAttributeName);
if (schemaAttribute != null) {
this.minInfoAttributes.put(schemaAttributeName, schemaAttribute);
}
}
}
}
@Override
......@@ -185,13 +225,25 @@ public class AtlasEntityType extends AtlasStructType {
return allSuperTypes;
}
public Set<String> getSubTypes() { return subTypes; }
public Set<String> getSubTypes() {
return subTypes;
}
public Set<String> getAllSubTypes() {
return allSubTypes;
}
public Set<String> getTypeAndAllSubTypes() {
return typeAndAllSubTypes;
}
public Set<String> getAllSubTypes() { return allSubTypes; }
public Set<String> getTypeAndAllSuperTypes() {
return typeAndAllSuperTypes;
}
public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
public Map<String, AtlasAttribute> getHeaderAttributes() { return headerAttributes; }
public Set<String> getTypeAndAllSuperTypes() { return typeAndAllSuperTypes; }
public Map<String, AtlasAttribute> getMinInfoAttributes() { return minInfoAttributes; }
public boolean isSuperTypeOf(AtlasEntityType entityType) {
return entityType != null && allSubTypes.contains(entityType.getTypeName());
......@@ -217,9 +269,13 @@ public class AtlasEntityType extends AtlasStructType {
return isInternalType;
}
public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; }
public Map<String, AtlasAttribute> getRelationshipAttributes() {
return relationshipAttributes;
}
public AtlasAttribute getRelationshipAttribute(String attributeName) { return relationshipAttributes.get(attributeName); }
public AtlasAttribute getRelationshipAttribute(String attributeName) {
return relationshipAttributes.get(attributeName);
}
// this method should be called from AtlasRelationshipType.resolveReferencesPhase2()
void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) {
......@@ -254,7 +310,9 @@ public class AtlasEntityType extends AtlasStructType {
return typeAndAllSubTypesQryStr;
}
public String getTypeQryStr() { return typeQryStr; }
public String getTypeQryStr() {
return typeQryStr;
}
public boolean hasRelationshipAttribute(String attributeName) {
return relationshipAttributes.containsKey(attributeName);
......@@ -280,7 +338,7 @@ public class AtlasEntityType extends AtlasStructType {
}
@Override
public AtlasEntity createDefaultValue(Object defaultValue){
public AtlasEntity createDefaultValue(Object defaultValue) {
AtlasEntity ret = new AtlasEntity(entityDef.getName());
populateDefaultValues(ret);
......@@ -528,6 +586,7 @@ public class AtlasEntityType extends AtlasStructType {
}
}
}
boolean isAssignableFrom(AtlasObjectId objId) {
boolean ret = AtlasTypeUtil.isValid(objId) && (StringUtils.equals(objId.getTypeName(), getTypeName()) || isSuperTypeOf(objId.getTypeName()));
......@@ -605,6 +664,7 @@ public class AtlasEntityType extends AtlasStructType {
return ret;
}
private boolean isValidRelationshipType(AtlasType attributeType) {
boolean ret = false;
......@@ -700,7 +760,7 @@ public class AtlasEntityType extends AtlasStructType {
}
} else if (obj instanceof Map) {
Map attributes = AtlasTypeUtil.toStructAttributes((Map)obj);
Map attributes = AtlasTypeUtil.toStructAttributes((Map) obj);
for (AtlasAttribute attribute : relationshipAttributes.values()) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
......@@ -21,6 +21,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.store.graph.v2.EntityStream;
......@@ -51,6 +52,23 @@ public interface AtlasEntityStore {
AtlasEntityWithExtInfo getById(String guid) throws AtlasBaseException;
/**
*
* Get entity definition by its guid
* @param guid
* @param isMinExtInfo
* @return AtlasEntity
*/
AtlasEntityWithExtInfo getById(String guid, boolean isMinExtInfo) throws AtlasBaseException;
/**
* Get entity header for the given GUID
* @param guid
* @return
* @throws AtlasBaseException
*/
AtlasEntityHeader getHeaderById(String guid) throws AtlasBaseException;
/**
* Batch GET to retrieve entities by their ID
* @param guid
* @return
......@@ -59,6 +77,15 @@ public interface AtlasEntityStore {
AtlasEntitiesWithExtInfo getByIds(List<String> guid) throws AtlasBaseException;
/**
* Batch GET to retrieve entities by their ID
* @param guid
* @param isMinExtInfo
* @return
* @throws AtlasBaseException
*/
AtlasEntitiesWithExtInfo getByIds(List<String> guid, boolean isMinExtInfo) throws AtlasBaseException;
/**
*
* Get an eneity by its unique attribute
* @param entityType type of the entity
......@@ -69,6 +96,17 @@ public interface AtlasEntityStore {
throws AtlasBaseException;
/**
*
* Get an eneity by its unique attribute
* @param entityType type of the entity
* @param uniqAttributes Attributes that uniquely identify the entity
* @param isMinExtInfo
* @return EntityMutationResponse details of the updates performed by this call
*/
AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes, boolean isMinExtInfo)
throws AtlasBaseException;
/**
* Create or update entities in the stream
* @param entityStream AtlasEntityStream
* @return EntityMutationResponse Entity mutations operations with the corresponding set of entities on which these operations were performed
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
......@@ -101,57 +101,98 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
@Override
@GraphTransaction
public AtlasEntityWithExtInfo getById(String guid) throws AtlasBaseException {
return getById(guid, false);
}
@Override
public AtlasEntityWithExtInfo getById(final String guid, final boolean isMinExtInfo) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getById({})", guid);
LOG.debug("==> getById({}, {})", guid, isMinExtInfo);
}
AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(guid);
EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry);
AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(ret.getEntity())), "read entity: guid=", guid);
AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(guid, isMinExtInfo);
if (ret == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== getById({}): {}", guid, ret);
LOG.debug("<== getById({}, {}): {}", guid, isMinExtInfo, ret);
}
return ret;
}
@Override
@GraphTransaction
public AtlasEntitiesWithExtInfo getByIds(List<String> guids) throws AtlasBaseException {
public AtlasEntityHeader getHeaderById(final String guid) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getByIds({})", guids);
LOG.debug("==> getHeaderById({})", guid);
}
AtlasEntitiesWithExtInfo ret = entityRetriever.toAtlasEntitiesWithExtInfo(guids);
EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry);
// verify authorization to read the entities
if(ret != null){
for(String guid : guids){
AtlasEntity entity = ret.getEntity(guid);
AtlasEntityHeader ret = entityRetriever.toAtlasEntityHeader(guid);
if (ret == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== getHeaderById({}): {}", guid, ret);
}
return ret;
}
AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(entity)), "read entity: guid=", guid);
@Override
public AtlasEntitiesWithExtInfo getByIds(List<String> guids) throws AtlasBaseException {
return getByIds(guids, false);
}
@Override
@GraphTransaction
public AtlasEntitiesWithExtInfo getByIds(List<String> guids, boolean isMinExtInfo) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getByIds({}, {})", guids, isMinExtInfo);
}
EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry);
AtlasEntitiesWithExtInfo ret = entityRetriever.toAtlasEntitiesWithExtInfo(guids, isMinExtInfo);
if (LOG.isDebugEnabled()) {
LOG.debug("<== getByIds({}): {}", guids, ret);
LOG.debug("<== getByIds({}, {}): {}", guids, isMinExtInfo, ret);
}
return ret;
}
@Override
public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
throws AtlasBaseException {
return getByUniqueAttributes(entityType, uniqAttributes, false);
}
@Override
@GraphTransaction
public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes) throws AtlasBaseException {
public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes, boolean isMinExtInfo)
throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getByUniqueAttribute({}, {})", entityType.getTypeName(), uniqAttributes);
}
AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(entityType, uniqAttributes);
AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex);
AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(ret.getEntity())), "read entity: typeName=", entityType.getTypeName(), ", uniqueAttributes=", uniqAttributes);
EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry);
AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex, isMinExtInfo);
if (ret == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(),
uniqAttributes.toString());
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== getByUniqueAttribute({}, {}): {}", entityType.getTypeName(), uniqAttributes, ret);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
......@@ -156,13 +156,21 @@ public final class EntityGraphRetriever {
return toAtlasEntityWithExtInfo(getEntityVertex(guid));
}
public AtlasEntityWithExtInfo toAtlasEntityWithExtInfo(String guid, boolean isMinExtInfo) throws AtlasBaseException {
return toAtlasEntityWithExtInfo(getEntityVertex(guid), isMinExtInfo);
}
public AtlasEntityWithExtInfo toAtlasEntityWithExtInfo(AtlasObjectId objId) throws AtlasBaseException {
return toAtlasEntityWithExtInfo(getEntityVertex(objId));
}
public AtlasEntityWithExtInfo toAtlasEntityWithExtInfo(AtlasVertex entityVertex) throws AtlasBaseException {
return toAtlasEntityWithExtInfo(entityVertex, false);
}
public AtlasEntityWithExtInfo toAtlasEntityWithExtInfo(AtlasVertex entityVertex, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEntityExtInfo entityExtInfo = new AtlasEntityExtInfo();
AtlasEntity entity = mapVertexToAtlasEntity(entityVertex, entityExtInfo);
AtlasEntity entity = mapVertexToAtlasEntity(entityVertex, entityExtInfo, isMinExtInfo);
AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(entity, entityExtInfo);
ret.compact();
......@@ -171,19 +179,7 @@ public final class EntityGraphRetriever {
}
public AtlasEntitiesWithExtInfo toAtlasEntitiesWithExtInfo(List<String> guids) throws AtlasBaseException {
AtlasEntitiesWithExtInfo ret = new AtlasEntitiesWithExtInfo();
for (String guid : guids) {
AtlasVertex vertex = getEntityVertex(guid);
AtlasEntity entity = mapVertexToAtlasEntity(vertex, ret);
ret.addEntity(entity);
}
ret.compact();
return ret;
return toAtlasEntitiesWithExtInfo(guids, false);
}
public AtlasEntityHeader toAtlasEntityHeader(String guid) throws AtlasBaseException {
......@@ -323,6 +319,22 @@ public final class EntityGraphRetriever {
return ret;
}
public AtlasEntitiesWithExtInfo toAtlasEntitiesWithExtInfo(List<String> guids, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEntitiesWithExtInfo ret = new AtlasEntitiesWithExtInfo();
for (String guid : guids) {
AtlasVertex vertex = getEntityVertex(guid);
AtlasEntity entity = mapVertexToAtlasEntity(vertex, ret, isMinExtInfo);
ret.addEntity(entity);
}
ret.compact();
return ret;
}
private AtlasVertex getEntityVertex(AtlasObjectId objId) throws AtlasBaseException {
AtlasVertex ret = null;
......@@ -347,7 +359,11 @@ public final class EntityGraphRetriever {
}
private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
String guid = getGuid(entityVertex);
return mapVertexToAtlasEntity(entityVertex, entityExtInfo, false);
}
private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
String guid = GraphHelper.getGuid(entityVertex);
AtlasEntity entity = entityExtInfo != null ? entityExtInfo.getEntity(guid) : null;
if (entity == null) {
......@@ -363,11 +379,44 @@ public final class EntityGraphRetriever {
mapSystemAttributes(entityVertex, entity);
mapAttributes(entityVertex, entity, entityExtInfo);
mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
mapRelationshipAttributes(entityVertex, entity);
mapClassifications(entityVertex, entity);
}
return entity;
}
private AtlasEntity mapVertexToAtlasEntityMin(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
return mapVertexToAtlasEntityMin(entityVertex, entityExtInfo, null);
}
private AtlasEntity mapVertexToAtlasEntityMin(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, Set<String> attributes) throws AtlasBaseException {
String guid = GraphHelper.getGuid(entityVertex);
AtlasEntity entity = entityExtInfo != null ? entityExtInfo.getEntity(guid) : null;
if (entity == null) {
entity = new AtlasEntity();
if (entityExtInfo != null) {
entityExtInfo.addReferredEntity(guid, entity);
}
mapSystemAttributes(entityVertex, entity);
mapClassifications(entityVertex, entity);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (entityType != null) {
for (AtlasAttribute attribute : entityType.getMinInfoAttributes().values()) {
Object attrValue = getVertexAttribute(entityVertex, attribute);
if (attrValue != null) {
entity.setAttribute(attribute.getName(), attrValue);
}
}
}
}
return entity;
......@@ -395,25 +444,17 @@ public final class EntityGraphRetriever {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType != null) {
for (AtlasAttribute uniqueAttribute : entityType.getUniqAttributes().values()) {
Object attrValue = getVertexAttribute(entityVertex, uniqueAttribute);
for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) {
Object attrValue = getVertexAttribute(entityVertex, headerAttribute);
if (attrValue != null) {
ret.setAttribute(uniqueAttribute.getName(), attrValue);
ret.setAttribute(headerAttribute.getName(), attrValue);
}
}
Object name = getVertexAttribute(entityVertex, entityType.getAttribute(NAME));
Object description = getVertexAttribute(entityVertex, entityType.getAttribute(DESCRIPTION));
Object owner = getVertexAttribute(entityVertex, entityType.getAttribute(OWNER));
Object createTime = getVertexAttribute(entityVertex, entityType.getAttribute(CREATE_TIME));
Object name = ret.getAttribute(NAME);
Object displayText = name != null ? name : ret.getAttribute(QUALIFIED_NAME);
ret.setAttribute(NAME, name);
ret.setAttribute(DESCRIPTION, description);
ret.setAttribute(OWNER, owner);
ret.setAttribute(CREATE_TIME, createTime);
if (displayText != null) {
ret.setDisplayText(displayText.toString());
}
......@@ -432,7 +473,6 @@ public final class EntityGraphRetriever {
}
}
}
}
return ret;
......@@ -471,6 +511,10 @@ public final class EntityGraphRetriever {
}
private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
mapAttributes(entityVertex, struct, entityExtInfo, false);
}
private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(struct.getTypeName());
if (!(objType instanceof AtlasStructType)) {
......@@ -480,7 +524,7 @@ public final class EntityGraphRetriever {
AtlasStructType structType = (AtlasStructType) objType;
for (AtlasAttribute attribute : structType.getAllAttributes().values()) {
Object attrValue = mapVertexToAttribute(entityVertex, attribute, entityExtInfo);
Object attrValue = mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo);
struct.setAttribute(attribute.getName(), attrValue);
}
......@@ -596,7 +640,7 @@ public final class EntityGraphRetriever {
}
}
private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) throws AtlasBaseException {
Object ret = null;
AtlasType attrType = attribute.getAttributeType();
String vertexPropertyName = attribute.getQualifiedName();
......@@ -616,16 +660,16 @@ public final class EntityGraphRetriever {
ret = GraphHelper.getProperty(entityVertex, vertexPropertyName);
break;
case STRUCT:
ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo);
ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo, isMinExtInfo);
break;
case OBJECT_ID_TYPE:
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection);
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
break;
case ARRAY:
ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute);
ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
break;
case MAP:
ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
break;
case CLASSIFICATION:
// do nothing
......@@ -636,7 +680,7 @@ public final class EntityGraphRetriever {
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, final String propertyName, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException {
Map<String, Object> ret = null;
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
......@@ -656,7 +700,7 @@ public final class EntityGraphRetriever {
String mapKey = entry.getKey();
Object keyValue = entry.getValue();
Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, attribute.getRelationshipEdgeLabel(),
entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection());
entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(), isMinExtInfo);
if (mapValue != null) {
ret.put(mapKey, mapValue);
}
......@@ -674,7 +718,7 @@ public final class EntityGraphRetriever {
}
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType();
......@@ -701,7 +745,7 @@ public final class EntityGraphRetriever {
}
Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel,
entityExtInfo, isOwnedAttribute, edgeDirection);
entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
if (arrValue != null) {
arrValues.add(arrValue);
......@@ -713,7 +757,7 @@ public final class EntityGraphRetriever {
private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value,
String edgeLabel, AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
AtlasRelationshipEdgeDirection edgeDirection, final boolean isMinExtInfo) throws AtlasBaseException {
Object ret = null;
switch (arrayElement.getTypeCategory()) {
......@@ -728,11 +772,11 @@ public final class EntityGraphRetriever {
break;
case STRUCT:
ret = mapVertexToStruct(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo);
ret = mapVertexToStruct(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isMinExtInfo);
break;
case OBJECT_ID_TYPE:
ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection);
ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
break;
default:
......@@ -792,7 +836,7 @@ public final class EntityGraphRetriever {
private AtlasObjectId mapVertexToObjectId(AtlasVertex entityVertex, String edgeLabel, AtlasEdge edge,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
AtlasRelationshipEdgeDirection edgeDirection, final boolean isMinExtInfo) throws AtlasBaseException {
AtlasObjectId ret = null;
if (edge == null) {
......@@ -808,7 +852,13 @@ public final class EntityGraphRetriever {
if (referenceVertex != null) {
if (entityExtInfo != null && isOwnedAttribute) {
AtlasEntity entity = mapVertexToAtlasEntity(referenceVertex, entityExtInfo);
final AtlasEntity entity;
if (isMinExtInfo) {
entity = mapVertexToAtlasEntityMin(referenceVertex, entityExtInfo);
} else {
entity = mapVertexToAtlasEntity(referenceVertex, entityExtInfo);
}
if (entity != null) {
ret = AtlasTypeUtil.getAtlasObjectId(entity);
......@@ -822,7 +872,7 @@ public final class EntityGraphRetriever {
return ret;
}
private AtlasStruct mapVertexToStruct(AtlasVertex entityVertex, String edgeLabel, AtlasEdge edge, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
private AtlasStruct mapVertexToStruct(AtlasVertex entityVertex, String edgeLabel, AtlasEdge edge, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) throws AtlasBaseException {
AtlasStruct ret = null;
if (edge == null) {
......@@ -833,14 +883,14 @@ public final class EntityGraphRetriever {
final AtlasVertex referenceVertex = edge.getInVertex();
ret = new AtlasStruct(getTypeName(referenceVertex));
mapAttributes(referenceVertex, ret, entityExtInfo);
mapAttributes(referenceVertex, ret, entityExtInfo, isMinExtInfo);
}
return ret;
}
private Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException {
return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null) : null;
return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null;
}
private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException {
......
webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java
......@@ -18,18 +18,15 @@
package org.apache.atlas.web.rest;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.model.audit.EntityAuditEventV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.ClassificationAssociateRequest;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.repository.store.graph.v2.EntityStream;
......@@ -42,22 +39,12 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
......@@ -74,24 +61,17 @@ import java.util.Map;
@Singleton
@Service
public class EntityREST {
private static final Logger LOG = LoggerFactory.getLogger(EntityREST.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.EntityREST");
public static final String PREFIX_ATTR = "attr:";
private final AtlasTypeRegistry typeRegistry;
private final AtlasEntityStore entitiesStore;
private final EntityAuditRepository auditRepository;
private final AtlasInstanceConverter instanceConverter;
@Inject
public EntityREST(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore,
EntityAuditRepository auditRepository, AtlasInstanceConverter instanceConverter) {
public EntityREST(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore) {
this.typeRegistry = typeRegistry;
this.entitiesStore = entitiesStore;
this.auditRepository = auditRepository;
this.instanceConverter = instanceConverter;
}
/**
......@@ -104,17 +84,43 @@ public class EntityREST {
@Path("/guid/{guid}")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasEntityWithExtInfo getById(@PathParam("guid") String guid) throws AtlasBaseException {
public AtlasEntityWithExtInfo getById(@PathParam("guid") String guid, @QueryParam("minExtInfo") @DefaultValue("false") boolean minExtInfo) throws AtlasBaseException {
Servlets.validateQueryParamLength("guid", guid);
AtlasPerfTracer perf = null;
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getById(" + guid + ")");
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getById(" + guid + ", " + minExtInfo + " )");
}
return entitiesStore.getById(guid);
return entitiesStore.getById(guid, minExtInfo);
} finally {
AtlasPerfTracer.log(perf);
}
}
/**
* Get entity header given its GUID.
* @param guid GUID for the entity
* @return AtlasEntity
* @throws AtlasBaseException
*/
@GET
@Path("/guid/{guid}/header")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasEntityHeader getHeaderById(@PathParam("guid") String guid) throws AtlasBaseException {
Servlets.validateQueryParamLength("guid", guid);
AtlasPerfTracer perf = null;
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getHeaderById(" + guid + ")");
}
return entitiesStore.getHeaderById(guid);
} finally {
AtlasPerfTracer.log(perf);
}
......@@ -141,7 +147,7 @@ public class EntityREST {
@Path("/uniqueAttribute/type/{typeName}")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasEntityWithExtInfo getByUniqueAttributes(@PathParam("typeName") String typeName,
public AtlasEntityWithExtInfo getByUniqueAttributes(@PathParam("typeName") String typeName, @QueryParam("minExtInfo") @DefaultValue("false") boolean minExtInfo,
@Context HttpServletRequest servletRequest) throws AtlasBaseException {
Servlets.validateQueryParamLength("typeName", typeName);
......@@ -158,7 +164,7 @@ public class EntityREST {
validateUniqueAttribute(entityType, attributes);
return entitiesStore.getByUniqueAttributes(entityType, attributes);
return entitiesStore.getByUniqueAttributes(entityType, attributes, minExtInfo);
} finally {
AtlasPerfTracer.log(perf);
}
......@@ -585,7 +591,7 @@ public class EntityREST {
@Path("/bulk")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasEntitiesWithExtInfo getByGuids(@QueryParam("guid") List<String> guids) throws AtlasBaseException {
public AtlasEntitiesWithExtInfo getByGuids(@QueryParam("guid") List<String> guids, @QueryParam("minExtInfo") @DefaultValue("false") boolean minExtInfo) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(guids)) {
for (String guid : guids) {
Servlets.validateQueryParamLength("guid", guid);
......@@ -603,7 +609,7 @@ public class EntityREST {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guids);
}
return entitiesStore.getByIds(guids);
return entitiesStore.getByIds(guids, minExtInfo);
} finally {
AtlasPerfTracer.log(perf);
}
......@@ -693,38 +699,6 @@ public class EntityREST {
}
}
@GET
@Path("{guid}/audit")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public List<EntityAuditEventV2> getAuditEvents(@PathParam("guid") String guid, @QueryParam("startKey") String startKey,
@QueryParam("count") @DefaultValue("100") short count) throws AtlasBaseException {
AtlasPerfTracer perf = null;
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getAuditEvents(" + guid + ", " + startKey + ", " + count + ")");
}
List events = auditRepository.listEvents(guid, startKey, count);
List<EntityAuditEventV2> ret = new ArrayList<>();
for (Object event : events) {
if (event instanceof EntityAuditEventV2) {
ret.add((EntityAuditEventV2) event);
} else if (event instanceof EntityAuditEvent) {
ret.add(instanceConverter.toV2AuditEvent((EntityAuditEvent) event));
} else {
LOG.warn("unknown entity-audit event type {}. Ignored", event != null ? event.getClass().getCanonicalName() : "null");
}
}
return ret;
} finally {
AtlasPerfTracer.log(perf);
}
}
private AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException {
AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName);
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
......@@ -165,7 +165,7 @@ public class TestEntitiesREST {
@Test(dependsOnMethods = "testCreateOrUpdateEntities")
public void testGetEntities() throws Exception {
final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids);
final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids, false);
final List<AtlasEntity> entities = response.getEntities();
Assert.assertNotNull(entities);
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java
......@@ -99,7 +99,7 @@ public class TestEntityREST {
@Test
public void testGetEntityById() throws Exception {
createTestEntity();
AtlasEntityWithExtInfo response = entityREST.getById(dbEntity.getGuid());
AtlasEntityWithExtInfo response = entityREST.getById(dbEntity.getGuid(), false);
Assert.assertNotNull(response);
Assert.assertNotNull(response.getEntity());
......@@ -183,7 +183,7 @@ public class TestEntityREST {
@Test(dependsOnMethods = "testAddAndGetClassification")
public void testGetEntityWithAssociations() throws Exception {
AtlasEntityWithExtInfo entity = entityREST.getById(dbEntity.getGuid());
AtlasEntityWithExtInfo entity = entityREST.getById(dbEntity.getGuid(), false);
final List<AtlasClassification> retrievedClassifications = entity.getEntity().getClassifications();
Assert.assertNotNull(retrievedClassifications);
......@@ -315,7 +315,7 @@ public class TestEntityREST {
Assert.assertEquals(response.getEntitiesByOperation(EntityMutations.EntityOperation.PARTIAL_UPDATE).get(0).getGuid(), dbGuid);
//Get By unique attribute
AtlasEntityWithExtInfo entity = entityREST.getByUniqueAttributes(TestUtilsV2.DATABASE_TYPE, toHttpServletRequest(TestUtilsV2.NAME, updatedDBName));
AtlasEntityWithExtInfo entity = entityREST.getByUniqueAttributes(TestUtilsV2.DATABASE_TYPE, false, toHttpServletRequest(TestUtilsV2.NAME, updatedDBName));
Assert.assertNotNull(entity);
Assert.assertNotNull(entity.getEntity().getGuid());
Assert.assertEquals(entity.getEntity().getGuid(), dbGuid);
......@@ -340,7 +340,7 @@ public class TestEntityREST {
Assert.assertEquals(response.getEntitiesByOperation(EntityMutations.EntityOperation.PARTIAL_UPDATE).get(0).getGuid(), dbGuid);
//Get By unique attribute
AtlasEntityWithExtInfo entity = entityREST.getByUniqueAttributes(TestUtilsV2.DATABASE_TYPE, toHttpServletRequest(TestUtilsV2.NAME, updatedDBName));
AtlasEntityWithExtInfo entity = entityREST.getByUniqueAttributes(TestUtilsV2.DATABASE_TYPE, false, toHttpServletRequest(TestUtilsV2.NAME, updatedDBName));
Assert.assertNotNull(entity);
Assert.assertNotNull(entity.getEntity().getGuid());
Assert.assertEquals(entity.getEntity().getGuid(), dbGuid);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment