Skip to content

A
atlas
Commit b84ed999 by Madhan Neethiraj
Options

ATLAS-3037: fixed entity-updated detection to handle object-id attributes correctly

parent 05e845f5
Showing with 167 additions and 78 deletions
authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizationUtils.java
......@@ -22,6 +22,7 @@ package org.apache.atlas.authorize;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -87,6 +88,8 @@ public class AtlasAuthorizationUtils {
}
public static boolean isAccessAllowed(AtlasAdminAccessRequest request) {
MetricRecorder metric = RequestContext.get().startMetricRecord("isAccessAllowed");
boolean ret = false;
String userName = getCurrentUserName();
......@@ -104,10 +107,14 @@ public class AtlasAuthorizationUtils {
ret = true;
}
RequestContext.get().endMetricRecord(metric);
return ret;
}
public static boolean isAccessAllowed(AtlasEntityAccessRequest request) {
MetricRecorder metric = RequestContext.get().startMetricRecord("isAccessAllowed");
boolean ret = false;
String userName = getCurrentUserName();
......@@ -125,10 +132,14 @@ public class AtlasAuthorizationUtils {
ret = true;
}
RequestContext.get().endMetricRecord(metric);
return ret;
}
public static boolean isAccessAllowed(AtlasTypeAccessRequest request) {
MetricRecorder metric = RequestContext.get().startMetricRecord("isAccessAllowed");
boolean ret = false;
String userName = getCurrentUserName();
......@@ -146,10 +157,14 @@ public class AtlasAuthorizationUtils {
ret = true;
}
RequestContext.get().endMetricRecord(metric);
return ret;
}
public static boolean isAccessAllowed(AtlasRelationshipAccessRequest request) {
MetricRecorder metric = RequestContext.get().startMetricRecord("isAccessAllowed");
boolean ret = false;
String userName = getCurrentUserName();
......@@ -167,6 +182,8 @@ public class AtlasAuthorizationUtils {
ret = true;
}
RequestContext.get().endMetricRecord(metric);
return ret;
}
......
intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java
......@@ -28,6 +28,7 @@ import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.SERVICE_TYPE_ATLAS_CORE;
......@@ -149,7 +150,7 @@ public class AtlasArrayType extends AtlasType {
}
@Override
public boolean areEqualValues(Object val1, Object val2) {
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
boolean ret = true;
if (val1 == null) {
......@@ -164,7 +165,7 @@ public class AtlasArrayType extends AtlasType {
ret = false;
} else {
for (int i = 0; i < len; i++) {
if (!elementType.areEqualValues(Array.get(val1, i), Array.get(val2, i))) {
if (!elementType.areEqualValues(Array.get(val1, i), Array.get(val2, i), guidAssignments)) {
ret = false;
break;
......@@ -182,7 +183,7 @@ public class AtlasArrayType extends AtlasType {
boolean foundInSet2 = false;
for (Object elem2 : set2) {
if (elementType.areEqualValues(elem1, elem2)) {
if (elementType.areEqualValues(elem1, elem2, guidAssignments)) {
foundInSet2 = true;
break;
......@@ -213,7 +214,7 @@ public class AtlasArrayType extends AtlasType {
ret = false;
} else {
for (int i = 0; i < len; i++) {
if (!elementType.areEqualValues(list1.get(i), list2.get(i))) {
if (!elementType.areEqualValues(list1.get(i), list2.get(i), guidAssignments)) {
ret = false;
break;
......
intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
......@@ -29,6 +29,7 @@ import java.math.BigInteger;
import java.text.ParseException;
import java.util.Date;
import java.util.Map;
import java.util.Objects;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.SERVICE_TYPE_ATLAS_CORE;
......@@ -277,7 +278,8 @@ public class AtlasBuiltInTypes {
return getNormalizedValue(obj) != null;
}
public boolean areEqualValues(Object val1, Object val2) {
@Override
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
final boolean ret;
if (val1 == null) {
......@@ -353,7 +355,8 @@ public class AtlasBuiltInTypes {
return getNormalizedValue(obj) != null;
}
public boolean areEqualValues(Object val1, Object val2) {
@Override
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
final boolean ret;
if (val1 == null) {
......@@ -626,6 +629,48 @@ public class AtlasBuiltInTypes {
}
@Override
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
boolean ret = true;
if (val1 == null) {
ret = val2 == null;
} else if (val2 == null) {
ret = false;
} else {
AtlasObjectId v1 = getNormalizedValue(val1);
AtlasObjectId v2 = getNormalizedValue(val2);
if (v1 == null || v2 == null) {
ret = false;
} else {
String guid1 = v1.getGuid();
String guid2 = v2.getGuid();
if (guidAssignments != null ) {
if (guidAssignments.containsKey(guid1)) {
guid1 = guidAssignments.get(guid1);
}
if (guidAssignments.containsKey(guid2)) {
guid2 = guidAssignments.get(guid2);
}
}
boolean isV1AssignedGuid = AtlasTypeUtil.isAssignedGuid(guid1);
boolean isV2AssignedGuid = AtlasTypeUtil.isAssignedGuid(guid2);
if (isV1AssignedGuid == isV2AssignedGuid) { // if both have assigned/unassigned guids, compare guids
ret = Objects.equals(guid1, guid2);
} else { // if one has assigned and other unassigned guid, compare typeName and unique-attribute
ret = Objects.equals(v1.getTypeName(), v2.getTypeName()) && Objects.equals(v1.getUniqueAttributes(), v2.getUniqueAttributes());
}
}
}
return ret;
}
@Override
public AtlasObjectId getNormalizedValue(Object obj) {
if (obj != null) {
if (obj instanceof AtlasObjectId) {
......
intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
......@@ -315,14 +315,14 @@ public class AtlasClassificationType extends AtlasStructType {
}
@Override
public boolean areEqualValues(Object val1, Object val2) {
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
for (AtlasClassificationType superType : superTypes) {
if (!superType.areEqualValues(val1, val2)) {
if (!superType.areEqualValues(val1, val2, guidAssignments)) {
return false;
}
}
return super.areEqualValues(val1, val2);
return super.areEqualValues(val1, val2, guidAssignments);
}
@Override
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -362,14 +362,14 @@ public class AtlasEntityType extends AtlasStructType {
}
@Override
public boolean areEqualValues(Object val1, Object val2) {
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
for (AtlasEntityType superType : superTypes) {
if (!superType.areEqualValues(val1, val2)) {
if (!superType.areEqualValues(val1, val2, guidAssignments)) {
return false;
}
}
return super.areEqualValues(val1, val2);
return super.areEqualValues(val1, val2, guidAssignments);
}
@Override
......
intg/src/main/java/org/apache/atlas/type/AtlasMapType.java
......@@ -119,7 +119,7 @@ public class AtlasMapType extends AtlasType {
}
@Override
public boolean areEqualValues(Object val1, Object val2) {
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
boolean ret = true;
if (val1 == null) {
......@@ -143,7 +143,7 @@ public class AtlasMapType extends AtlasType {
ret = false;
} else {
for (Object key : map1.keySet()) {
if (!valueType.areEqualValues(map1.get(key), map2.get(key))) {
if (!valueType.areEqualValues(map1.get(key), map2.get(key), guidAssignments)) {
ret = false;
break;
......
intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
......@@ -172,7 +172,7 @@ public class AtlasRelationshipType extends AtlasStructType {
}
@Override
public boolean areEqualValues(Object val1, Object val2) {
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
final boolean ret;
if (val1 == null) {
......@@ -189,7 +189,7 @@ public class AtlasRelationshipType extends AtlasStructType {
if (rel2 == null) {
ret = false;
} else if (!super.areEqualValues(rel1, rel2)) {
} else if (!super.areEqualValues(rel1, rel2, guidAssignments)) {
ret = false;
} else {
ret = Objects.equals(rel1.getGuid(), rel2.getGuid()) &&
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -244,7 +244,7 @@ public class AtlasStructType extends AtlasType {
}
@Override
public boolean areEqualValues(Object val1, Object val2) {
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
boolean ret = true;
if (val1 == null) {
......@@ -264,21 +264,14 @@ public class AtlasStructType extends AtlasType {
} else if (!StringUtils.equalsIgnoreCase(structVal1.getTypeName(), structVal2.getTypeName())) {
ret = false;
} else {
for (Map.Entry<String, Object> entry : structVal1.getAttributes().entrySet()) {
String attrName = entry.getKey();
AtlasAttribute attribute = getAttribute(attrName);
for (AtlasAttribute attribute : getAllAttributes().values()) {
Object attrValue1 = structVal1.getAttribute(attribute.getName());
Object attrValue2 = structVal2.getAttribute(attribute.getName());
if (attribute == null) { // ignore unknown attribute
continue;
} else {
Object attrValue1 = entry.getValue();
Object attrValue2 = structVal2.getAttribute(attrName);
if (!attribute.getAttributeType().areEqualValues(attrValue1, attrValue2, guidAssignments)) {
ret = false;
if (!attribute.getAttributeType().areEqualValues(attrValue1, attrValue2)) {
ret = false;
break;
}
break;
}
}
}
......
intg/src/main/java/org/apache/atlas/type/AtlasType.java
......@@ -27,6 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.Objects;
......@@ -79,7 +80,7 @@ public abstract class AtlasType {
public abstract boolean isValidValue(Object obj);
public boolean areEqualValues(Object val1, Object val2) {
public boolean areEqualValues(Object val1, Object val2, Map<String, String> guidAssignments) {
final boolean ret;
if (val1 == null) {
......
intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
......@@ -43,38 +43,6 @@ public class AtlasEntityUtil {
private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1;
public static boolean hasAnyAttributeUpdate(AtlasEntityType entityType, AtlasEntity currEntity, AtlasEntity newEntity) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> hasAnyAttributeUpdate(guid={}, typeName={})", currEntity.getGuid(), currEntity.getTypeName());
}
boolean ret = false;
for (AtlasAttribute attribute : entityType.getAllAttributes().values()) {
String attrName = attribute.getName();
AtlasType attrType = attribute.getAttributeType();
Object currValue = currEntity.getAttribute(attrName);
Object newValue = newEntity.getAttribute(attrName);
if (!attrType.areEqualValues(currEntity.getAttribute(attrName), newEntity.getAttribute(attrName))) {
ret = true;
if (LOG.isDebugEnabled()) {
LOG.debug("hasAnyAttributeUpdate(guid={}, typeName={}): attribute '{}' is found updated - currentValue={}, newValue={}",
currEntity.getGuid(), currEntity.getTypeName(), attrName, currValue, newValue);
}
break;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== hasAnyAttributeUpdate(guid={}, typeName={}): ret={}", currEntity.getGuid(), currEntity.getTypeName(), ret);
}
return ret;
}
public static String formatSoftRefValue(String typeName, String guid) {
return String.format(SOFT_REF_FORMAT, typeName, guid);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
......@@ -40,7 +40,6 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
......@@ -694,24 +693,68 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
// for existing entities, skip update if incoming entity doesn't have any change
if (CollectionUtils.isNotEmpty(context.getUpdatedEntities())) {
MetricRecorder checkForUnchangedEntities = RequestContext.get().startMetricRecord("checkForUnchangedEntities");
List<AtlasEntity> entitiesToSkipUpdate = null;
for (AtlasEntity entity : context.getUpdatedEntities()) {
String guid = entity.getGuid();
AtlasVertex vertex = context.getVertex(guid);
AtlasEntity entityInStore = entityRetriever.toAtlasEntity(vertex);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (!AtlasEntityUtil.hasAnyAttributeUpdate(entityType, entity, entityInStore)) {
// if classifications are to be replaced as well, then skip updates only when no change in classifications as well
if (!replaceClassifications || Objects.equals(entity.getClassifications(), entityInStore.getClassifications())) {
if (entitiesToSkipUpdate == null) {
entitiesToSkipUpdate = new ArrayList<>();
String guid = entity.getGuid();
AtlasVertex vertex = context.getVertex(guid);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
boolean hasUpdates = false;
if (MapUtils.isNotEmpty(entity.getRelationshipAttributes())) {
hasUpdates = true; // if relationship attributes are provided, assume there is an update
}
if (!hasUpdates) {
hasUpdates = entity.getStatus() == AtlasEntity.Status.DELETED; // entity status could be updated during import
}
if (!hasUpdates) {
for (AtlasAttribute attribute : entityType.getAllAttributes().values()) {
if (!entity.getAttributes().containsKey(attribute.getName())) { // if value is not provided, current value will not be updated
continue;
}
entitiesToSkipUpdate.add(entity);
Object newVal = entity.getAttribute(attribute.getName());
Object currVal = entityRetriever.getEntityAttribute(vertex, attribute);
if (!attribute.getAttributeType().areEqualValues(currVal, newVal, context.getGuidAssignments())) {
hasUpdates = true;
if (LOG.isDebugEnabled()) {
LOG.debug("found attribute update: entity(guid={}, typeName={}), attrName={}, currValue={}, newValue={}", guid, entity.getTypeName(), attribute.getName(), currVal, newVal);
}
break;
}
}
}
// if classifications are to be replaced, then skip updates only when no change in classifications
if (!hasUpdates && replaceClassifications) {
List<AtlasClassification> newVal = entity.getClassifications();
List<AtlasClassification> currVal = entityRetriever.getAllClassifications(vertex);
if (!Objects.equals(currVal, newVal)) {
hasUpdates = true;
if (LOG.isDebugEnabled()) {
LOG.debug("found classifications update: entity(guid={}, typeName={}), currValue={}, newValue={}", guid, entity.getTypeName(), currVal, newVal);
}
}
}
if (!hasUpdates) {
if (entitiesToSkipUpdate == null) {
entitiesToSkipUpdate = new ArrayList<>();
}
LOG.info("skipping unchanged entity: {}", entity);
entitiesToSkipUpdate.add(entity);
}
}
if (entitiesToSkipUpdate != null) {
......@@ -725,6 +768,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
"update entity: type=", entity.getTypeName());
}
}
RequestContext.get().endMetricRecord(checkForUnchangedEntities);
}
EntityMutationResponse ret = entityGraphMapper.mapAttributesAndClassifications(context, isPartialUpdate, replaceClassifications);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
......@@ -197,6 +197,18 @@ public class EntityGraphRetriever {
return ret;
}
public Object getEntityAttribute(AtlasVertex entityVertex, AtlasAttribute attribute) {
Object ret = null;
try {
ret = getVertexAttribute(entityVertex, attribute);
} catch (AtlasBaseException excp) {
// ignore
}
return ret;
}
public AtlasEntityHeader toAtlasEntityHeader(AtlasEntity entity) {
AtlasEntityHeader ret = null;
String typeName = entity.getTypeName();
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
......@@ -388,6 +388,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
//Drop the first key and change the class type as well to col0
columnsMap.clear();
columnsMap.put("col0", AtlasTypeUtil.getAtlasObjectId(col0));
tableEntity.setAttribute(TestUtilsV2.COLUMNS_MAP, columnsMap);
init();
response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
......@@ -420,6 +421,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
//Remove an entry
paramsMap.remove("key1");
tableEntity.setAttribute("parametersMap", paramsMap);
init();
response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 1);
......@@ -446,21 +448,24 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
//add a new element to array of struct
partitions.add(new AtlasStruct(TestUtilsV2.PARTITION_STRUCT_TYPE, TestUtilsV2.NAME, "part3"));
tableEntity.setAttribute("partitions", partitions);
init();
response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
//remove one of the struct values
init();
partitions.remove(1);
tableEntity.setAttribute("partitions", partitions);
init();
response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
//Update struct value within array of struct
init();
partitions.get(0).setAttribute(TestUtilsV2.NAME, "part4");
tableEntity.setAttribute("partitions", partitions);
init();
response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
......@@ -468,6 +473,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
//add a repeated element to array of struct
partitions.add(new AtlasStruct(TestUtilsV2.PARTITION_STRUCT_TYPE, TestUtilsV2.NAME, "part4"));
tableEntity.setAttribute("partitions", partitions);
init();
response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
......@@ -475,6 +481,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
// Remove all elements. Should set array attribute to null
partitions.clear();
tableEntity.setAttribute("partitions", partitions);
init();
response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
......
webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
......@@ -51,7 +51,7 @@ public class AdminExportImportTestIT extends BaseResourceIT {
static final String IMPORT_TRANSFORM_CLEAR_ATTRS =
"{ \"Asset\": { \"*\":[ \"clearAttrValue:replicatedTo,replicatedFrom\" ] } }";
static final String IMPORT_TRANSFORM_SET_DELETED =
"{ \"Asset\": { \"*\":[ \"setDeleted\" ] } }";
"{ \"Referenceable\": { \"*\":[ \"setDeleted\" ] } }";
@Test
public void isActive() throws AtlasServiceException {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment