Skip to content

A
atlas
Commit 1e60ca4d by Ashutosh Mestry
Options

ATLAS-2624: (2): Fixed the case where same classifications are associated with multiple entities.

Signed-off-by: 's avatarAshutosh Mestry <amestry@hortonworks.com>
parent e311bdc2
Showing with 126 additions and 36 deletions
intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java
...@@ -83,6 +83,9 @@ public class AtlasClassification extends AtlasStruct implements Serializable { ...@@ -83,6 +83,9 @@ public class AtlasClassification extends AtlasStruct implements Serializable {
if (other != null) { if (other != null) {
setTypeName(other.getTypeName()); setTypeName(other.getTypeName());
setAttributes(other.getAttributes()); setAttributes(other.getAttributes());
setEntityGuid(other.getEntityGuid());
setPropagate(other.isPropagate());
setValidityPeriods(other.getValidityPeriods());
} }
} }
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
...@@ -1353,9 +1353,10 @@ public class EntityGraphMapper { ...@@ -1353,9 +1353,10 @@ public class EntityGraphMapper {
List<AtlasVertex> entitiesToPropagateTo = null; List<AtlasVertex> entitiesToPropagateTo = null;
Map<AtlasVertex, List<AtlasClassification>> propagations = null; Map<AtlasVertex, List<AtlasClassification>> propagations = null;
for (AtlasClassification classification : classifications) { for (AtlasClassification c : classifications) {
String classificationName = classification.getTypeName(); AtlasClassification classification = new AtlasClassification(c);
Boolean propagateTags = classification.isPropagate(); String classificationName = classification.getTypeName();
Boolean propagateTags = classification.isPropagate();
if (propagateTags != null && propagateTags && if (propagateTags != null && propagateTags &&
classification.getEntityGuid() != null && classification.getEntityGuid() != null &&
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
...@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; ...@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.collections.Transformer;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -65,12 +66,7 @@ import org.testng.annotations.Guice; ...@@ -65,12 +66,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import javax.inject.Inject; import javax.inject.Inject;
import java.util.ArrayList; import java.util.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtilsV2.COLUMNS_ATTR_NAME; import static org.apache.atlas.TestUtilsV2.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE; import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE;
...@@ -110,6 +106,8 @@ public class AtlasEntityStoreV1Test { ...@@ -110,6 +106,8 @@ public class AtlasEntityStoreV1Test {
@Inject @Inject
private Configuration configuration; private Configuration configuration;
private String dbEntityGuid;
private String tblEntityGuid;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
...@@ -220,6 +218,7 @@ public class AtlasEntityStoreV1Test { ...@@ -220,6 +218,7 @@ public class AtlasEntityStoreV1Test {
init(); init();
EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false);
validateMutationResponse(dbCreationResponse, EntityOperation.CREATE, 1); validateMutationResponse(dbCreationResponse, EntityOperation.CREATE, 1);
dbEntityGuid = dbCreationResponse.getCreatedEntities().get(0).getGuid();
AtlasEntityHeader db1 = dbCreationResponse.getFirstCreatedEntityByTypeName(TestUtilsV2.DATABASE_TYPE); AtlasEntityHeader db1 = dbCreationResponse.getFirstCreatedEntityByTypeName(TestUtilsV2.DATABASE_TYPE);
validateEntity(dbEntity, getEntityFromStore(db1)); validateEntity(dbEntity, getEntityFromStore(db1));
...@@ -233,6 +232,7 @@ public class AtlasEntityStoreV1Test { ...@@ -233,6 +232,7 @@ public class AtlasEntityStoreV1Test {
init(); init();
EntityMutationResponse tableCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false); EntityMutationResponse tableCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false);
validateMutationResponse(tableCreationResponse, EntityOperation.CREATE, 1); validateMutationResponse(tableCreationResponse, EntityOperation.CREATE, 1);
tblEntityGuid = tableCreationResponse.getCreatedEntities().get(0).getGuid();
AtlasEntityHeader tableEntity = tableCreationResponse.getFirstCreatedEntityByTypeName(TABLE_TYPE); AtlasEntityHeader tableEntity = tableCreationResponse.getFirstCreatedEntityByTypeName(TABLE_TYPE);
validateEntity(tblEntity, getEntityFromStore(tableEntity)); validateEntity(tblEntity, getEntityFromStore(tableEntity));
...@@ -909,15 +909,8 @@ public class AtlasEntityStoreV1Test { ...@@ -909,15 +909,8 @@ public class AtlasEntityStoreV1Test {
@Test @Test
public void testTagAssociationAfterRedefinition(){ public void testTagAssociationAfterRedefinition(){
AtlasClassificationDef aTag = new AtlasClassificationDef("testTag");
AtlasAttributeDef attributeDef = new AtlasAttributeDef("testAttribute", "int", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, true,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList());
aTag.addAttribute(attributeDef);
AtlasTypesDef typesDef = new AtlasTypesDef(); AtlasTypesDef typesDef = new AtlasTypesDef();
typesDef.setClassificationDefs(Arrays.asList(aTag)); getTagWithName(typesDef,"testTag","int");
try { try {
typeDefStore.createTypesDef(typesDef); typeDefStore.createTypesDef(typesDef);
...@@ -931,13 +924,7 @@ public class AtlasEntityStoreV1Test { ...@@ -931,13 +924,7 @@ public class AtlasEntityStoreV1Test {
fail("Tag deletion should've succeeded"); fail("Tag deletion should've succeeded");
} }
aTag = new AtlasClassificationDef("testTag"); AtlasClassificationDef aTag = getTagWithName(typesDef, "testTag", "string");
attributeDef = new AtlasAttributeDef("testAttribute", "string", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, true,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList());
aTag.addAttribute(attributeDef);
typesDef.setClassificationDefs(Arrays.asList(aTag));
try { try {
typeDefStore.createTypesDef(typesDef); typeDefStore.createTypesDef(typesDef);
...@@ -956,7 +943,86 @@ public class AtlasEntityStoreV1Test { ...@@ -956,7 +943,86 @@ public class AtlasEntityStoreV1Test {
} catch (AtlasBaseException e) { } catch (AtlasBaseException e) {
fail("DB entity creation should've succeeded, e.getMessage() => " + e.getMessage()); fail("DB entity creation should've succeeded, e.getMessage() => " + e.getMessage());
} }
}
@Test(dependsOnMethods = "testCreate")
public void associateMultipleTagsToOneEntity() throws AtlasBaseException {
final String TAG_NAME = "tag_xy";
final String TAG_NAME_2 = TAG_NAME + "_2";
final String TAG_ATTRIBUTE_NAME = "testAttribute";
final String TAG_ATTRIBUTE_VALUE = "test-string";
final String TAG_ATTRIBUTE_VALUE_2 = TAG_ATTRIBUTE_VALUE + "-2";
createTag(TAG_NAME, "string");
createTag(TAG_NAME_2, "string");
List<AtlasClassification> addedClassifications = new ArrayList<>();
addedClassifications.add(new AtlasClassification(TAG_NAME, TAG_ATTRIBUTE_NAME, TAG_ATTRIBUTE_VALUE));
addedClassifications.add(new AtlasClassification(TAG_NAME_2, TAG_ATTRIBUTE_NAME, TAG_ATTRIBUTE_VALUE_2));
entityStore.addClassifications(dbEntityGuid, addedClassifications);
AtlasEntity dbEntityFromDb = getEntityFromStore(dbEntityGuid);
List<String> actualClassifications = (List<String>) CollectionUtils.collect(dbEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getTypeName());
Set<String> classificationSet = new HashSet<>(actualClassifications);
assertTrue(classificationSet.contains(TAG_NAME));
assertTrue(classificationSet.contains(TAG_NAME_2));
entityStore.deleteClassifications(dbEntityGuid, actualClassifications);
}
@Test(dependsOnMethods = "testCreate")
public void associateSameTagToMultipleEntities() throws AtlasBaseException {
final String TAG_NAME = "tagx";
final String TAG_ATTRIBUTE_NAME = "testAttribute";
final String TAG_ATTRIBUTE_VALUE = "test-string";
createTag(TAG_NAME, "string");
List<AtlasClassification> addedClassifications = new ArrayList<>();
addedClassifications.add(new AtlasClassification(TAG_NAME, TAG_ATTRIBUTE_NAME, TAG_ATTRIBUTE_VALUE));
entityStore.addClassifications(dbEntityGuid, addedClassifications);
entityStore.addClassifications(tblEntityGuid, addedClassifications);
AtlasEntity dbEntityFromDb = getEntityFromStore(dbEntityGuid);
AtlasEntity tblEntityFromDb = getEntityFromStore(tblEntityGuid);
Set<String> actualDBClassifications = new HashSet<>(CollectionUtils.collect(dbEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getTypeName()));
Set<String> actualTblClassifications = new HashSet<>(CollectionUtils.collect(tblEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getTypeName()));
assertTrue(actualDBClassifications.contains(TAG_NAME));
assertTrue(actualTblClassifications.contains(TAG_NAME));
Set<String> actualDBAssociatedEntityGuid = new HashSet<>(CollectionUtils.collect(dbEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getEntityGuid()));
Set<String> actualTblAssociatedEntityGuid = new HashSet<>(CollectionUtils.collect(tblEntityFromDb.getClassifications(), o -> ((AtlasClassification) o).getEntityGuid()));
assertTrue(actualDBAssociatedEntityGuid.contains(dbEntityGuid));
assertTrue(actualTblAssociatedEntityGuid.contains(tblEntityGuid));
entityStore.deleteClassifications(dbEntityGuid, Collections.singletonList(TAG_NAME));
entityStore.deleteClassifications(tblEntityGuid, Collections.singletonList(TAG_NAME));
}
private AtlasClassificationDef getTagWithName(AtlasTypesDef typesDef, String tagName, String attributeType) {
AtlasClassificationDef aTag = new AtlasClassificationDef(tagName);
AtlasAttributeDef attributeDef = new AtlasAttributeDef("testAttribute", attributeType, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, true,
Collections.emptyList());
aTag.addAttribute(attributeDef);
typesDef.setClassificationDefs(Arrays.asList(aTag));
return aTag;
}
private void createTag(String tagName, String attributeType) {
try {
AtlasTypesDef typesDef = new AtlasTypesDef();
getTagWithName(typesDef, tagName, attributeType);
typeDefStore.createTypesDef(typesDef);
} catch (AtlasBaseException e) {
fail("Tag creation should've succeeded");
}
} }
private String randomStrWithReservedChars() { private String randomStrWithReservedChars() {
......
repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
...@@ -345,21 +345,24 @@ public class ClassificationPropagationTest { ...@@ -345,21 +345,24 @@ public class ClassificationPropagationTest {
addClassification(employees2, PII_tag3); addClassification(employees2, PII_tag3);
// check 4 PII tags exists in employee_union table // check 4 PII tags exists in employee_union table
assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag1); assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag1.getTypeName(), hdfs_path.getGuid());
assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag2); assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag2.getTypeName(), employees1.getGuid());
assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag3); assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag3.getTypeName(), employees2.getGuid());
AtlasRelationship process3_employee_union_relationship = getRelationship(EMPLOYEES_UNION_PROCESS, EMPLOYEES_UNION_TABLE); AtlasRelationship process3_employee_union_relationship = getRelationship(EMPLOYEES_UNION_PROCESS, EMPLOYEES_UNION_TABLE);
List<AtlasClassification> propagatedClassifications = process3_employee_union_relationship.getPropagatedClassifications(); List<AtlasClassification> propagatedClassifications = process3_employee_union_relationship.getPropagatedClassifications();
List<AtlasClassification> blockedClassifications = process3_employee_union_relationship.getBlockedPropagatedClassifications(); List<AtlasClassification> blockedClassifications = process3_employee_union_relationship.getBlockedPropagatedClassifications();
assertNotNull(propagatedClassifications); assertNotNull(propagatedClassifications);
assertTrue(propagatedClassifications.contains(PII_tag1)); assertClassificationEquals(propagatedClassifications, PII_tag1);
assertTrue(propagatedClassifications.contains(PII_tag2)); assertClassificationEquals(propagatedClassifications, PII_tag2);
assertTrue(propagatedClassifications.contains(PII_tag3)); assertClassificationEquals(propagatedClassifications, PII_tag3);
assertTrue(blockedClassifications.isEmpty()); assertTrue(blockedClassifications.isEmpty());
// block PII tag propagating from employees1 and employees2 // block PII tag propagating from employees1 and employees2
PII_tag2.setEntityGuid(employees1.getGuid());
PII_tag3.setEntityGuid(employees2.getGuid());
process3_employee_union_relationship.setBlockedPropagatedClassifications(Arrays.asList(PII_tag2, PII_tag3)); process3_employee_union_relationship.setBlockedPropagatedClassifications(Arrays.asList(PII_tag2, PII_tag3));
relationshipStore.update(process3_employee_union_relationship); relationshipStore.update(process3_employee_union_relationship);
...@@ -367,17 +370,30 @@ public class ClassificationPropagationTest { ...@@ -367,17 +370,30 @@ public class ClassificationPropagationTest {
propagatedClassifications = process3_employee_union_relationship.getPropagatedClassifications(); propagatedClassifications = process3_employee_union_relationship.getPropagatedClassifications();
blockedClassifications = process3_employee_union_relationship.getBlockedPropagatedClassifications(); blockedClassifications = process3_employee_union_relationship.getBlockedPropagatedClassifications();
assertTrue(propagatedClassifications.contains(PII_tag1)); assertClassificationEquals(propagatedClassifications, PII_tag1);
assertTrue(!blockedClassifications.isEmpty()); assertTrue(!blockedClassifications.isEmpty());
assertTrue(blockedClassifications.contains(PII_tag2)); assertClassificationEquals(blockedClassifications, PII_tag2);
assertTrue(blockedClassifications.contains(PII_tag3)); assertClassificationEquals(blockedClassifications, PII_tag3);
assertClassificationNotExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag2); assertClassificationNotExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag2);
assertClassificationNotExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag3); assertClassificationNotExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag3);
// assert only PII from hdfs_path is propagated to employees_union, PII from employees1 and employees2 is blocked. // assert only PII from hdfs_path is propagated to employees_union, PII from employees1 and employees2 is blocked.
assertEquals(getEntity(EMPLOYEES_UNION_TABLE).getClassifications().size(), 1); assertEquals(getEntity(EMPLOYEES_UNION_TABLE).getClassifications().size(), 1);
assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag1); assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, PII_tag1.getTypeName(), hdfs_path.getGuid());
}
private void assertClassificationEquals(List<AtlasClassification> propagatedClassifications, AtlasClassification expected) {
String expectedTypeName = expected.getTypeName();
for (AtlasClassification c : propagatedClassifications) {
if(c.getTypeName().equals(expectedTypeName)) {
assertTrue(c.isPropagate() == expected.isPropagate(), "isPropgate does not match");
assertTrue(c.getValidityPeriods() == expected.getValidityPeriods(), "validityPeriods do not match");
return;
}
}
fail(expectedTypeName + " could not be found");
} }
@Test(dependsOnMethods = {"addBlockedPropagatedClassifications"}) @Test(dependsOnMethods = {"addBlockedPropagatedClassifications"})
...@@ -449,9 +465,13 @@ public class ClassificationPropagationTest { ...@@ -449,9 +465,13 @@ public class ClassificationPropagationTest {
} }
private void assertClassificationExistInEntity(String entityName, AtlasClassification classification) throws AtlasBaseException { private void assertClassificationExistInEntity(String entityName, AtlasClassification classification) throws AtlasBaseException {
assertClassificationExistInEntity(entityName, classification.getTypeName(), classification.getEntityGuid());
}
private void assertClassificationExistInEntity(String entityName, String tagName, String sourceEntityGuid) throws AtlasBaseException {
List<AtlasClassification> classifications = getEntity(entityName).getClassifications(); List<AtlasClassification> classifications = getEntity(entityName).getClassifications();
String classificationName = classification.getTypeName(); String classificationName = tagName;
String entityGuid = classification.getEntityGuid(); String entityGuid = sourceEntityGuid;
if (CollectionUtils.isNotEmpty(classifications)) { if (CollectionUtils.isNotEmpty(classifications)) {
boolean foundClassification = false; boolean foundClassification = false;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment