Commit b7dcfc60 by Ashutosh Mestry

ATLAS-3048: Remove use of hard coded qualifiedName.

parent d561404b
...@@ -65,7 +65,10 @@ public class ClassificationAssociator { ...@@ -65,7 +65,10 @@ public class ClassificationAssociator {
Map<String, AtlasEntityHeader> guidEntityHeaderMap = new HashMap<>(); Map<String, AtlasEntityHeader> guidEntityHeaderMap = new HashMap<>();
for (String guid : guids) { for (String guid : guids) {
AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(guid); AtlasEntityHeader entityHeader = getEntityHeaderByGuid(guid);
if (entityHeader == null) {
continue;
}
guidEntityHeaderMap.put(guid, entityHeader); guidEntityHeaderMap.put(guid, entityHeader);
} }
...@@ -74,6 +77,16 @@ public class ClassificationAssociator { ...@@ -74,6 +77,16 @@ public class ClassificationAssociator {
return new AtlasEntityHeaders(guidEntityHeaderMap); return new AtlasEntityHeaders(guidEntityHeaderMap);
} }
private AtlasEntityHeader getEntityHeaderByGuid(String guid) {
try {
return entityRetriever.toAtlasEntityHeaderWithClassifications(guid);
} catch (AtlasBaseException e) {
LOG.error("Error fetching entity: {}", guid, e);
}
return null;
}
private long incrementTimestamp(long t) { private long incrementTimestamp(long t) {
return t + 1; return t + 1;
} }
...@@ -95,6 +108,7 @@ public class ClassificationAssociator { ...@@ -95,6 +108,7 @@ public class ClassificationAssociator {
private final AtlasEntityStore entitiesStore; private final AtlasEntityStore entitiesStore;
private final EntityGraphRetriever entityRetriever; private final EntityGraphRetriever entityRetriever;
private StringBuilder actionSummary = new StringBuilder(); private StringBuilder actionSummary = new StringBuilder();
private Map<String, String> typeNameUniqueAttributeNameMap = new HashMap<>();
public Updater(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore) { public Updater(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore) {
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
...@@ -105,6 +119,7 @@ public class ClassificationAssociator { ...@@ -105,6 +119,7 @@ public class ClassificationAssociator {
public String setClassifications(Map<String, AtlasEntityHeader> map) { public String setClassifications(Map<String, AtlasEntityHeader> map) {
for (AtlasEntityHeader incomingEntityHeader : map.values()) { for (AtlasEntityHeader incomingEntityHeader : map.values()) {
String typeName = incomingEntityHeader.getTypeName(); String typeName = incomingEntityHeader.getTypeName();
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType == null) { if (entityType == null) {
LOG.warn("Entity type: {}: Not found: {}!", typeName, STATUS_SKIPPED); LOG.warn("Entity type: {}: Not found: {}!", typeName, STATUS_SKIPPED);
...@@ -112,7 +127,11 @@ public class ClassificationAssociator { ...@@ -112,7 +127,11 @@ public class ClassificationAssociator {
continue; continue;
} }
String qualifiedName = getQualifiedName(incomingEntityHeader); String qualifiedName = getUniqueAttributeName(entityType, incomingEntityHeader);
if (StringUtils.isEmpty(qualifiedName)) {
qualifiedName = "<no unique name>";
}
AtlasEntityHeader entityToBeChanged = getByUniqueAttributes(entityType, qualifiedName, incomingEntityHeader.getAttributes()); AtlasEntityHeader entityToBeChanged = getByUniqueAttributes(entityType, qualifiedName, incomingEntityHeader.getAttributes());
if (entityToBeChanged == null) { if (entityToBeChanged == null) {
summarizeFormat("Entity:%s:%s:[Not found]:%s", entityType.getTypeName(), qualifiedName, STATUS_SKIPPED); summarizeFormat("Entity:%s:%s:[Not found]:%s", entityType.getTypeName(), qualifiedName, STATUS_SKIPPED);
...@@ -246,8 +265,21 @@ public class ClassificationAssociator { ...@@ -246,8 +265,21 @@ public class ClassificationAssociator {
return list.stream().map(AtlasClassification::getTypeName).collect(Collectors.joining(", ")); return list.stream().map(AtlasClassification::getTypeName).collect(Collectors.joining(", "));
} }
private String getQualifiedName(AtlasEntityHeader entityHeader) { private String getUniqueAttributeName(AtlasEntityType entityType, AtlasEntityHeader entityHeader) {
return (String) entityHeader.getAttribute(ATTR_NAME_QUALIFIED_NAME); String uniqueAttrName = ATTR_NAME_QUALIFIED_NAME;
if (!entityHeader.getAttributes().containsKey(uniqueAttrName)) {
uniqueAttrName = getUniqueAttributeName(entityType);
}
return uniqueAttrName;
}
private String getUniqueAttributeName(AtlasEntityType entityType) {
return entityType.getUniqAttributes()
.entrySet()
.stream()
.findFirst()
.get().getKey();
} }
private void summarize(String... s) { private void summarize(String... s) {
......
...@@ -19,18 +19,25 @@ ...@@ -19,18 +19,25 @@
package org.apache.atlas.repository.store.graph.v2; package org.apache.atlas.repository.store.graph.v2;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasEntityHeaders; import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.audit.EntityAuditRepository; import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.utils.AtlasJson;
import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.atlas.utils.TestResourceFileUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.elasticsearch.common.util.CollectionUtils;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
...@@ -161,6 +168,26 @@ public class ClassificationAssociatorTest { ...@@ -161,6 +168,26 @@ public class ClassificationAssociatorTest {
assertSummaryElement(summaryArray[0], "Entity", STATUS_SKIPPED, ""); assertSummaryElement(summaryArray[0], "Entity", STATUS_SKIPPED, "");
} }
@Test
public void updaterEntityWithUniqueName() throws IOException, AtlasBaseException {
AtlasEntityDef ed = getAtlasEntityDefFromFile("col-entity-def-unique-name");
AtlasEntityHeaders entityHeaderMap = getEntityHeaderMapFromFile("header-PII-no-qualifiedName");
AtlasEntityStore entitiesStore = mock(AtlasEntityStore.class);
AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
AtlasTypeRegistry.AtlasTransientTypeRegistry ttr = typeRegistry.lockTypeRegistryForUpdate();
ttr.addTypes(CollectionUtils.newSingletonArrayList(ed));
ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(ttr, entitiesStore, "col-entity-PII");
String summary = updater.setClassifications(entityHeaderMap.getGuidHeaderMap());
TypeReference<String[]> typeReference = new TypeReference<String[]>() {};
String[] summaryArray = AtlasJson.fromJson(summary, typeReference);
assertEquals(summaryArray.length, 1);
assertSummaryElement(summaryArray[0], "Update", STATUS_DONE, "PII");
}
@Test @Test
public void updaterTests() throws IOException { public void updaterTests() throws IOException {
updaterAssert("header-None", "col-entity-None"); updaterAssert("header-None", "col-entity-None");
...@@ -227,6 +254,9 @@ public class ClassificationAssociatorTest { ...@@ -227,6 +254,9 @@ public class ClassificationAssociatorTest {
private AtlasEntityHeaders getEntityHeaderMapFromFile(String filename) throws IOException { private AtlasEntityHeaders getEntityHeaderMapFromFile(String filename) throws IOException {
return TestResourceFileUtils.readObjectFromJson(TEST_FILES_SUBDIR, filename, AtlasEntityHeaders.class); return TestResourceFileUtils.readObjectFromJson(TEST_FILES_SUBDIR, filename, AtlasEntityHeaders.class);
} }
private AtlasEntityDef getAtlasEntityDefFromFile(String filename) throws IOException {
return TestResourceFileUtils.readObjectFromJson(TEST_FILES_SUBDIR, filename, AtlasEntityDef.class);
}
private void updaterAssert(String incoming, String entity, String... opNamePair) throws IOException { private void updaterAssert(String incoming, String entity, String... opNamePair) throws IOException {
String[] summary = setupUpdater(incoming, entity, opNamePair.length); String[] summary = setupUpdater(incoming, entity, opNamePair.length);
......
{
"attributeDefs": [
{
"cardinality": "SINGLE",
"includeInNotification": false,
"isIndexable": true,
"isOptional": false,
"isUnique": true,
"name": "name",
"typeName": "string",
"valuesMaxCount": 1,
"valuesMinCount": 1
},
{
"cardinality": "SINGLE",
"includeInNotification": false,
"isIndexable": false,
"isOptional": true,
"isUnique": false,
"name": "optional",
"typeName": "string",
"valuesMaxCount": 1,
"valuesMinCount": 0
},
{
"cardinality": "LIST",
"includeInNotification": false,
"isIndexable": false,
"isOptional": false,
"isUnique": false,
"name": "collection",
"typeName": "array<string>",
"valuesMaxCount": 2147483647,
"valuesMinCount": 1
},
{
"cardinality": "SET",
"includeInNotification": false,
"isIndexable": false,
"isOptional": false,
"isUnique": false,
"name": "set",
"typeName": "array<string>",
"valuesMaxCount": 2147483647,
"valuesMinCount": 1
},
{
"cardinality": "SINGLE",
"includeInNotification": false,
"isIndexable": false,
"isOptional": true,
"isUnique": false,
"name": "short",
"typeName": "short",
"valuesMaxCount": 1,
"valuesMinCount": 0
}
],
"category": "ENTITY",
"createTime": 1549271328342,
"createdBy": "atlas",
"name": "hive_column",
"subTypes": [],
"superTypes": [],
"typeVersion": "1.0",
"updateTime": 1549271328342,
"updatedBy": "atlas",
"version": 1
}
\ No newline at end of file
{
"guidHeaderMap": {
"0ce68113-77fe-4ed1-9585-69371202bd74": {
"typeName": "hive_column",
"attributes": {
"owner": "hive",
"uniqueName": "hortoniabank.us_customers.nationalid@cl1",
"name": "nationalid"
},
"guid": "0ce68113-77fe-4ed1-9585-69371202bd74",
"status": "ACTIVE",
"displayText": "nationalid",
"classificationNames": [
"PII"
],
"classifications": [
{
"typeName": "PII",
"attributes": {
"type": "ssn"
},
"entityGuid": "0ce68113-77fe-4ed1-9585-69371202bd74",
"entityStatus": "ACTIVE",
"propagate": true,
"removePropagationsOnEntityDelete": false
}
],
"meaningNames": [],
"meanings": []
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment