Skip to content

A
atlas
Commit 854a8a01 by Le Ma Committed by Sarath Subramanian
Options

ATLAS-3327 Create internal vertex property for classification names and…

ATLAS-3327 Create internal vertex property for classification names and propagated classification names
parent 7054e2b1
Showing with 97 additions and 19 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -89,6 +89,8 @@ public final class Constants {
public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -92,6 +92,7 @@ import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_BLOCKED_PROPAGATED_CLASSIFICATIONS_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
......@@ -658,12 +659,16 @@ public final class GraphHelper {
return element.toString();
}
public static void addToPropagatedTraitNames(AtlasVertex entityVertex, String classificationName) {
public static void addToPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName, string(entityVertex));
}
entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
String propClsNames = entityVertex.getProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, String.class);
propClsNames = StringUtils.isEmpty(propClsNames)? classificationName : propClsNames + "|" + classificationName;
entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
}
/**
......
repository/src/main/java/org/apache/atlas/repository/patches/ClassificationTextPatch.java
......@@ -72,6 +72,7 @@ public class ClassificationTextPatch extends AtlasPatchHandler {
}
getEntityGraphMapper().updateClassificationText(vertex);
getEntityGraphMapper().setClassificationNames(vertex);
if(LOG.isDebugEnabled()) {
LOG.debug("processItem(typeName={}, vertexId={}): Done!", typeName, vertexId);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -64,6 +64,7 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.*;
......@@ -449,7 +450,7 @@ public abstract class DeleteHandlerV1 {
graphHelper.addClassificationEdge(propagatedEntityVertex, classificationVertex, true);
addToPropagatedTraitNames(propagatedEntityVertex, classificationName);
addToPropagatedClassificationAndTraitNames(propagatedEntityVertex, classificationName);
// record add propagation details to send notifications at the end
RequestContext context = RequestContext.get();
......@@ -613,7 +614,7 @@ public abstract class DeleteHandlerV1 {
graphHelper.removeEdge(propagatedEdge);
removeFromPropagatedTraitNames(impactedEntityVertex, classificationName);
removeFromPropagatedClassificationAndTraitNames(impactedEntityVertex, classificationName);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug(" --> Not removing propagated classification edge from [{}] --> [{}][{}] using edge label: [{}], since edge doesn't exist",
......@@ -665,7 +666,7 @@ public abstract class DeleteHandlerV1 {
getTypeName(entityVertex), GraphHelper.getGuid(entityVertex), CLASSIFICATION_LABEL);
}
removeFromPropagatedTraitNames(entityVertex, classificationName);
removeFromPropagatedClassificationAndTraitNames(entityVertex, classificationName);
deleteEdge(edge, true);
......@@ -986,16 +987,17 @@ public abstract class DeleteHandlerV1 {
return Objects.nonNull(entityType) && entityType.isInternalType();
}
private void removeFromPropagatedTraitNames(AtlasVertex entityVertex, String classificationName) {
private void removeFromPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
if (entityVertex != null && StringUtils.isNotEmpty(classificationName)) {
List<String> propagatedTraitNames = getTraitNames(entityVertex, true);
propagatedTraitNames.remove(classificationName);
entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY);
for (String propagatedTraitName : propagatedTraitNames) {
addToPropagatedTraitNames(entityVertex, propagatedTraitName);
addToPropagatedClassificationAndTraitNames(entityVertex, propagatedTraitName);
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
......@@ -81,6 +81,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PA
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graph.GraphHelper.DEFAULT_REMOVE_PROPAGATIONS_ON_ENTITY_DELETE;
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
......@@ -107,6 +108,7 @@ public class EntityGraphMapper {
private static final String SOFT_REF_FORMAT = "%s:%s";
private static final int INDEXED_STR_SAFE_LEN = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private static final String CLASSIFICATION_NAME_DELIMITER = "|";
private final GraphHelper graphHelper = GraphHelper.getInstance();
private final AtlasGraph graph;
......@@ -1510,7 +1512,7 @@ public class EntityGraphMapper {
LOG.debug("Adding classification [{}] to [{}] using edge label: [{}]", classificationName, entityType.getTypeName(), getTraitLabel(classificationName));
}
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
addClassificationAndTraitNames(entityVertex, classificationName);
// add a new AtlasVertex for the struct or trait instance
AtlasVertex classificationVertex = createClassificationVertex(classification);
......@@ -1575,7 +1577,6 @@ public class EntityGraphMapper {
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex);
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
if (CollectionUtils.isNotEmpty(addedClassifications)) {
entityChangeNotifier.onClassificationAddedToEntity(entity, addedClassifications);
......@@ -1675,7 +1676,7 @@ public class EntityGraphMapper {
traitNames.remove(classificationName);
updateTraitNamesProperty(entityVertex, traitNames);
updateClassificationAndTraitNames(entityVertex, traitNames);
updateModificationMetadata(entityVertex);
......@@ -1696,6 +1697,38 @@ public class EntityGraphMapper {
return entity;
}
public void setClassificationNames(AtlasVertex vertex) {
List<String> clsNamesList = getTraitNames(vertex, false);
List<String> propClsNamesList = getTraitNames(vertex, true);
String clsNames = StringUtils.join(clsNamesList, CLASSIFICATION_NAME_DELIMITER);
String propClsNames = StringUtils.join(propClsNamesList, CLASSIFICATION_NAME_DELIMITER);
vertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
vertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
}
private void addClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
String clsNames = entityVertex.getProperty(CLASSIFICATION_NAMES_KEY, String.class);
clsNames = StringUtils.isEmpty(clsNames) ? classificationName : clsNames + CLASSIFICATION_NAME_DELIMITER + classificationName;
entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
}
private void updateClassificationAndTraitNames(AtlasVertex entityVertex, List<String> traitNames) {
if (entityVertex != null) {
entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(CLASSIFICATION_NAMES_KEY);
for (String traitName : traitNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
}
entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, StringUtils.join(traitNames, CLASSIFICATION_NAME_DELIMITER));
}
}
public void updateClassifications(EntityMutationContext context, String guid, List<AtlasClassification> classifications) throws AtlasBaseException {
if (CollectionUtils.isEmpty(classifications)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "update", guid);
......@@ -1918,16 +1951,6 @@ public class EntityGraphMapper {
}
}
private void updateTraitNamesProperty(AtlasVertex entityVertex, List<String> traitNames) {
if (entityVertex != null) {
entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
for (String traitName : traitNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
}
}
}
private void validateClassificationExists(List<String> existingClassifications, List<String> suppliedClassifications) throws AtlasBaseException {
Set<String> existingNames = new HashSet<>(existingClassifications);
for (String classificationName : suppliedClassifications) {
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java
......@@ -30,6 +30,9 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.web.rest.EntityREST;
......@@ -126,6 +129,13 @@ public class TestEntityREST {
Assert.assertNotNull(retrievedClassification);
Assert.assertEquals(retrievedClassification, testClassification);
// For ATLAS-3327 to test internal properties are added properly.
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, TestUtilsV2.CLASSIFICATION);
}
@Test(dependsOnMethods = "testGetEntityById")
......@@ -156,6 +166,14 @@ public class TestEntityREST {
Assert.assertEquals(retrievedClassification.getAttribute(attrName), phiClassification.getAttribute(attrName));
}
// For ATLAS-3327 to test internal properties are added properly.
String expectedClsNames = TestUtilsV2.CLASSIFICATION + "|" + TestUtilsV2.PHI;
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
// update multiple tags attributes
phiClassification = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
put("stringAttr", "sample_string_v2");
......@@ -178,7 +196,20 @@ public class TestEntityREST {
Assert.assertNotNull(updatedClassification);
Assert.assertEquals(updatedClassification.getAttribute("tag"), testClassification.getAttribute("tag"));
vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
deleteClassification(dbEntity.getGuid(), TestUtilsV2.PHI);
expectedClsNames = TestUtilsV2.CLASSIFICATION;
vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
}
@Test(dependsOnMethods = "testAddAndGetClassification")
......@@ -197,6 +228,13 @@ public class TestEntityREST {
deleteClassification(dbEntity.getGuid(), TestUtilsV2.CLASSIFICATION);
final AtlasClassification.AtlasClassifications retrievedClassifications = entityREST.getClassifications(dbEntity.getGuid());
String expectedClsNames = "";
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
Assert.assertNotNull(retrievedClassifications);
Assert.assertEquals(retrievedClassifications.getList().size(), 0);
}
......@@ -219,6 +257,13 @@ public class TestEntityREST {
Assert.assertEquals(classifications, retrievedClassificationsList);
String expectedClsNames = TestUtilsV2.CLASSIFICATION;
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
final AtlasClassification retrievedClassification = entityREST.getClassification(dbEntity.getGuid(), TestUtilsV2.CLASSIFICATION);
Assert.assertNotNull(retrievedClassification);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment