Skip to content

A
atlas
Commit 7174257b by Ashutosh Mestry
Options

ATLAS-2748, ATLAS-2849, ATLAS-2750: Edges' state preserved during import.…

ATLAS-2748, ATLAS-2849, ATLAS-2750: Edges' state preserved during import. Classification dissociation fails on imported entities with classifications. Processing classifications with attributes containing maps and arrays.
parent 9e762c09
Showing with 62 additions and 28 deletions
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java
...@@ -34,6 +34,7 @@ import java.util.UUID; ...@@ -34,6 +34,7 @@ import java.util.UUID;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID; import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
...@@ -354,9 +355,8 @@ public class ElementProcessors { ...@@ -354,9 +355,8 @@ public class ElementProcessors {
private void addMandatoryRelationshipProperties(String label, Map<String, Object> props) { private void addMandatoryRelationshipProperties(String label, Map<String, Object> props) {
props.put(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, UUID.randomUUID().toString()); props.put(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, UUID.randomUUID().toString());
props.put(RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, String.valueOf(getDefaultPropagateValue(label))); props.put(RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, String.valueOf(getDefaultPropagateValue(label)));
props.put(STATE_PROPERTY_KEY, "ACTIVE"); props.put(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY, false);
} }
} }
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
...@@ -47,6 +47,7 @@ public class TypesWithCollectionsFinder { ...@@ -47,6 +47,7 @@ public class TypesWithCollectionsFinder {
addVertexPropertiesForCollectionAttributes(typeRegistry.getAllEntityTypes(), ret); addVertexPropertiesForCollectionAttributes(typeRegistry.getAllEntityTypes(), ret);
addVertexPropertiesForCollectionAttributes(typeRegistry.getAllStructTypes(), ret); addVertexPropertiesForCollectionAttributes(typeRegistry.getAllStructTypes(), ret);
addVertexPropertiesForCollectionAttributes(typeRegistry.getAllClassificationTypes(), ret);
displayInfo("types with properties: ", ret); displayInfo("types with properties: ", ret);
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
...@@ -38,6 +38,7 @@ import java.util.Set; ...@@ -38,6 +38,7 @@ import java.util.Set;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID; import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY; import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY;
...@@ -229,7 +230,6 @@ public class GraphSONUtilityTest extends BaseUtils { ...@@ -229,7 +230,6 @@ public class GraphSONUtilityTest extends BaseUtils {
@Test @Test
public void tagAssociated_NewAttributesAdded() { public void tagAssociated_NewAttributesAdded() {
ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap(), new HashMap<>()); ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap(), new HashMap<>());
TinkerGraph tg = TinkerGraph.open(); TinkerGraph tg = TinkerGraph.open();
GraphSONUtility gu = new GraphSONUtility(elementProcessors); GraphSONUtility gu = new GraphSONUtility(elementProcessors);
...@@ -238,22 +238,8 @@ public class GraphSONUtilityTest extends BaseUtils { ...@@ -238,22 +238,8 @@ public class GraphSONUtilityTest extends BaseUtils {
addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag()); addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
Iterator<Vertex> vertices = tg.vertices(); Iterator<Vertex> vertices = tg.vertices();
while(vertices.hasNext()) { assertVertex(vertices, "16752");
Vertex v = vertices.next(); assertEdge(tg);
if(v.id().toString() != "16752") continue;
assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
}
Iterator<Edge> edges = tg.edges();
while(edges.hasNext()) {
Edge e = edges.next();
assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
}
} }
@Test @Test
...@@ -266,21 +252,31 @@ public class GraphSONUtilityTest extends BaseUtils { ...@@ -266,21 +252,31 @@ public class GraphSONUtilityTest extends BaseUtils {
addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag()); addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
Iterator<Vertex> vertices = tg.vertices(); Iterator<Vertex> vertices = tg.vertices();
while(vertices.hasNext()) { assertVertex(vertices, "16752");
Vertex v = vertices.next(); assertEdge(tg);
if(v.id().toString() != "16752") continue; }
assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
}
private void assertEdge(TinkerGraph tg) {
Iterator<Edge> edges = tg.edges(); Iterator<Edge> edges = tg.edges();
while(edges.hasNext()) { while(edges.hasNext()) {
Edge e = edges.next(); Edge e = edges.next();
assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent()); assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
assertTrue(e.property(Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY).isPresent());
assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE"); assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent()); assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
assertTrue(e.property(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY).isPresent());
assertFalse((boolean) e.property(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY).value());
}
}
private void assertVertex(Iterator<Vertex> vertices, String vertexId) {
while(vertices.hasNext()) {
Vertex v = vertices.next();
if(v.id().toString() != vertexId) continue;
assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
} }
} }
......
repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
...@@ -20,10 +20,13 @@ package org.apache.atlas.repository.migration; ...@@ -20,10 +20,13 @@ package org.apache.atlas.repository.migration;
import com.google.inject.Inject; import com.google.inject.Inject;
import org.apache.atlas.TestModules; import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDBMigrator; import org.apache.atlas.repository.graphdb.GraphDBMigrator;
import org.apache.atlas.repository.graphdb.janus.migration.TypesWithCollectionsFinder; import org.apache.atlas.repository.graphdb.janus.migration.TypesWithCollectionsFinder;
import org.apache.atlas.utils.TestResourceFileUtils;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
...@@ -45,13 +48,15 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts { ...@@ -45,13 +48,15 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts {
@BeforeClass @BeforeClass
public void setup() throws IOException, AtlasBaseException { public void setup() throws IOException, AtlasBaseException {
loadTypesFromJson(); loadTypesFromJson();
typeDefStore.createTypesDef(TestResourceFileUtils.readObjectFromJson(".", "typesDef-classification-with-map", AtlasTypesDef.class));
} }
@Test @Test
public void fetchAll() { public void fetchAll() {
Map<String, Map<String, List<String>>> typeAttrMap = TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry); Map<String, Map<String, List<String>>> typeAttrMap = TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry);
assertEquals(typeAttrMap.size(), 9); assertEquals(typeAttrMap.size(), 10);
assertProperties(typeAttrMap, "__AtlasUserProfile", "ARRAY", "__AtlasUserProfile.savedSearches"); assertProperties(typeAttrMap, "__AtlasUserProfile", "ARRAY", "__AtlasUserProfile.savedSearches");
...@@ -75,6 +80,8 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts { ...@@ -75,6 +80,8 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts {
assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.partitionKeys"); assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.partitionKeys");
assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.columns"); assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.columns");
assertProperties(typeAttrMap, "hive_table", "MAP_PRIMITIVE", "hive_table.parameters"); assertProperties(typeAttrMap, "hive_table", "MAP_PRIMITIVE", "hive_table.parameters");
assertProperties(typeAttrMap, "tag_with_map_of_map", "MAP_PRIMITIVE", "tag_with_map_of_map.tag_with_map_of_map");
} }
private void assertProperties(Map<String, Map<String, List<String>>> typeAttrMap, String typeName, String typeCategory, String propertyName) { private void assertProperties(Map<String, Map<String, List<String>>> typeAttrMap, String typeName, String typeCategory, String propertyName) {
......
repository/src/test/resources/json/typesDef-classification-with-map.json 0 → 100644
{
"classificationDefs": [
{
"category": "CLASSIFICATION",
"guid": "d154cc2e-cd67-4620-9893-0cd6aec54244",
"createdBy": "admin",
"updatedBy": "admin",
"createTime": 1528460320484,
"updateTime": 1528460320484,
"version": 1,
"name": "tag_with_map_of_map",
"description": "tag_with_map_of_map",
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "tag_with_map_of_map",
"typeName": "map<string,map<string,string>>",
"isOptional": true,
"cardinality": "SINGLE",
"valuesMinCount": 0,
"valuesMaxCount": 1,
"isUnique": false,
"isIndexable": false
}
],
"superTypes": [],
"subTypes": []
}
]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment