Commit cbca591d by Sarath Subramanian

ATLAS-2672: Change primitive map type storage in vertex

parent b7641e74
...@@ -70,7 +70,6 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex, ...@@ -70,7 +70,6 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
private static volatile JanusGraph graphInstance; private static volatile JanusGraph graphInstance;
public AtlasJanusGraphDatabase() { public AtlasJanusGraphDatabase() {
//update registry //update registry
GraphSONMapper.build().addRegistry(JanusGraphIoRegistry.getInstance()).create(); GraphSONMapper.build().addRegistry(JanusGraphIoRegistry.getInstance()).create();
} }
...@@ -79,14 +78,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex, ...@@ -79,14 +78,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
startLocalSolr(); startLocalSolr();
Configuration configProperties = ApplicationProperties.get(); Configuration configProperties = ApplicationProperties.get();
Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
//add serializers for non-standard property value types that Atlas uses //add serializers for non-standard property value types that Atlas uses
janusConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName()); janusConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
janusConfig.addProperty("attributes.custom.attribute1.serializer-class", janusConfig.addProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());
TypeCategorySerializer.class.getName());
//not ideal, but avoids making large changes to Atlas //not ideal, but avoids making large changes to Atlas
janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName()); janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.atlas; package org.apache.atlas;
import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
...@@ -67,9 +68,10 @@ public final class TestUtilsV2 { ...@@ -67,9 +68,10 @@ public final class TestUtilsV2 {
public static final long TEST_DATE_IN_LONG = 1418265358440L; public static final long TEST_DATE_IN_LONG = 1418265358440L;
public static final String TEST_USER = "testUser"; public static final String TEST_USER = "testUser";
public static final String STRUCT_TYPE = "struct_type"; public static final String STRUCT_TYPE = "struct_type";
public static final String ENTITY_TYPE = "entity_type"; public static final String ENTITY_TYPE = "entity_type";
public static final String ENTITY_TYPE_MAP = "map_entity_type";
private static AtomicInteger seq = new AtomicInteger(); private static AtomicInteger seq = new AtomicInteger();
...@@ -977,6 +979,21 @@ public final class TestUtilsV2 { ...@@ -977,6 +979,21 @@ public final class TestUtilsV2 {
return ret; return ret;
} }
public static AtlasTypesDef defineTypeWithMapAttributes() {
AtlasEntityDef entityType = createClassTypeDef(ENTITY_TYPE_MAP, "entity_type_map_description", Collections.emptySet(),
createUniqueRequiredAttrDef("mapAttr1", "map<string,string>"),
createUniqueRequiredAttrDef("mapAttr2", "map<string,int>"),
createUniqueRequiredAttrDef("mapAttr3", "map<string,boolean>"),
createOptionalAttrDef("mapAttr4", "map<string,float>"),
createOptionalAttrDef("mapAttr5", "map<string,date>"));
AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Arrays.asList(entityType));
populateSystemAttributes(ret);
return ret;
}
public static AtlasEntityWithExtInfo createNestedCollectionAttrEntity() { public static AtlasEntityWithExtInfo createNestedCollectionAttrEntity() {
AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR); AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR);
...@@ -1059,6 +1076,45 @@ public final class TestUtilsV2 { ...@@ -1059,6 +1076,45 @@ public final class TestUtilsV2 {
return ret; return ret;
} }
public static AtlasEntityWithExtInfo createMapAttrEntity() {
AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_MAP);
Map<String, String> map1 = new HashMap<>();
map1.put("map1Key1", "value1");
map1.put("map1Key2", "value2");
map1.put("map1Key3", "value3");
Map<String, Integer> map2 = new HashMap<>();
map2.put("map2Key1", 100);
map2.put("map2Key2", 200);
map2.put("map2Key3", 300);
Map<String, Boolean> map3 = new HashMap<>();
map3.put("map3Key1", false);
map3.put("map3Key2", true);
map3.put("map3Key3", false);
Map<String, Float> map4 = new HashMap<>();
map4.put("map4Key1", 1.0f);
map4.put("map4Key2", 2.0f);
map4.put("map4Key3", 3.0f);
Map<String, Date> map5 = new HashMap<>();
map5.put("map5Key1", new Date());
map5.put("map5Key2", new Date());
map5.put("map5Key3", new Date());
entity.setAttribute("mapAttr1", map1);
entity.setAttribute("mapAttr2", map2);
entity.setAttribute("mapAttr3", map3);
entity.setAttribute("mapAttr4", map4);
entity.setAttribute("mapAttr5", map5);
AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(entity);
return ret;
}
public static final String randomString() { public static final String randomString() {
return RandomStringUtils.randomAlphanumeric(10); return RandomStringUtils.randomAlphanumeric(10);
} }
......
...@@ -42,9 +42,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; ...@@ -42,9 +42,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex; import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement; import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey; import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasEnumType; import org.apache.atlas.type.AtlasEnumType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasType;
...@@ -62,6 +65,7 @@ import java.math.BigInteger; ...@@ -62,6 +65,7 @@ import java.math.BigInteger;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
...@@ -71,6 +75,9 @@ import static org.apache.atlas.repository.Constants.*; ...@@ -71,6 +75,9 @@ import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST; import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET; import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SINGLE; import static org.apache.atlas.repository.graphdb.AtlasCardinality.SINGLE;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
import static org.apache.atlas.type.AtlasTypeUtil.isArrayType;
import static org.apache.atlas.type.AtlasTypeUtil.isMapType;
/** /**
...@@ -267,7 +274,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang ...@@ -267,7 +274,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, TYPENAME_PROPERTY_KEY, String.class, true, SINGLE, true, true); createVertexIndex(management, TYPENAME_PROPERTY_KEY, String.class, true, SINGLE, true, true);
createVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, String.class, false, SINGLE, true, true); createVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, String.class, false, SINGLE, true, true);
createVertexIndex(management, CLASSIFICATION_ENTITY_GUID, String.class, false, SINGLE, true, true); createVertexIndex(management, CLASSIFICATION_ENTITY_GUID, String.class, false, SINGLE, true, true);
createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, Long.class, false, SINGLE, true, false); createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, Long.class, false, SINGLE, true, false);
// create vertex-centric index // create vertex-centric index
...@@ -316,19 +322,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang ...@@ -316,19 +322,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
boolean isIndexable = attributeDef.getIsIndexable(); boolean isIndexable = attributeDef.getIsIndexable();
String attribTypeName = attributeDef.getTypeName(); String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName); boolean isArrayType = isArrayType(attribTypeName);
boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName); boolean isMapType = isMapType(attribTypeName);
try { try {
AtlasType atlasType = typeRegistry.getType(typeName); AtlasType atlasType = typeRegistry.getType(typeName);
AtlasType attributeType = typeRegistry.getType(attribTypeName); AtlasType attributeType = typeRegistry.getType(attribTypeName);
if (isMapType || isClassificationType(attributeType)) { if (isClassificationType(attributeType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName); LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
} if (isArrayType) { }
if (isArrayType) {
createLabelIfNeeded(management, propertyName, attribTypeName); createLabelIfNeeded(management, propertyName, attribTypeName);
} if (isEntityType(attributeType)) {
AtlasArrayType arrayType = (AtlasArrayType) attributeType;
boolean isReference = isReference(arrayType.getElementType());
if (!isReference) {
createPropertyKey(management, propertyName, ArrayList.class, SINGLE);
}
}
if (isMapType) {
createLabelIfNeeded(management, propertyName, attribTypeName);
AtlasMapType mapType = (AtlasMapType) attributeType;
boolean isReference = isReference(mapType.getValueType());
if (!isReference) {
createPropertyKey(management, propertyName, HashMap.class, SINGLE);
}
}
if (isEntityType(attributeType)) {
createEdgeLabel(management, propertyName); createEdgeLabel(management, propertyName);
} else if (isBuiltInType) { } else if (isBuiltInType) {
if (isRelationshipType(atlasType)) { if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false); createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false);
...@@ -381,7 +410,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang ...@@ -381,7 +410,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
} }
private Class getPrimitiveClass(String attribTypeName) { private Class getPrimitiveClass(String attribTypeName) {
switch (attribTypeName.toLowerCase()) { String attributeTypeName = attribTypeName.toLowerCase();
switch (attributeTypeName) {
case ATLAS_TYPE_BOOLEAN: case ATLAS_TYPE_BOOLEAN:
return Boolean.class; return Boolean.class;
case ATLAS_TYPE_BYTE: case ATLAS_TYPE_BYTE:
...@@ -437,6 +468,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang ...@@ -437,6 +468,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
} }
} }
private AtlasPropertyKey createPropertyKey(AtlasGraphManagement management, String propertyName, Class propertyClass, AtlasCardinality cardinality) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
}
return propertyKey;
}
private AtlasPropertyKey createVertexIndex(AtlasGraphManagement management, String propertyName, Class propertyClass, private AtlasPropertyKey createVertexIndex(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality, boolean createCompositeIndex, boolean isUnique, AtlasCardinality cardinality, boolean createCompositeIndex,
boolean createCompositeIndexWithTypeAndSuperTypes) { boolean createCompositeIndexWithTypeAndSuperTypes) {
...@@ -689,17 +730,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang ...@@ -689,17 +730,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private void cleanupIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) { private void cleanupIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName()); final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName());
String attribTypeName = attributeDef.getTypeName(); String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName); boolean isArrayType = isArrayType(attribTypeName);
boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName); boolean isMapType = isMapType(attribTypeName);
try { try {
AtlasType atlasType = typeRegistry.getType(attribTypeName); AtlasType atlasType = typeRegistry.getType(attribTypeName);
if (isMapType || isArrayType || isClassificationType(atlasType) || isEntityType(atlasType)) { if (isClassificationType(atlasType) || isEntityType(atlasType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName); LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
} else if (isBuiltInType || isEnumType(atlasType)) { } else if (isBuiltInType || isEnumType(atlasType) || isArrayType || isMapType) {
cleanupIndex(management, propertyName); cleanupIndex(management, propertyName);
} else if (isStructType(atlasType)) { } else if (isStructType(atlasType)) {
AtlasStructDef structDef = typeRegistry.getStructDefByName(attribTypeName); AtlasStructDef structDef = typeRegistry.getStructDefByName(attribTypeName);
...@@ -714,6 +755,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang ...@@ -714,6 +755,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Invalidating property key = {}", propertyKey); LOG.debug("Invalidating property key = {}", propertyKey);
} }
management.deletePropertyKey(propertyKey); management.deletePropertyKey(propertyKey);
} }
......
...@@ -1567,23 +1567,13 @@ public final class GraphHelper { ...@@ -1567,23 +1567,13 @@ public final class GraphHelper {
return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX); return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX);
} }
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) { public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, AtlasAttribute attribute) {
String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName); String propertyName = attribute.getVertexPropertyName();
if (!AtlasGraphUtilsV1.isReference(elementType)) {
return instanceVertex.getProperty(vertexPropertyName, Object.class);
}
return null;
}
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (isReference(elementType)) { if (isReference(elementType)) {
return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute); return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
} else { } else {
return (List) instanceVertex.getListProperty(encodedPropertyName); return (List) instanceVertex.getListProperty(propertyName);
} }
} }
...@@ -1593,7 +1583,7 @@ public final class GraphHelper { ...@@ -1593,7 +1583,7 @@ public final class GraphHelper {
if (isReference(mapValueType)) { if (isReference(mapValueType)) {
return getReferenceMap(instanceVertex, attribute); return getReferenceMap(instanceVertex, attribute);
} else { } else {
return getPrimitiveMap(instanceVertex, propertyName, mapValueType); return (Map) instanceVertex.getProperty(propertyName, Map.class);
} }
} }
...@@ -1622,18 +1612,11 @@ public final class GraphHelper { ...@@ -1622,18 +1612,11 @@ public final class GraphHelper {
} }
// map elements for primitive types // map elements for primitive types
public static Map<String,Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName, AtlasType mapValueType) { public static Map<String, Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName) {
String encodedPropertyName = encodePropertyKey(propertyName); Map<String, Object> ret = instanceVertex.getProperty(encodePropertyKey(propertyName), Map.class);
List<String> currentKeys = getListProperty(instanceVertex, encodedPropertyName);
Map<String, Object> ret = new HashMap<>();
if (CollectionUtils.isNotEmpty(currentKeys)) { if (ret == null) {
for (String key : currentKeys) { ret = new HashMap<>();
String propertyNameForKey = getQualifiedNameForMapKey(encodedPropertyName, encodePropertyKey(key));
Object propertyValueForKey = getMapValueProperty(mapValueType, instanceVertex, propertyNameForKey);
ret.put(key, propertyValueForKey);
}
} }
return ret; return ret;
......
...@@ -202,7 +202,7 @@ public abstract class DeleteHandlerV1 { ...@@ -202,7 +202,7 @@ public abstract class DeleteHandlerV1 {
vertexInfoMap.put(guid, new GraphHelper.VertexInfo(entity, vertex)); vertexInfoMap.put(guid, new GraphHelper.VertexInfo(entity, vertex));
for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) { for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) {
if (! attributeInfo.isOwnedRef()) { if (!attributeInfo.isOwnedRef()) {
continue; continue;
} }
......
...@@ -619,7 +619,7 @@ public final class EntityGraphRetriever { ...@@ -619,7 +619,7 @@ public final class EntityGraphRetriever {
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection); ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection);
break; break;
case ARRAY: case ARRAY:
ret = mapVertexToArray(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute); ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute);
break; break;
case MAP: case MAP:
ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute); ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
...@@ -660,7 +660,7 @@ public final class EntityGraphRetriever { ...@@ -660,7 +660,7 @@ public final class EntityGraphRetriever {
} }
} }
} else { } else {
ret = getPrimitiveMap(entityVertex, propertyName, mapValueType); ret = getPrimitiveMap(entityVertex, propertyName);
} }
if (MapUtils.isEmpty(ret)) { if (MapUtils.isEmpty(ret)) {
...@@ -670,12 +670,12 @@ public final class EntityGraphRetriever { ...@@ -670,12 +670,12 @@ public final class EntityGraphRetriever {
return ret; return ret;
} }
private List<Object> mapVertexToArray(AtlasVertex entityVertex, String propertyName, AtlasEntityExtInfo entityExtInfo, private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException { boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType(); AtlasType arrayElementType = arrayType.getElementType();
List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, propertyName, attribute); List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, attribute);
if (CollectionUtils.isEmpty(arrayElements)) { if (CollectionUtils.isEmpty(arrayElements)) {
return null; return null;
......
...@@ -37,9 +37,9 @@ public class HiveStocksTest extends MigrationBaseAsserts { ...@@ -37,9 +37,9 @@ public class HiveStocksTest extends MigrationBaseAsserts {
@Test @Test
public void migrateStocks() throws AtlasBaseException, IOException { public void migrateStocks() throws AtlasBaseException, IOException {
final int EXPECTED_TOTAL_COUNT = 188; final int EXPECTED_TOTAL_COUNT = 188;
final int EXPECTED_DB_COUNT = 1; final int EXPECTED_DB_COUNT = 1;
final int EXPECTED_TABLE_COUNT = 1; final int EXPECTED_TABLE_COUNT = 1;
final int EXPECTED_COLUMN_COUNT = 7; final int EXPECTED_COLUMN_COUNT = 7;
runFileImporter("stocks_db"); runFileImporter("stocks_db");
......
...@@ -30,18 +30,21 @@ import org.apache.atlas.model.typedef.AtlasTypesDef; ...@@ -30,18 +30,21 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.commons.lang.time.DateUtils;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE; import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_MAP;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR; import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE; import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE;
import static org.apache.atlas.TestUtilsV2.NAME; import static org.apache.atlas.TestUtilsV2.NAME;
...@@ -54,18 +57,21 @@ import static org.testng.AssertJUnit.assertEquals; ...@@ -54,18 +57,21 @@ import static org.testng.AssertJUnit.assertEquals;
public class AtlasComplexAttributesTest extends AtlasEntityTestBase { public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
private AtlasEntityWithExtInfo complexCollectionAttrEntity; private AtlasEntityWithExtInfo complexCollectionAttrEntity;
private AtlasEntityWithExtInfo complexCollectionAttrEntityForDelete; private AtlasEntityWithExtInfo complexCollectionAttrEntityForDelete;
private AtlasEntityWithExtInfo mapAttributesEntity;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
super.setUp(); super.setUp();
// create typeDefs // create typeDefs
AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes() }; AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes(),
TestUtilsV2.defineTypeWithMapAttributes() };
createTypesDef(testTypesDefs); createTypesDef(testTypesDefs);
// create entity // create entity
complexCollectionAttrEntity = TestUtilsV2.createComplexCollectionAttrEntity(); complexCollectionAttrEntity = TestUtilsV2.createComplexCollectionAttrEntity();
complexCollectionAttrEntityForDelete = TestUtilsV2.createComplexCollectionAttrEntity(); complexCollectionAttrEntityForDelete = TestUtilsV2.createComplexCollectionAttrEntity();
mapAttributesEntity = TestUtilsV2.createMapAttrEntity();
} }
@Test @Test
...@@ -78,6 +84,116 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { ...@@ -78,6 +84,116 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
validateEntity(complexCollectionAttrEntity, getEntityFromStore(entityCreated)); validateEntity(complexCollectionAttrEntity, getEntityFromStore(entityCreated));
} }
@Test
public void testPrimitiveMapAttributes() throws Exception {
init();
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(mapAttributesEntity), false);
AtlasEntityHeader entityCreated = response.getFirstCreatedEntityByTypeName(ENTITY_TYPE_MAP);
AtlasEntity entityFromStore = getEntityFromStore(entityCreated);
validateEntity(mapAttributesEntity, entityFromStore);
// Modify map of primitives
AtlasEntity attrEntity = getEntityFromStore(mapAttributesEntity.getEntity().getGuid());
Map<String, String> map1 = new HashMap<String, String>() {{ put("map1Key11", "value11");
put("map1Key22", "value22");
put("map1Key33", "value33"); }};
Map<String, Integer> map2 = new HashMap<String, Integer>() {{ put("map2Key11", 1100);
put("map2Key22", 2200);
put("map2Key33", 3300); }};
Map<String, Boolean> map3 = new HashMap<String, Boolean>() {{ put("map3Key11", true);
put("map3Key22", false);
put("map3Key33", true); }};
Map<String, Float> map4 = new HashMap<String, Float>() {{ put("map4Key11", 11.0f);
put("map4Key22", 22.0f);
put("map4Key33", 33.0f); }};
Map<String, Date> map5 = new HashMap<String, Date>() {{ put("map5Key11", DateUtils.addHours(new Date(), 1));
put("map5Key22", DateUtils.addHours(new Date(), 2));
put("map5Key33", DateUtils.addHours(new Date(), 3)); }};
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
AtlasEntitiesWithExtInfo attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
AtlasEntityHeader updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
AtlasEntity updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
// Add new entry to map of primitives
map1.put("map1Key44", "value44");
map2.put("map2Key44", 4400);
map3.put("map3Key44", false);
map4.put("map4Key44", 44.0f);
map5.put("map5Key44", DateUtils.addHours(new Date(), 4));
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
// Remove an entry from map of primitives
map1.remove("map1Key11");
map2.remove("map2Key11");
map3.remove("map3Key11");
map4.remove("map4Key11");
map5.remove("map5Key11");
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
// Edit existing entry to map of primitives
map1.put("map1Key44", "value44-edit");
map2.put("map2Key44", 5555);
map3.put("map3Key44", true);
map4.put("map4Key44", 55.5f);
map5.put("map5Key44", DateUtils.addHours(new Date(), 5));
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
// clear primitive map entries
map1.clear();
map2.clear();
map3.clear();
map4.clear();
map5.clear();
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
}
private void updateEntityMapAttributes(AtlasEntity attrEntity, Map<String, String> map1, Map<String, Integer> map2,
Map<String, Boolean> map3, Map<String, Float> map4, Map<String, Date> map5) {
attrEntity.setAttribute("mapAttr1", map1);
attrEntity.setAttribute("mapAttr2", map2);
attrEntity.setAttribute("mapAttr3", map3);
attrEntity.setAttribute("mapAttr4", map4);
attrEntity.setAttribute("mapAttr5", map5);
}
@Test(dependsOnMethods = "testCreateComplexAttributeEntity") @Test(dependsOnMethods = "testCreateComplexAttributeEntity")
public void testStructArray() throws Exception { public void testStructArray() throws Exception {
init(); init();
......
...@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; ...@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.impexp.ExportService;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.runner.LocalSolrRunner; import org.apache.atlas.runner.LocalSolrRunner;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment