Skip to content

A
atlas
Commit cbca591d by Sarath Subramanian
Options

ATLAS-2672: Change primitive map type storage in vertex

parent b7641e74
Showing with 287 additions and 140 deletions
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
......@@ -70,7 +70,6 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
private static volatile JanusGraph graphInstance;
public AtlasJanusGraphDatabase() {
//update registry
GraphSONMapper.build().addRegistry(JanusGraphIoRegistry.getInstance()).create();
}
......@@ -79,14 +78,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
startLocalSolr();
Configuration configProperties = ApplicationProperties.get();
Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
//add serializers for non-standard property value types that Atlas uses
janusConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
janusConfig.addProperty("attributes.custom.attribute1.serializer-class",
TypeCategorySerializer.class.getName());
janusConfig.addProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());
//not ideal, but avoids making large changes to Atlas
janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
......
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
......@@ -18,6 +18,7 @@
package org.apache.atlas;
import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
......@@ -67,9 +68,10 @@ public final class TestUtilsV2 {
public static final long TEST_DATE_IN_LONG = 1418265358440L;
public static final String TEST_USER = "testUser";
public static final String TEST_USER = "testUser";
public static final String STRUCT_TYPE = "struct_type";
public static final String ENTITY_TYPE = "entity_type";
public static final String ENTITY_TYPE_MAP = "map_entity_type";
private static AtomicInteger seq = new AtomicInteger();
......@@ -977,6 +979,21 @@ public final class TestUtilsV2 {
return ret;
}
public static AtlasTypesDef defineTypeWithMapAttributes() {
AtlasEntityDef entityType = createClassTypeDef(ENTITY_TYPE_MAP, "entity_type_map_description", Collections.emptySet(),
createUniqueRequiredAttrDef("mapAttr1", "map<string,string>"),
createUniqueRequiredAttrDef("mapAttr2", "map<string,int>"),
createUniqueRequiredAttrDef("mapAttr3", "map<string,boolean>"),
createOptionalAttrDef("mapAttr4", "map<string,float>"),
createOptionalAttrDef("mapAttr5", "map<string,date>"));
AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Arrays.asList(entityType));
populateSystemAttributes(ret);
return ret;
}
public static AtlasEntityWithExtInfo createNestedCollectionAttrEntity() {
AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR);
......@@ -1059,6 +1076,45 @@ public final class TestUtilsV2 {
return ret;
}
public static AtlasEntityWithExtInfo createMapAttrEntity() {
AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_MAP);
Map<String, String> map1 = new HashMap<>();
map1.put("map1Key1", "value1");
map1.put("map1Key2", "value2");
map1.put("map1Key3", "value3");
Map<String, Integer> map2 = new HashMap<>();
map2.put("map2Key1", 100);
map2.put("map2Key2", 200);
map2.put("map2Key3", 300);
Map<String, Boolean> map3 = new HashMap<>();
map3.put("map3Key1", false);
map3.put("map3Key2", true);
map3.put("map3Key3", false);
Map<String, Float> map4 = new HashMap<>();
map4.put("map4Key1", 1.0f);
map4.put("map4Key2", 2.0f);
map4.put("map4Key3", 3.0f);
Map<String, Date> map5 = new HashMap<>();
map5.put("map5Key1", new Date());
map5.put("map5Key2", new Date());
map5.put("map5Key3", new Date());
entity.setAttribute("mapAttr1", map1);
entity.setAttribute("mapAttr2", map2);
entity.setAttribute("mapAttr3", map3);
entity.setAttribute("mapAttr4", map4);
entity.setAttribute("mapAttr5", map5);
AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(entity);
return ret;
}
public static final String randomString() {
return RandomStringUtils.randomAlphanumeric(10);
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -42,9 +42,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasEnumType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
......@@ -62,6 +65,7 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
......@@ -71,6 +75,9 @@ import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SINGLE;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
import static org.apache.atlas.type.AtlasTypeUtil.isArrayType;
import static org.apache.atlas.type.AtlasTypeUtil.isMapType;
/**
......@@ -267,7 +274,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, TYPENAME_PROPERTY_KEY, String.class, true, SINGLE, true, true);
createVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, String.class, false, SINGLE, true, true);
createVertexIndex(management, CLASSIFICATION_ENTITY_GUID, String.class, false, SINGLE, true, true);
createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, Long.class, false, SINGLE, true, false);
// create vertex-centric index
......@@ -316,19 +322,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
boolean isIndexable = attributeDef.getIsIndexable();
String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName);
boolean isArrayType = isArrayType(attribTypeName);
boolean isMapType = isMapType(attribTypeName);
try {
AtlasType atlasType = typeRegistry.getType(typeName);
AtlasType attributeType = typeRegistry.getType(attribTypeName);
if (isMapType || isClassificationType(attributeType)) {
if (isClassificationType(attributeType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
} if (isArrayType) {
}
if (isArrayType) {
createLabelIfNeeded(management, propertyName, attribTypeName);
} if (isEntityType(attributeType)) {
AtlasArrayType arrayType = (AtlasArrayType) attributeType;
boolean isReference = isReference(arrayType.getElementType());
if (!isReference) {
createPropertyKey(management, propertyName, ArrayList.class, SINGLE);
}
}
if (isMapType) {
createLabelIfNeeded(management, propertyName, attribTypeName);
AtlasMapType mapType = (AtlasMapType) attributeType;
boolean isReference = isReference(mapType.getValueType());
if (!isReference) {
createPropertyKey(management, propertyName, HashMap.class, SINGLE);
}
}
if (isEntityType(attributeType)) {
createEdgeLabel(management, propertyName);
} else if (isBuiltInType) {
if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false);
......@@ -381,7 +410,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
private Class getPrimitiveClass(String attribTypeName) {
switch (attribTypeName.toLowerCase()) {
String attributeTypeName = attribTypeName.toLowerCase();
switch (attributeTypeName) {
case ATLAS_TYPE_BOOLEAN:
return Boolean.class;
case ATLAS_TYPE_BYTE:
......@@ -437,6 +468,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private AtlasPropertyKey createPropertyKey(AtlasGraphManagement management, String propertyName, Class propertyClass, AtlasCardinality cardinality) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
}
return propertyKey;
}
private AtlasPropertyKey createVertexIndex(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality, boolean createCompositeIndex,
boolean createCompositeIndexWithTypeAndSuperTypes) {
......@@ -689,17 +730,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private void cleanupIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName());
String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName);
String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = isArrayType(attribTypeName);
boolean isMapType = isMapType(attribTypeName);
try {
AtlasType atlasType = typeRegistry.getType(attribTypeName);
if (isMapType || isArrayType || isClassificationType(atlasType) || isEntityType(atlasType)) {
if (isClassificationType(atlasType) || isEntityType(atlasType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
} else if (isBuiltInType || isEnumType(atlasType)) {
} else if (isBuiltInType || isEnumType(atlasType) || isArrayType || isMapType) {
cleanupIndex(management, propertyName);
} else if (isStructType(atlasType)) {
AtlasStructDef structDef = typeRegistry.getStructDefByName(attribTypeName);
......@@ -714,6 +755,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (LOG.isDebugEnabled()) {
LOG.debug("Invalidating property key = {}", propertyKey);
}
management.deletePropertyKey(propertyKey);
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -1567,23 +1567,13 @@ public final class GraphHelper {
return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX);
}
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (!AtlasGraphUtilsV1.isReference(elementType)) {
return instanceVertex.getProperty(vertexPropertyName, Object.class);
}
return null;
}
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, AtlasAttribute attribute) {
String propertyName = attribute.getVertexPropertyName();
if (isReference(elementType)) {
return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
} else {
return (List) instanceVertex.getListProperty(encodedPropertyName);
return (List) instanceVertex.getListProperty(propertyName);
}
}
......@@ -1593,7 +1583,7 @@ public final class GraphHelper {
if (isReference(mapValueType)) {
return getReferenceMap(instanceVertex, attribute);
} else {
return getPrimitiveMap(instanceVertex, propertyName, mapValueType);
return (Map) instanceVertex.getProperty(propertyName, Map.class);
}
}
......@@ -1622,18 +1612,11 @@ public final class GraphHelper {
}
// map elements for primitive types
public static Map<String,Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName, AtlasType mapValueType) {
String encodedPropertyName = encodePropertyKey(propertyName);
List<String> currentKeys = getListProperty(instanceVertex, encodedPropertyName);
Map<String, Object> ret = new HashMap<>();
public static Map<String, Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName) {
Map<String, Object> ret = instanceVertex.getProperty(encodePropertyKey(propertyName), Map.class);
if (CollectionUtils.isNotEmpty(currentKeys)) {
for (String key : currentKeys) {
String propertyNameForKey = getQualifiedNameForMapKey(encodedPropertyName, encodePropertyKey(key));
Object propertyValueForKey = getMapValueProperty(mapValueType, instanceVertex, propertyNameForKey);
ret.put(key, propertyValueForKey);
}
if (ret == null) {
ret = new HashMap<>();
}
return ret;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -202,7 +202,7 @@ public abstract class DeleteHandlerV1 {
vertexInfoMap.put(guid, new GraphHelper.VertexInfo(entity, vertex));
for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) {
if (! attributeInfo.isOwnedRef()) {
if (!attributeInfo.isOwnedRef()) {
continue;
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
......@@ -19,7 +19,6 @@ package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TimeBoundary;
......@@ -82,9 +81,7 @@ import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
import static org.apache.atlas.repository.graph.GraphHelper.getListProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getMapElementsProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getQualifiedNameForMapKey;
import static org.apache.atlas.repository.graph.GraphHelper.getStatus;
import static org.apache.atlas.repository.graph.GraphHelper.getTraitLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getTraitNames;
......@@ -92,7 +89,6 @@ import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeNames;
import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled;
import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
import static org.apache.atlas.repository.graph.GraphHelper.setListProperty;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
......@@ -811,51 +807,45 @@ public class EntityGraphMapper {
boolean isReference = isReference(mapType.getValueType());
if (MapUtils.isNotEmpty(newVal)) {
AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
String propertyName = ctx.getVertexProperty();
for (Map.Entry<Object, Object> entry : newVal.entrySet()) {
String key = entry.getKey().toString();
String propertyName = (isReference) ? ctx.getVertexProperty() : getQualifiedNameForMapKey(ctx.getVertexProperty(), GraphHelper.encodePropertyKey(key));
AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key);
AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(),
propertyName, mapType.getValueType(), existingEdge);
//Add/Update/Remove property value
Object newEntry = mapCollectionElementsToVertex(mapCtx, context);
if (isReference) {
for (Map.Entry<Object, Object> entry : newVal.entrySet()) {
String key = entry.getKey().toString();
AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key);
if (isReference) {
AtlasEdge edge = (AtlasEdge) newEntry;
edge.setProperty(ATTRIBUTE_KEY_PROPERTY_KEY, key);
} else {
ctx.getReferringVertex().setProperty(propertyName, newEntry);
}
AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(),
propertyName, mapType.getValueType(), existingEdge);
// Add/Update/Remove property value
Object newEntry = mapCollectionElementsToVertex(mapCtx, context);
newMap.put(key, newEntry);
if (newEntry instanceof AtlasEdge) {
AtlasEdge edge = (AtlasEdge) newEntry;
// If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute,
// update the inverse reference value.
if (isReference && newEntry instanceof AtlasEdge && inverseRefAttribute != null) {
AtlasEdge newEdge = (AtlasEdge) newEntry;
edge.setProperty(ATTRIBUTE_KEY_PROPERTY_KEY, key);
addInverseReference(context, inverseRefAttribute, newEdge, getRelationshipAttributes(ctx.getValue()));
}
}
}
// If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute,
// update the inverse reference value.
AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), ctx.getVertexProperty(), currentMap, newMap);
if (inverseRefAttribute != null) {
addInverseReference(context, inverseRefAttribute, edge, getRelationshipAttributes(ctx.getValue()));
}
for (Object newEntry : newMap.values()) {
updateInConsistentOwnedMapVertices(ctx, mapType, newEntry);
}
updateInConsistentOwnedMapVertices(ctx, mapType, newEntry);
// for dereference on way out for primitive map types
if (!isReference) {
Set<String> newKeys = new LinkedHashSet<>(newMap.keySet());
newMap.put(key, newEntry);
}
}
newKeys.addAll(finalMap.keySet());
Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), currentMap, newMap);
newMap.putAll(finalMap);
} else {
// primitive type map
ctx.getReferringVertex().setProperty(propertyName, new HashMap<>(newVal));
setListProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), new ArrayList<>(newKeys));
newVal.forEach((key, value) -> newMap.put(key.toString(), value));
}
}
if (LOG.isDebugEnabled()) {
......@@ -1084,60 +1074,23 @@ public class EntityGraphMapper {
return ret;
}
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) {
if (isReference(elementType)) {
return vertex.getProperty(vertexPropertyName, AtlasEdge.class);
} else if (elementType instanceof AtlasArrayType) {
return vertex.getProperty(vertexPropertyName, List.class);
} else if (elementType instanceof AtlasMapType) {
return vertex.getProperty(vertexPropertyName, Map.class);
}
else {
return vertex.getProperty(vertexPropertyName, String.class).toString();
}
}
private static void setMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, Object value) {
if (isReference(elementType)) {
vertex.setPropertyFromElementId(vertexPropertyName, (AtlasEdge)value);
}
else {
vertex.setProperty(vertexPropertyName, value);
}
}
//Remove unused entries from map
private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, String propertyName,
Map<String, Object> currentMap, Map<String, Object> newMap)
throws AtlasBaseException {
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
boolean isReference = isReference(mapType.getValueType());
//Remove unused entries for reference map
private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, Map<String, Object> currentMap,
Map<String, Object> newMap) throws AtlasBaseException {
Map<String, Object> additionalMap = new HashMap<>();
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
for (String currentKey : currentMap.keySet()) {
boolean shouldDeleteKey = !newMap.containsKey(currentKey);
if (isReference) {
//Delete the edge reference if its not part of new edges created/updated
AtlasEdge currentEdge = (AtlasEdge) currentMap.get(currentKey);
//Delete the edge reference if its not part of new edges created/updated
AtlasEdge currentEdge = (AtlasEdge) currentMap.get(currentKey);
if (!newMap.values().contains(currentEdge)) {
boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex);
if (!newMap.values().contains(currentEdge)) {
boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex);
if (!deleted) {
additionalMap.put(currentKey, currentEdge);
shouldDeleteKey = false;
}
if (!deleted) {
additionalMap.put(currentKey, currentEdge);
}
}
if (!isReference && shouldDeleteKey) {
String propertyNameForKey = getQualifiedNameForMapKey(propertyName, GraphHelper.encodePropertyKey(currentKey));
GraphHelper.setProperty(vertex, propertyNameForKey, null);
}
}
return additionalMap;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
......@@ -619,7 +619,7 @@ public final class EntityGraphRetriever {
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection);
break;
case ARRAY:
ret = mapVertexToArray(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute);
break;
case MAP:
ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
......@@ -660,7 +660,7 @@ public final class EntityGraphRetriever {
}
}
} else {
ret = getPrimitiveMap(entityVertex, propertyName, mapValueType);
ret = getPrimitiveMap(entityVertex, propertyName);
}
if (MapUtils.isEmpty(ret)) {
......@@ -670,12 +670,12 @@ public final class EntityGraphRetriever {
return ret;
}
private List<Object> mapVertexToArray(AtlasVertex entityVertex, String propertyName, AtlasEntityExtInfo entityExtInfo,
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType();
List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, propertyName, attribute);
List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, attribute);
if (CollectionUtils.isEmpty(arrayElements)) {
return null;
......
repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
......@@ -37,9 +37,9 @@ public class HiveStocksTest extends MigrationBaseAsserts {
@Test
public void migrateStocks() throws AtlasBaseException, IOException {
final int EXPECTED_TOTAL_COUNT = 188;
final int EXPECTED_DB_COUNT = 1;
final int EXPECTED_TABLE_COUNT = 1;
final int EXPECTED_TOTAL_COUNT = 188;
final int EXPECTED_DB_COUNT = 1;
final int EXPECTED_TABLE_COUNT = 1;
final int EXPECTED_COLUMN_COUNT = 7;
runFileImporter("stocks_db");
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java
......@@ -30,18 +30,21 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.commons.lang.time.DateUtils;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_MAP;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE;
import static org.apache.atlas.TestUtilsV2.NAME;
......@@ -54,18 +57,21 @@ import static org.testng.AssertJUnit.assertEquals;
public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
private AtlasEntityWithExtInfo complexCollectionAttrEntity;
private AtlasEntityWithExtInfo complexCollectionAttrEntityForDelete;
private AtlasEntityWithExtInfo mapAttributesEntity;
@BeforeClass
public void setUp() throws Exception {
super.setUp();
// create typeDefs
AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes() };
AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes(),
TestUtilsV2.defineTypeWithMapAttributes() };
createTypesDef(testTypesDefs);
// create entity
complexCollectionAttrEntity = TestUtilsV2.createComplexCollectionAttrEntity();
complexCollectionAttrEntityForDelete = TestUtilsV2.createComplexCollectionAttrEntity();
mapAttributesEntity = TestUtilsV2.createMapAttrEntity();
}
@Test
......@@ -78,6 +84,116 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
validateEntity(complexCollectionAttrEntity, getEntityFromStore(entityCreated));
}
@Test
public void testPrimitiveMapAttributes() throws Exception {
init();
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(mapAttributesEntity), false);
AtlasEntityHeader entityCreated = response.getFirstCreatedEntityByTypeName(ENTITY_TYPE_MAP);
AtlasEntity entityFromStore = getEntityFromStore(entityCreated);
validateEntity(mapAttributesEntity, entityFromStore);
// Modify map of primitives
AtlasEntity attrEntity = getEntityFromStore(mapAttributesEntity.getEntity().getGuid());
Map<String, String> map1 = new HashMap<String, String>() {{ put("map1Key11", "value11");
put("map1Key22", "value22");
put("map1Key33", "value33"); }};
Map<String, Integer> map2 = new HashMap<String, Integer>() {{ put("map2Key11", 1100);
put("map2Key22", 2200);
put("map2Key33", 3300); }};
Map<String, Boolean> map3 = new HashMap<String, Boolean>() {{ put("map3Key11", true);
put("map3Key22", false);
put("map3Key33", true); }};
Map<String, Float> map4 = new HashMap<String, Float>() {{ put("map4Key11", 11.0f);
put("map4Key22", 22.0f);
put("map4Key33", 33.0f); }};
Map<String, Date> map5 = new HashMap<String, Date>() {{ put("map5Key11", DateUtils.addHours(new Date(), 1));
put("map5Key22", DateUtils.addHours(new Date(), 2));
put("map5Key33", DateUtils.addHours(new Date(), 3)); }};
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
AtlasEntitiesWithExtInfo attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
AtlasEntityHeader updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
AtlasEntity updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
// Add new entry to map of primitives
map1.put("map1Key44", "value44");
map2.put("map2Key44", 4400);
map3.put("map3Key44", false);
map4.put("map4Key44", 44.0f);
map5.put("map5Key44", DateUtils.addHours(new Date(), 4));
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
// Remove an entry from map of primitives
map1.remove("map1Key11");
map2.remove("map2Key11");
map3.remove("map3Key11");
map4.remove("map4Key11");
map5.remove("map5Key11");
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
// Edit existing entry to map of primitives
map1.put("map1Key44", "value44-edit");
map2.put("map2Key44", 5555);
map3.put("map3Key44", true);
map4.put("map4Key44", 55.5f);
map5.put("map5Key44", DateUtils.addHours(new Date(), 5));
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
// clear primitive map entries
map1.clear();
map2.clear();
map3.clear();
map4.clear();
map5.clear();
updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
updatedFromStore = getEntityFromStore(updatedAttrEntity);
validateEntity(attrEntitiesInfo, updatedFromStore);
}
private void updateEntityMapAttributes(AtlasEntity attrEntity, Map<String, String> map1, Map<String, Integer> map2,
Map<String, Boolean> map3, Map<String, Float> map4, Map<String, Date> map5) {
attrEntity.setAttribute("mapAttr1", map1);
attrEntity.setAttribute("mapAttr2", map2);
attrEntity.setAttribute("mapAttr3", map3);
attrEntity.setAttribute("mapAttr4", map4);
attrEntity.setAttribute("mapAttr5", map5);
}
@Test(dependsOnMethods = "testCreateComplexAttributeEntity")
public void testStructArray() throws Exception {
init();
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
......@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.impexp.ExportService;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.runner.LocalSolrRunner;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment