diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java index 3636ca6..16aecd5 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java @@ -70,7 +70,6 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex, private static volatile JanusGraph graphInstance; public AtlasJanusGraphDatabase() { - //update registry GraphSONMapper.build().addRegistry(JanusGraphIoRegistry.getInstance()).create(); } @@ -79,14 +78,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex, startLocalSolr(); Configuration configProperties = ApplicationProperties.get(); - - Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX); + Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX); //add serializers for non-standard property value types that Atlas uses - janusConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName()); - janusConfig.addProperty("attributes.custom.attribute1.serializer-class", - TypeCategorySerializer.class.getName()); + janusConfig.addProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName()); //not ideal, but avoids making large changes to Atlas janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName()); diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index f48ccef..886ce77 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -18,6 +18,7 @@ package org.apache.atlas; +import com.sun.org.apache.xpath.internal.operations.Bool; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; @@ -67,9 +68,10 @@ public final class TestUtilsV2 { public static final long TEST_DATE_IN_LONG = 1418265358440L; - public static final String TEST_USER = "testUser"; + public static final String TEST_USER = "testUser"; public static final String STRUCT_TYPE = "struct_type"; public static final String ENTITY_TYPE = "entity_type"; + public static final String ENTITY_TYPE_MAP = "map_entity_type"; private static AtomicInteger seq = new AtomicInteger(); @@ -977,6 +979,21 @@ public final class TestUtilsV2 { return ret; } + public static AtlasTypesDef defineTypeWithMapAttributes() { + AtlasEntityDef entityType = createClassTypeDef(ENTITY_TYPE_MAP, "entity_type_map_description", Collections.emptySet(), + createUniqueRequiredAttrDef("mapAttr1", "map<string,string>"), + createUniqueRequiredAttrDef("mapAttr2", "map<string,int>"), + createUniqueRequiredAttrDef("mapAttr3", "map<string,boolean>"), + createOptionalAttrDef("mapAttr4", "map<string,float>"), + createOptionalAttrDef("mapAttr5", "map<string,date>")); + + AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Arrays.asList(entityType)); + + populateSystemAttributes(ret); + + return ret; + } + public static AtlasEntityWithExtInfo createNestedCollectionAttrEntity() { AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR); @@ -1059,6 +1076,45 @@ public final class TestUtilsV2 { return ret; } + public static AtlasEntityWithExtInfo createMapAttrEntity() { + AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_MAP); + + Map<String, String> map1 = new HashMap<>(); + map1.put("map1Key1", "value1"); + map1.put("map1Key2", "value2"); + map1.put("map1Key3", "value3"); + + Map<String, Integer> map2 = new HashMap<>(); + map2.put("map2Key1", 100); + map2.put("map2Key2", 200); + map2.put("map2Key3", 300); + + Map<String, Boolean> map3 = new HashMap<>(); + map3.put("map3Key1", false); + map3.put("map3Key2", true); + map3.put("map3Key3", false); + + Map<String, Float> map4 = new HashMap<>(); + map4.put("map4Key1", 1.0f); + map4.put("map4Key2", 2.0f); + map4.put("map4Key3", 3.0f); + + Map<String, Date> map5 = new HashMap<>(); + map5.put("map5Key1", new Date()); + map5.put("map5Key2", new Date()); + map5.put("map5Key3", new Date()); + + entity.setAttribute("mapAttr1", map1); + entity.setAttribute("mapAttr2", map2); + entity.setAttribute("mapAttr3", map3); + entity.setAttribute("mapAttr4", map4); + entity.setAttribute("mapAttr5", map5); + + AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(entity); + + return ret; + } + public static final String randomString() { return RandomStringUtils.randomAlphanumeric(10); } diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index f75f786..f289365 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -42,9 +42,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphIndex; import org.apache.atlas.repository.graphdb.AtlasGraphManagement; import org.apache.atlas.repository.graphdb.AtlasPropertyKey; +import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; +import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEnumType; +import org.apache.atlas.type.AtlasMapType; import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasType; @@ -62,6 +65,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -71,6 +75,9 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST; import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET; import static org.apache.atlas.repository.graphdb.AtlasCardinality.SINGLE; +import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference; +import static org.apache.atlas.type.AtlasTypeUtil.isArrayType; +import static org.apache.atlas.type.AtlasTypeUtil.isMapType; /** @@ -267,7 +274,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang createVertexIndex(management, TYPENAME_PROPERTY_KEY, String.class, true, SINGLE, true, true); createVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, String.class, false, SINGLE, true, true); createVertexIndex(management, CLASSIFICATION_ENTITY_GUID, String.class, false, SINGLE, true, true); - createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, Long.class, false, SINGLE, true, false); // create vertex-centric index @@ -316,19 +322,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang boolean isIndexable = attributeDef.getIsIndexable(); String attribTypeName = attributeDef.getTypeName(); boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); - boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName); - boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName); + boolean isArrayType = isArrayType(attribTypeName); + boolean isMapType = isMapType(attribTypeName); try { AtlasType atlasType = typeRegistry.getType(typeName); AtlasType attributeType = typeRegistry.getType(attribTypeName); - if (isMapType || isClassificationType(attributeType)) { + if (isClassificationType(attributeType)) { LOG.warn("Ignoring non-indexable attribute {}", attribTypeName); - } if (isArrayType) { + } + + if (isArrayType) { createLabelIfNeeded(management, propertyName, attribTypeName); - } if (isEntityType(attributeType)) { + + AtlasArrayType arrayType = (AtlasArrayType) attributeType; + boolean isReference = isReference(arrayType.getElementType()); + + if (!isReference) { + createPropertyKey(management, propertyName, ArrayList.class, SINGLE); + } + } + + if (isMapType) { + createLabelIfNeeded(management, propertyName, attribTypeName); + + AtlasMapType mapType = (AtlasMapType) attributeType; + boolean isReference = isReference(mapType.getValueType()); + + if (!isReference) { + createPropertyKey(management, propertyName, HashMap.class, SINGLE); + } + } + + if (isEntityType(attributeType)) { createEdgeLabel(management, propertyName); + } else if (isBuiltInType) { if (isRelationshipType(atlasType)) { createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false); @@ -381,7 +410,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } private Class getPrimitiveClass(String attribTypeName) { - switch (attribTypeName.toLowerCase()) { + String attributeTypeName = attribTypeName.toLowerCase(); + + switch (attributeTypeName) { case ATLAS_TYPE_BOOLEAN: return Boolean.class; case ATLAS_TYPE_BYTE: @@ -437,6 +468,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } } + private AtlasPropertyKey createPropertyKey(AtlasGraphManagement management, String propertyName, Class propertyClass, AtlasCardinality cardinality) { + AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName); + + if (propertyKey == null) { + propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality); + } + + return propertyKey; + } + private AtlasPropertyKey createVertexIndex(AtlasGraphManagement management, String propertyName, Class propertyClass, boolean isUnique, AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes) { @@ -689,17 +730,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang private void cleanupIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) { final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName()); - String attribTypeName = attributeDef.getTypeName(); - boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); - boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName); - boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName); + String attribTypeName = attributeDef.getTypeName(); + boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); + boolean isArrayType = isArrayType(attribTypeName); + boolean isMapType = isMapType(attribTypeName); try { AtlasType atlasType = typeRegistry.getType(attribTypeName); - if (isMapType || isArrayType || isClassificationType(atlasType) || isEntityType(atlasType)) { + if (isClassificationType(atlasType) || isEntityType(atlasType)) { LOG.warn("Ignoring non-indexable attribute {}", attribTypeName); - } else if (isBuiltInType || isEnumType(atlasType)) { + } else if (isBuiltInType || isEnumType(atlasType) || isArrayType || isMapType) { cleanupIndex(management, propertyName); } else if (isStructType(atlasType)) { AtlasStructDef structDef = typeRegistry.getStructDefByName(attribTypeName); @@ -714,6 +755,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang if (LOG.isDebugEnabled()) { LOG.debug("Invalidating property key = {}", propertyKey); } + management.deletePropertyKey(propertyKey); } diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 7490a15..57dc5bd 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -1567,23 +1567,13 @@ public final class GraphHelper { return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX); } - public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) { - String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName); - - if (!AtlasGraphUtilsV1.isReference(elementType)) { - return instanceVertex.getProperty(vertexPropertyName, Object.class); - } - - return null; - } - - public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) { - String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName); + public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, AtlasAttribute attribute) { + String propertyName = attribute.getVertexPropertyName(); if (isReference(elementType)) { return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute); } else { - return (List) instanceVertex.getListProperty(encodedPropertyName); + return (List) instanceVertex.getListProperty(propertyName); } } @@ -1593,7 +1583,7 @@ public final class GraphHelper { if (isReference(mapValueType)) { return getReferenceMap(instanceVertex, attribute); } else { - return getPrimitiveMap(instanceVertex, propertyName, mapValueType); + return (Map) instanceVertex.getProperty(propertyName, Map.class); } } @@ -1622,18 +1612,11 @@ public final class GraphHelper { } // map elements for primitive types - public static Map<String,Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName, AtlasType mapValueType) { - String encodedPropertyName = encodePropertyKey(propertyName); - List<String> currentKeys = getListProperty(instanceVertex, encodedPropertyName); - Map<String, Object> ret = new HashMap<>(); + public static Map<String, Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName) { + Map<String, Object> ret = instanceVertex.getProperty(encodePropertyKey(propertyName), Map.class); - if (CollectionUtils.isNotEmpty(currentKeys)) { - for (String key : currentKeys) { - String propertyNameForKey = getQualifiedNameForMapKey(encodedPropertyName, encodePropertyKey(key)); - Object propertyValueForKey = getMapValueProperty(mapValueType, instanceVertex, propertyNameForKey); - - ret.put(key, propertyValueForKey); - } + if (ret == null) { + ret = new HashMap<>(); } return ret; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index d285e69..e62c041 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -202,7 +202,7 @@ public abstract class DeleteHandlerV1 { vertexInfoMap.put(guid, new GraphHelper.VertexInfo(entity, vertex)); for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) { - if (! attributeInfo.isOwnedRef()) { + if (!attributeInfo.isOwnedRef()) { continue; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index cd00639..287dacc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -19,7 +19,6 @@ package org.apache.atlas.repository.store.graph.v1; import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContextV1; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TimeBoundary; @@ -82,9 +81,7 @@ import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex; -import static org.apache.atlas.repository.graph.GraphHelper.getListProperty; import static org.apache.atlas.repository.graph.GraphHelper.getMapElementsProperty; -import static org.apache.atlas.repository.graph.GraphHelper.getQualifiedNameForMapKey; import static org.apache.atlas.repository.graph.GraphHelper.getStatus; import static org.apache.atlas.repository.graph.GraphHelper.getTraitLabel; import static org.apache.atlas.repository.graph.GraphHelper.getTraitNames; @@ -92,7 +89,6 @@ import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.graph.GraphHelper.getTypeNames; import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled; import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; -import static org.apache.atlas.repository.graph.GraphHelper.setListProperty; import static org.apache.atlas.repository.graph.GraphHelper.string; import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex; @@ -811,51 +807,45 @@ public class EntityGraphMapper { boolean isReference = isReference(mapType.getValueType()); if (MapUtils.isNotEmpty(newVal)) { - AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); + String propertyName = ctx.getVertexProperty(); - for (Map.Entry<Object, Object> entry : newVal.entrySet()) { - String key = entry.getKey().toString(); - String propertyName = (isReference) ? ctx.getVertexProperty() : getQualifiedNameForMapKey(ctx.getVertexProperty(), GraphHelper.encodePropertyKey(key)); - AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key); - - AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(), - propertyName, mapType.getValueType(), existingEdge); - - //Add/Update/Remove property value - Object newEntry = mapCollectionElementsToVertex(mapCtx, context); + if (isReference) { + for (Map.Entry<Object, Object> entry : newVal.entrySet()) { + String key = entry.getKey().toString(); + AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key); - if (isReference) { - AtlasEdge edge = (AtlasEdge) newEntry; - edge.setProperty(ATTRIBUTE_KEY_PROPERTY_KEY, key); - } else { - ctx.getReferringVertex().setProperty(propertyName, newEntry); - } + AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(), + propertyName, mapType.getValueType(), existingEdge); + // Add/Update/Remove property value + Object newEntry = mapCollectionElementsToVertex(mapCtx, context); - newMap.put(key, newEntry); + if (newEntry instanceof AtlasEdge) { + AtlasEdge edge = (AtlasEdge) newEntry; - // If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute, - // update the inverse reference value. - if (isReference && newEntry instanceof AtlasEdge && inverseRefAttribute != null) { - AtlasEdge newEdge = (AtlasEdge) newEntry; + edge.setProperty(ATTRIBUTE_KEY_PROPERTY_KEY, key); - addInverseReference(context, inverseRefAttribute, newEdge, getRelationshipAttributes(ctx.getValue())); - } - } - } + // If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute, + // update the inverse reference value. + AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); - Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), ctx.getVertexProperty(), currentMap, newMap); + if (inverseRefAttribute != null) { + addInverseReference(context, inverseRefAttribute, edge, getRelationshipAttributes(ctx.getValue())); + } - for (Object newEntry : newMap.values()) { - updateInConsistentOwnedMapVertices(ctx, mapType, newEntry); - } + updateInConsistentOwnedMapVertices(ctx, mapType, newEntry); - // for dereference on way out for primitive map types - if (!isReference) { - Set<String> newKeys = new LinkedHashSet<>(newMap.keySet()); + newMap.put(key, newEntry); + } + } - newKeys.addAll(finalMap.keySet()); + Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), currentMap, newMap); + newMap.putAll(finalMap); + } else { + // primitive type map + ctx.getReferringVertex().setProperty(propertyName, new HashMap<>(newVal)); - setListProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), new ArrayList<>(newKeys)); + newVal.forEach((key, value) -> newMap.put(key.toString(), value)); + } } if (LOG.isDebugEnabled()) { @@ -1084,60 +1074,23 @@ public class EntityGraphMapper { return ret; } - - public static Object getMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) { - if (isReference(elementType)) { - return vertex.getProperty(vertexPropertyName, AtlasEdge.class); - } else if (elementType instanceof AtlasArrayType) { - return vertex.getProperty(vertexPropertyName, List.class); - } else if (elementType instanceof AtlasMapType) { - return vertex.getProperty(vertexPropertyName, Map.class); - } - else { - return vertex.getProperty(vertexPropertyName, String.class).toString(); - } - } - - private static void setMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, Object value) { - if (isReference(elementType)) { - vertex.setPropertyFromElementId(vertexPropertyName, (AtlasEdge)value); - } - else { - vertex.setProperty(vertexPropertyName, value); - } - } - - //Remove unused entries from map - private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, String propertyName, - Map<String, Object> currentMap, Map<String, Object> newMap) - throws AtlasBaseException { - - AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType(); - boolean isReference = isReference(mapType.getValueType()); + //Remove unused entries for reference map + private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, Map<String, Object> currentMap, + Map<String, Object> newMap) throws AtlasBaseException { Map<String, Object> additionalMap = new HashMap<>(); + AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType(); for (String currentKey : currentMap.keySet()) { - boolean shouldDeleteKey = !newMap.containsKey(currentKey); - - if (isReference) { - //Delete the edge reference if its not part of new edges created/updated - AtlasEdge currentEdge = (AtlasEdge) currentMap.get(currentKey); + //Delete the edge reference if its not part of new edges created/updated + AtlasEdge currentEdge = (AtlasEdge) currentMap.get(currentKey); - if (!newMap.values().contains(currentEdge)) { - boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex); + if (!newMap.values().contains(currentEdge)) { + boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex); - if (!deleted) { - additionalMap.put(currentKey, currentEdge); - shouldDeleteKey = false; - } + if (!deleted) { + additionalMap.put(currentKey, currentEdge); } } - - if (!isReference && shouldDeleteKey) { - String propertyNameForKey = getQualifiedNameForMapKey(propertyName, GraphHelper.encodePropertyKey(currentKey)); - - GraphHelper.setProperty(vertex, propertyNameForKey, null); - } } return additionalMap; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index 183a2f6..7ca51e4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -619,7 +619,7 @@ public final class EntityGraphRetriever { ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection); break; case ARRAY: - ret = mapVertexToArray(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute); + ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute); break; case MAP: ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute); @@ -660,7 +660,7 @@ public final class EntityGraphRetriever { } } } else { - ret = getPrimitiveMap(entityVertex, propertyName, mapValueType); + ret = getPrimitiveMap(entityVertex, propertyName); } if (MapUtils.isEmpty(ret)) { @@ -670,12 +670,12 @@ public final class EntityGraphRetriever { return ret; } - private List<Object> mapVertexToArray(AtlasVertex entityVertex, String propertyName, AtlasEntityExtInfo entityExtInfo, + private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException { AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); AtlasType arrayElementType = arrayType.getElementType(); - List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, propertyName, attribute); + List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, attribute); if (CollectionUtils.isEmpty(arrayElements)) { return null; diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java index 2363fca..e216749 100644 --- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java @@ -37,9 +37,9 @@ public class HiveStocksTest extends MigrationBaseAsserts { @Test public void migrateStocks() throws AtlasBaseException, IOException { - final int EXPECTED_TOTAL_COUNT = 188; - final int EXPECTED_DB_COUNT = 1; - final int EXPECTED_TABLE_COUNT = 1; + final int EXPECTED_TOTAL_COUNT = 188; + final int EXPECTED_DB_COUNT = 1; + final int EXPECTED_TABLE_COUNT = 1; final int EXPECTED_COLUMN_COUNT = 7; runFileImporter("stocks_db"); diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java index e494728..922f7e0 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java @@ -30,18 +30,21 @@ import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.commons.lang.time.DateUtils; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE; +import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_MAP; import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR; import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE; import static org.apache.atlas.TestUtilsV2.NAME; @@ -54,18 +57,21 @@ import static org.testng.AssertJUnit.assertEquals; public class AtlasComplexAttributesTest extends AtlasEntityTestBase { private AtlasEntityWithExtInfo complexCollectionAttrEntity; private AtlasEntityWithExtInfo complexCollectionAttrEntityForDelete; + private AtlasEntityWithExtInfo mapAttributesEntity; @BeforeClass public void setUp() throws Exception { super.setUp(); // create typeDefs - AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes() }; + AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes(), + TestUtilsV2.defineTypeWithMapAttributes() }; createTypesDef(testTypesDefs); // create entity complexCollectionAttrEntity = TestUtilsV2.createComplexCollectionAttrEntity(); complexCollectionAttrEntityForDelete = TestUtilsV2.createComplexCollectionAttrEntity(); + mapAttributesEntity = TestUtilsV2.createMapAttrEntity(); } @Test @@ -78,6 +84,116 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { validateEntity(complexCollectionAttrEntity, getEntityFromStore(entityCreated)); } + @Test + public void testPrimitiveMapAttributes() throws Exception { + init(); + + EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(mapAttributesEntity), false); + AtlasEntityHeader entityCreated = response.getFirstCreatedEntityByTypeName(ENTITY_TYPE_MAP); + AtlasEntity entityFromStore = getEntityFromStore(entityCreated); + validateEntity(mapAttributesEntity, entityFromStore); + + // Modify map of primitives + AtlasEntity attrEntity = getEntityFromStore(mapAttributesEntity.getEntity().getGuid()); + + Map<String, String> map1 = new HashMap<String, String>() {{ put("map1Key11", "value11"); + put("map1Key22", "value22"); + put("map1Key33", "value33"); }}; + + Map<String, Integer> map2 = new HashMap<String, Integer>() {{ put("map2Key11", 1100); + put("map2Key22", 2200); + put("map2Key33", 3300); }}; + + Map<String, Boolean> map3 = new HashMap<String, Boolean>() {{ put("map3Key11", true); + put("map3Key22", false); + put("map3Key33", true); }}; + + Map<String, Float> map4 = new HashMap<String, Float>() {{ put("map4Key11", 11.0f); + put("map4Key22", 22.0f); + put("map4Key33", 33.0f); }}; + + Map<String, Date> map5 = new HashMap<String, Date>() {{ put("map5Key11", DateUtils.addHours(new Date(), 1)); + put("map5Key22", DateUtils.addHours(new Date(), 2)); + put("map5Key33", DateUtils.addHours(new Date(), 3)); }}; + + updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5); + + AtlasEntitiesWithExtInfo attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity); + response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false); + AtlasEntityHeader updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP); + AtlasEntity updatedFromStore = getEntityFromStore(updatedAttrEntity); + validateEntity(attrEntitiesInfo, updatedFromStore); + + // Add new entry to map of primitives + map1.put("map1Key44", "value44"); + map2.put("map2Key44", 4400); + map3.put("map3Key44", false); + map4.put("map4Key44", 44.0f); + map5.put("map5Key44", DateUtils.addHours(new Date(), 4)); + + updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5); + + attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity); + response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false); + updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP); + updatedFromStore = getEntityFromStore(updatedAttrEntity); + validateEntity(attrEntitiesInfo, updatedFromStore); + + // Remove an entry from map of primitives + map1.remove("map1Key11"); + map2.remove("map2Key11"); + map3.remove("map3Key11"); + map4.remove("map4Key11"); + map5.remove("map5Key11"); + + updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5); + + attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity); + response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false); + updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP); + updatedFromStore = getEntityFromStore(updatedAttrEntity); + validateEntity(attrEntitiesInfo, updatedFromStore); + + // Edit existing entry to map of primitives + map1.put("map1Key44", "value44-edit"); + map2.put("map2Key44", 5555); + map3.put("map3Key44", true); + map4.put("map4Key44", 55.5f); + map5.put("map5Key44", DateUtils.addHours(new Date(), 5)); + + updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5); + + attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity); + response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false); + updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP); + updatedFromStore = getEntityFromStore(updatedAttrEntity); + validateEntity(attrEntitiesInfo, updatedFromStore); + + // clear primitive map entries + map1.clear(); + map2.clear(); + map3.clear(); + map4.clear(); + map5.clear(); + + updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5); + + attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity); + response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false); + updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP); + updatedFromStore = getEntityFromStore(updatedAttrEntity); + validateEntity(attrEntitiesInfo, updatedFromStore); + } + + private void updateEntityMapAttributes(AtlasEntity attrEntity, Map<String, String> map1, Map<String, Integer> map2, + Map<String, Boolean> map3, Map<String, Float> map4, Map<String, Date> map5) { + attrEntity.setAttribute("mapAttr1", map1); + attrEntity.setAttribute("mapAttr2", map2); + attrEntity.setAttribute("mapAttr3", map3); + attrEntity.setAttribute("mapAttr4", map4); + attrEntity.setAttribute("mapAttr5", map5); + } + @Test(dependsOnMethods = "testCreateComplexAttributeEntity") public void testStructArray() throws Exception { init(); diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java index 56dc5bd..d810a11 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java @@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; +import org.apache.atlas.repository.impexp.ExportService; import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.runner.LocalSolrRunner;