ATLAS-1498: added unit-tests to validate handling of array/map/struct attributes…

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -60,7 +60,7 @@ public enum AtlasErrorCode {
     TYPE_NAME_NOT_FOUND(404, "ATLAS4041E", "Given typename {0} was invalid"),
     TYPE_GUID_NOT_FOUND(404, "ATLAS4042E", "Given type guid {0} was invalid"),
     EMPTY_RESULTS(404, "ATLAS4044E", "No result found for {0}"),
-    INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid"),
+    INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid/not found"),
     INSTANCE_LINEAGE_INVALID_PARAMS(404, "ATLAS4046E", "Invalid lineage query parameters passed {0}: {1}"),
     INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"),
     DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"),

intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java

@@ -146,8 +146,9 @@ public class AtlasStruct implements Serializable {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         AtlasStruct that = (AtlasStruct) o;
+
         return Objects.equals(typeName, that.typeName) &&
-                Objects.equals(attributes, that.attributes);
+            Objects.equals(attributes, that.attributes);
     }
 
     @Override

intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java

@@ -18,9 +18,11 @@
 package org.apache.atlas.model.instance;
 
 
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnore;
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.map.annotate.JsonSerialize;
 
@@ -43,7 +45,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
 @XmlAccessorType(XmlAccessType.PROPERTY)
 public class EntityMutationResponse {
 
-    Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> entitiesMutated = new HashMap<>();
+    Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> entitiesMutated;
 
     public EntityMutationResponse() {
     }
@@ -67,15 +69,39 @@ public class EntityMutationResponse {
         return null;
     }
 
+    @JsonIgnore
+    public AtlasEntityHeader getFirstEntityCreated() {
+        final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
+        if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
+            return entitiesByOperation.get(0);
+        }
+
+        return null;
+    }
+
+    @JsonIgnore
+    public AtlasEntityHeader getFirstEntityUpdated() {
+        final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE);
+        if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
+            return entitiesByOperation.get(0);
+        }
+
+        return null;
+    }
+
     public void addEntity(EntityMutations.EntityOperation op, AtlasEntityHeader header) {
         if (entitiesMutated == null) {
             entitiesMutated = new HashMap<>();
         }
 
-        if (entitiesMutated != null && entitiesMutated.get(op) == null) {
-            entitiesMutated.put(op, new ArrayList<AtlasEntityHeader>());
+        List<AtlasEntityHeader> opEntities = entitiesMutated.get(op);
+
+        if (opEntities == null) {
+            opEntities = new ArrayList<>();
+            entitiesMutated.put(op, opEntities);
         }
-        entitiesMutated.get(op).add(header);
+
+        opEntities.add(header);
     }
 
 
@@ -84,24 +110,7 @@ public class EntityMutationResponse {
             sb = new StringBuilder();
         }
 
-        if (MapUtils.isNotEmpty(entitiesMutated)) {
-            int i = 0;
-            for (Map.Entry<EntityMutations.EntityOperation, List<AtlasEntityHeader>> e : entitiesMutated.entrySet()) {
-                if (i > 0) {
-                    sb.append(",");
-                }
-                sb.append(e.getKey()).append(":");
-                if (CollectionUtils.isNotEmpty(e.getValue())) {
-                    for (int j = 0; i < e.getValue().size(); j++) {
-                        if (j > 0) {
-                            sb.append(",");
-                        }
-                        e.getValue().get(i).toString(sb);
-                    }
-                }
-                i++;
-            }
-        }
+        AtlasBaseTypeDef.dumpObjects(entitiesMutated, sb);
 
         return sb;
     }

intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java

@@ -477,6 +477,7 @@ public class AtlasBuiltInTypes {
      */
     public static class AtlasStringType extends AtlasType {
         private static final String DEFAULT_VALUE = "";
+        private static final String OPTIONAL_DEFAULT_VALUE = null;
 
         public AtlasStringType() {
             super(AtlasBaseTypeDef.ATLAS_TYPE_STRING, TypeCategory.PRIMITIVE);
@@ -488,6 +489,11 @@ public class AtlasBuiltInTypes {
         }
 
         @Override
+        public Object createOptionalDefaultValue() {
+            return OPTIONAL_DEFAULT_VALUE;
+        }
+
+        @Override
         public boolean isValidValue(Object obj) {
             return true;
         }

intg/src/main/java/org/apache/atlas/type/AtlasMapType.java

@@ -95,7 +95,11 @@ public class AtlasMapType extends AtlasType {
     public Map<Object, Object>  createDefaultValue() {
         Map<Object, Object> ret = new HashMap<>();
 
-        ret.put(keyType.createDefaultValue(), valueType.createDefaultValue());
+        Object key = keyType.createDefaultValue();
+
+        if ( key != null) {
+            ret.put(key, valueType.createDefaultValue());
+        }
 
         return ret;
     }

intg/src/main/java/org/apache/atlas/type/AtlasStructType.java

@@ -328,6 +328,7 @@ public class AtlasStructType extends AtlasType {
 
             if (attribute != null) {
                 AtlasType dataType = attribute.getAttributeType();
+
                 ret = dataType.createDefaultValue();
             }
         }

intg/src/main/java/org/apache/atlas/type/AtlasType.java

@@ -33,7 +33,7 @@ import java.util.List;
 public abstract class AtlasType {
 
     private static final Gson GSON =
-            new GsonBuilder().setDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR).create();
+            new GsonBuilder().serializeNulls().setDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR).create();
 
     private final String       typeName;
     private final TypeCategory typeCategory;
@@ -59,6 +59,10 @@ public abstract class AtlasType {
 
     public abstract Object createDefaultValue();
 
+    public Object createOptionalDefaultValue() {
+        return createDefaultValue();
+    }
+
     public abstract boolean isValidValue(Object obj);
 
     public abstract Object getNormalizedValue(Object obj);

intg/src/test/java/org/apache/atlas/TestUtilsV2.java

@@ -71,7 +71,7 @@ public final class TestUtilsV2 {
                 new AtlasEnumDef("OrgLevel", "OrgLevel"+_description, "1.0",
                         Arrays.asList(
                                 new AtlasEnumElementDef("L1", "Element"+_description, 1),
-                                new AtlasEnumElementDef("L1", "Element"+_description, 2)
+                                new AtlasEnumElementDef("L2", "Element"+_description, 2)
                         ));
 
         AtlasStructDef addressDetails =
@@ -149,7 +149,7 @@ public final class TestUtilsV2 {
                 new AtlasEnumDef("OrgLevel", "OrgLevel"+_description, "1.0",
                         Arrays.asList(
                                 new AtlasEnumElementDef("L1", "Element"+ _description, 1),
-                                new AtlasEnumElementDef("L1", "Element"+ _description, 2)
+                                new AtlasEnumElementDef("L2", "Element"+ _description, 2)
                         ));
 
         AtlasStructDef addressDetails =
@@ -443,6 +443,7 @@ public final class TestUtilsV2 {
         AtlasEntityDef databaseTypeDefinition =
                 AtlasTypeUtil.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME),
                         AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"),
+                        AtlasTypeUtil.createOptionalAttrDef("isReplicated", "boolean"),
                         AtlasTypeUtil.createOptionalAttrDef("created", "string"),
                         AtlasTypeUtil.createRequiredAttrDef("description", "string"));
 
@@ -543,7 +544,7 @@ public final class TestUtilsV2 {
         AtlasEntityDef tableTypeDefinition =
                 AtlasTypeUtil.createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME),
                         AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
-                        AtlasTypeUtil.createRequiredAttrDef("description", "string"),
+                        AtlasTypeUtil.createOptionalAttrDef("description", "string"),
                         AtlasTypeUtil.createRequiredAttrDef("type", "string"),
                         AtlasTypeUtil.createOptionalAttrDef("created", "date"),
                         // enum
@@ -582,6 +583,11 @@ public final class TestUtilsV2 {
                                 AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
                                 false, false,
                                 Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()),
+
+                      // new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
+                     //add(new AtlasStructDef.AtlasConstraintDef(
+                       // AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF, new HashMap<String, Object>()));
+                       //}}),
                         //map of structs
                         new AtlasAttributeDef("partitionsMap",
                                 String.format("map<%s,%s>", "string", "partition_struct_type"),
@@ -646,7 +652,7 @@ public final class TestUtilsV2 {
         entity.setAttribute("description", "random table");
         entity.setAttribute("type", "type");
         entity.setAttribute("tableType", "MANAGED");
-        entity.setAttribute("database", dbId);
+        entity.setAttribute("database", new AtlasObjectId(DATABASE_TYPE, dbId));
         entity.setAttribute("created", new Date());
 
         Map<String, Object> partAttributes = new HashMap<String, Object>() {{

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1498 added unit-tests to validate handling of array/map/struct attributes in entity create/update (sumasai via mneethiraj)
 ATLAS-1114 Performance improvements for create/update entity (jnhagelb)
 ATLAS-1403 Perf and stability improvements to DSL search and lineage query execution (sarath.kum4r@gmail.com via mneethiraj)
 ATLAS-1425 Integrate Discovery/Search API in Atlas UI (kevalbhatt via mneethiraj)

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java

@@ -82,7 +82,7 @@ public class ArrayVertexMapper implements InstanceGraphMapper<List> {
 
                 Optional<AtlasEdge> existingEdge = getEdgeAt(currentElements, index, arrType.getElementType());
 
-                GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getAttribute(),
+                GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(),
                     arrType.getElementType(), newElements.get(index))
                     .referringVertex(ctx.getReferringVertex())
                     .edge(existingEdge)

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java

@@ -18,8 +18,8 @@
 package org.apache.atlas.repository.store.graph.v1;
 
 import atlas.shaded.hbase.guava.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
 import com.google.inject.Provider;
+import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.instance.AtlasEntity;
@@ -112,7 +112,11 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
 
     protected void discover(final List<AtlasEntity> entities) throws AtlasBaseException {
         for (AtlasEntity entity : entities) {
-            AtlasType type = typeRegistry.getType(entity.getTypeName());
+            AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName());
+
+            if (type == null) {
+                throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
+            }
 
             discoveredEntities.addRootEntity(entity);
             walkEntityGraph(type, entity);
@@ -120,27 +124,24 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
     }
 
     private void visitReference(AtlasEntityType type, Object entity, boolean isManagedEntity) throws AtlasBaseException {
-        if ( entity != null) {
-            if ( entity instanceof String ) {
-                String guid = (String) entity;
-                discoveredEntities.addUnResolvedIdReference(type, guid);
-            } else if ( entity instanceof AtlasObjectId ) {
+        if (entity != null) {
+            if (entity instanceof AtlasObjectId) {
                 final String guid = ((AtlasObjectId) entity).getGuid();
                 discoveredEntities.addUnResolvedIdReference(type, guid);
-            } else if ( entity instanceof  AtlasEntity ) {
-                AtlasEntity entityObj = ( AtlasEntity ) entity;
-                if (!processedIds.contains(entityObj.getGuid())) {
-                    processedIds.add(entityObj.getGuid());
+            } else if (entity instanceof AtlasEntity) {
+                AtlasEntity entityObj = (AtlasEntity) entity;
+                if (isManagedEntity) {
+                    if (!processedIds.contains(entityObj.getGuid())) {
+                        processedIds.add(entityObj.getGuid());
 
-                    if ( isManagedEntity ) {
                         discoveredEntities.addRootEntity(entityObj);
                         visitStruct(type, entityObj);
-                    } else if ( entity instanceof AtlasObjectId) {
-                        discoveredEntities.addUnResolvedIdReference(type, ((AtlasObjectId) entity).getGuid());
-                    } else {
-                        discoveredEntities.addUnResolvedEntityReference(entityObj);
                     }
+                } else {
+                    discoveredEntities.addUnResolvedEntityReference(entityObj);
                 }
+            } else {
+                throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, "Invalid object type " + entity.getClass());
             }
         }
     }
@@ -161,12 +162,14 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
             } else if (attrType.getTypeCategory() == TypeCategory.STRUCT) {
                 visitStruct(attrType, val);
             } else if (attrType.getTypeCategory() == TypeCategory.ENTITY) {
-                if ( val instanceof AtlasObjectId || val instanceof String) {
+                if ( val instanceof AtlasObjectId) {
                     visitReference((AtlasEntityType) attrType,  val, false);
                 } else if ( val instanceof AtlasEntity ) {
                     //TODO - Change this to foreign key checks after changes in the model
                    if ( parentType.isMappedFromRefAttribute(attrDef.getName())) {
                        visitReference((AtlasEntityType) attrType,  val, true);
+                   } else {
+                       visitReference((AtlasEntityType) attrType,  val, false);
                    }
                 }
             }
@@ -179,10 +182,9 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
         }
 
         if (val != null) {
-            Iterator<Map.Entry> it = null;
+
             if (Map.class.isAssignableFrom(val.getClass())) {
-                it = ((Map) val).entrySet().iterator();
-                ImmutableMap.Builder b = ImmutableMap.builder();
+                Iterator<Map.Entry> it = ((Map) val).entrySet().iterator();
                 while (it.hasNext()) {
                     Map.Entry e = it.next();
                     visitAttribute(parentType, keyType, attrDef, e.getKey());
@@ -224,11 +226,10 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
 
         AtlasStructType structType = (AtlasStructType) type;
 
-        for (AtlasStructDef.AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) {
-            String attrName = attributeDef.getName();
-            AtlasType attrType = structType.getAttributeType(attrName);
-            Object attrVal = ((AtlasStruct) val).getAttribute(attrName);
-            visitAttribute(structType, attrType, attributeDef, attrVal);
+        for (AtlasStructType.AtlasAttribute attribute : structType.getAllAttributes().values()) {
+            AtlasType attrType = attribute.getAttributeType();
+            Object attrVal = ((AtlasStruct) val).getAttribute(attribute.getAttributeDef().getName());
+            visitAttribute(structType, attrType, attribute.getAttributeDef(), attrVal);
         }
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java

@@ -70,43 +70,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
         return createOrUpdate(new ArrayList<AtlasEntity>() {{ add(entity); }});
     }
 
-    public EntityMutationContext preCreateOrUpdate(final List<AtlasEntity> atlasEntities) throws AtlasBaseException {
-
-        EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities);
-        EntityMutationContext context = new EntityMutationContext(discoveredEntities);
-        for (AtlasEntity entity : discoveredEntities.getRootEntities()) {
-
-            AtlasVertex vertex = null;
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity);
-            }
-
-            AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName());
-
-            if ( discoveredEntities.isResolved(entity.getGuid()) ) {
-                vertex = discoveredEntities.getResolvedReference(entity.getGuid());
-                context.addUpdated(entity, entityType, vertex);
-
-                String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
-                RequestContextV1.get().recordEntityUpdate(guid);
-            } else {
-                //Create vertices which do not exist in the repository
-                vertex = graphMapper.createVertexTemplate(entity, entityType);
-                context.addCreated(entity, entityType, vertex);
-                discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex);
-
-                String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
-                RequestContextV1.get().recordEntityCreate(guid);
-            }
-
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex);
-            }
-        }
-
-        return context;
-    }
-
     @Override
     public EntityMutationResponse updateById(final String guid, final AtlasEntity entity) {
         return null;
@@ -205,15 +168,56 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
         return null;
     }
 
+    private EntityMutationContext preCreateOrUpdate(final List<AtlasEntity> atlasEntities) throws AtlasBaseException {
+
+        EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities);
+        EntityMutationContext context = new EntityMutationContext(discoveredEntities);
+        for (AtlasEntity entity : discoveredEntities.getRootEntities()) {
+
+            AtlasVertex vertex = null;
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity);
+            }
+
+            AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
+
+            if ( entityType == null) {
+                throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
+            }
+
+            if ( discoveredEntities.isResolved(entity.getGuid()) ) {
+                vertex = discoveredEntities.getResolvedReference(entity.getGuid());
+                context.addUpdated(entity, entityType, vertex);
+
+                String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
+                RequestContextV1.get().recordEntityUpdate(guid);
+            } else {
+                //Create vertices which do not exist in the repository
+                vertex = graphMapper.createVertexTemplate(entity, entityType);
+                context.addCreated(entity, entityType, vertex);
+                discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex);
+
+                String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
+                RequestContextV1.get().recordEntityCreate(guid);
+            }
+
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex);
+            }
+        }
+
+        return context;
+    }
+
     private List<AtlasEntity> validateAndNormalize(final List<AtlasEntity> entities) throws AtlasBaseException {
 
         List<AtlasEntity> normalizedEntities = new ArrayList<>();
         List<String> messages = new ArrayList<>();
 
         for (AtlasEntity entity : entities) {
-            AtlasType type = typeRegistry.getType(entity.getTypeName());
-            if (type.getTypeCategory() != TypeCategory.ENTITY) {
-                throw new AtlasBaseException(AtlasErrorCode.TYPE_MATCH_FAILED, type.getTypeCategory().name(), TypeCategory.ENTITY.name());
+            AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName());
+            if (type == null) {
+                throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
             }
 
             type.validateValue(entity, entity.getTypeName(), messages);
@@ -232,16 +236,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
         return normalizedEntities;
     }
 
-    @VisibleForTesting
-    EntityGraphDiscovery getGraphDiscoverer() {
-        return graphDiscoverer;
-    }
-
-    @VisibleForTesting
-    void setGraphDiscoverer(EntityGraphDiscovery discoverer) {
-        this.graphDiscoverer = discoverer;
-    }
-
     public void cleanUp() throws AtlasBaseException {
         this.graphDiscoverer.cleanUp();
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java

@@ -94,8 +94,12 @@ public class AtlasGraphUtilsV1 {
         return PROPERTY_PREFIX + "edge." + fromNode + "." + toNode;
     }
 
+    public static String getEdgeLabel(String property) {
+        return GraphHelper.EDGE_LABEL_PREFIX + property;
+    }
+
     public static String getAttributeEdgeLabel(AtlasStructType fromType, String attributeName) throws AtlasBaseException {
-        return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedAttributePropertyKey(fromType, attributeName);
+        return getEdgeLabel(getQualifiedAttributePropertyKey(fromType, attributeName));
     }
 
     public static String getQualifiedAttributePropertyKey(AtlasStructType fromType, String attributeName) throws AtlasBaseException {

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java

@@ -25,6 +25,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
 import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.GraphHelper;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.AtlasStructDefStore;
 import org.apache.atlas.repository.util.FilterUtil;
@@ -462,7 +463,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
         if (CollectionUtils.isNotEmpty(attrNames)) {
             for (String attrName : attrNames) {
                 String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(ret, attrName);
-                String attribJson  = vertex.getProperty(propertyKey, String.class);
+                String attribJson  = vertex.getProperty(GraphHelper.encodePropertyKey(propertyKey), String.class);
 
                 attributeDefs.add(toAttributeDefFromJson(structDef, AtlasType.fromJson(attribJson, Map.class),
                                   typeDefStore));

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java

@@ -18,6 +18,7 @@
 package org.apache.atlas.repository.store.graph.v1;
 
 
+import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.RequestContextV1;
 import org.apache.atlas.exception.AtlasBaseException;
@@ -129,7 +130,12 @@ public abstract class DeleteHandlerV1 {
                 continue;
             }
             result.add(new GraphHelper.VertexInfo(guid, vertex, typeName));
-            AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(typeName);
+            AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
+
+            if (entityType == null) {
+                throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName);
+            }
+
             for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) {
                 if (!entityType.isMappedFromRefAttribute(attributeInfo.getAttributeDef().getName())) {
                     continue;

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java

@@ -22,6 +22,7 @@ import com.google.inject.Inject;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.RequestContextV1;
 import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.AtlasObjectId;
@@ -36,9 +37,11 @@ import org.apache.atlas.repository.graphdb.AtlasEdge;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.AtlasType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Map;
 import java.util.UUID;
 
 public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
@@ -86,13 +89,12 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
         AtlasEdge result = null;
 
         String guid = getId(ctx.getValue());
-
         AtlasVertex entityVertex = context.getDiscoveryContext().getResolvedReference(guid);
-        String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(ctx.getParentType(), ctx.getAttributeDef().getName());
         if ( ctx.getCurrentEdge().isPresent() ) {
             updateEdge(ctx.getAttributeDef(), ctx.getValue(), ctx.getCurrentEdge().get(), entityVertex);
             result = ctx.getCurrentEdge().get();
-        } else {
+        } else if (ctx.getValue() != null) {
+            String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey());
             try {
                 result = graphHelper.getOrCreateEdge(ctx.getReferringVertex(), entityVertex, edgeLabel);
             } catch (RepositoryException e) {
@@ -112,7 +114,7 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
         LOG.debug("Updating entity reference {} for reference attribute {}",  attributeDef.getName());
         // Update edge if it exists
 
-        AtlasVertex currentVertex = currentEdge.getOutVertex();
+        AtlasVertex currentVertex = currentEdge.getInVertex();
         String currentEntityId = AtlasGraphUtilsV1.getIdFromVertex(currentVertex);
         String newEntityId = getId(value);
         AtlasEdge newEdge = currentEdge;
@@ -140,17 +142,17 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
         if (ctx.getCreatedEntities() != null) {
             for (AtlasEntity createdEntity : ctx.getCreatedEntities()) {
                 AtlasVertex vertex = ctx.getVertex(createdEntity);
-                structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(createdEntity), createdEntity, vertex);
-                resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, vertex));
+                structVertexMapper.mapAttributestoVertex(EntityMutations.EntityOperation.CREATE, ctx.getType(createdEntity), createdEntity, vertex);
+                resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, ctx.getType(createdEntity), vertex));
             }
         }
 
         if (ctx.getUpdatedEntities() != null) {
             for (AtlasEntity updated : ctx.getUpdatedEntities()) {
                 AtlasVertex vertex = ctx.getVertex(updated);
-                structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(updated), updated, vertex);
+                structVertexMapper.mapAttributestoVertex(EntityMutations.EntityOperation.UPDATE, ctx.getType(updated), updated, vertex);
 
-                resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, vertex));
+                resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, ctx.getType(updated), vertex));
             }
         }
 
@@ -165,13 +167,30 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
             } else if (value instanceof AtlasEntity) {
                 return ((AtlasEntity) value).getGuid();
             }
+
+            throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, (String) value);
         }
-        throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, (String) value);
+
+        return null;
     }
 
-    private AtlasEntityHeader constructHeader(AtlasEntity entity, AtlasVertex vertex) {
+    private AtlasEntityHeader constructHeader(AtlasEntity entity, final AtlasEntityType type, AtlasVertex vertex) {
         //TODO - enhance to return only selective attributes
-        return new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes());
+        AtlasEntityHeader header = new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes());
+        final Map<String, AtlasStructType.AtlasAttribute> allAttributes = type.getAllAttributes();
+        for (String attribute : allAttributes.keySet()) {
+            AtlasType attributeType = allAttributes.get(attribute).getAttributeType();
+            AtlasStructDef.AtlasAttributeDef attributeDef = allAttributes.get(attribute).getAttributeDef();
+            if ( header.getAttribute(attribute) == null && (TypeCategory.PRIMITIVE == attributeType.getTypeCategory())) {
+
+                if ( attributeDef.getIsOptional()) {
+                    header.setAttribute(attribute, attributeType.createOptionalDefaultValue());
+                } else {
+                    header.setAttribute(attribute, attributeType.createDefaultValue());
+                }
+            }
+        }
+        return header;
     }
 
     public EntityMutationContext getContext() {
@@ -180,6 +199,11 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
 
     public AtlasEntityType getInstanceType(Object val) throws AtlasBaseException {
         String guid = getId(val);
-        return (AtlasEntityType) getContext().getType(guid);
+
+        if ( guid != null) {
+            return (AtlasEntityType) getContext().getType(guid);
+        }
+
+        return null;
     }
 }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java

@@ -21,6 +21,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
 
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
+import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasType;
 
 import java.util.ArrayList;
@@ -35,20 +36,20 @@ public class EntityMutationContext {
     private List<AtlasEntity> entitiesUpdated  = new ArrayList<>();
 
     private EntityGraphDiscoveryContext context;
-    private Map<String, AtlasType> entityVsType = new HashMap<>();
+    private Map<String, AtlasEntityType> entityVsType = new HashMap<>();
     private Map<String, AtlasVertex> entityVsVertex = new HashMap<>();
 
     public EntityMutationContext(final EntityGraphDiscoveryContext context) {
         this.context = context;
     }
 
-    public void addCreated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) {
+    public void addCreated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
         entitiesCreated.add(entity);
         entityVsVertex.put(entity.getGuid(), atlasVertex);
         entityVsType.put(entity.getGuid(), type);
     }
 
-    public void addUpdated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) {
+    public void addUpdated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
         entitiesUpdated.add(entity);
         entityVsVertex.put(entity.getGuid(), atlasVertex);
         entityVsType.put(entity.getGuid(), type);
@@ -62,7 +63,7 @@ public class EntityMutationContext {
         return entitiesUpdated;
     }
 
-    public AtlasType getType(AtlasEntity entity) {
+    public AtlasEntityType getType(AtlasEntity entity) {
         return entityVsType.get(entity.getGuid());
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java

@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v1;
 
 
 import com.google.common.base.Optional;
+import org.apache.atlas.model.instance.EntityMutations;
 import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.repository.graphdb.AtlasEdge;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
@@ -30,6 +31,7 @@ import java.util.Objects;
 public class GraphMutationContext {
 
 
+    private EntityMutations.EntityOperation op;
     /**
      * Atlas Attribute
      */
@@ -65,6 +67,7 @@ public class GraphMutationContext {
 
 
     private GraphMutationContext(final Builder builder) {
+        this.op = builder.op;
         this.attribute = builder.attribute;
         this.currentElementType = builder.elementType;
         this.existingEdge = builder.currentEdge;
@@ -79,7 +82,7 @@ public class GraphMutationContext {
 
     @Override
     public int hashCode() {
-        return Objects.hash(attribute, value, referringVertex, vertexPropertyKey, existingEdge);
+        return Objects.hash(op, attribute, value, referringVertex, vertexPropertyKey, existingEdge);
     }
 
     @Override
@@ -96,13 +99,16 @@ public class GraphMutationContext {
                  && Objects.equals(value, rhs.getValue())
                  && Objects.equals(referringVertex, rhs.getReferringVertex())
                  && Objects.equals(vertexPropertyKey, rhs.getReferringVertex())
-                 && Objects.equals(existingEdge, rhs.getCurrentEdge());
+                 && Objects.equals(existingEdge, rhs.getCurrentEdge())
+                 && Objects.equals(op, rhs.getOp());
         }
     }
 
 
     public static final class Builder {
 
+        private final EntityMutations.EntityOperation op;
+
         private final AtlasStructType.AtlasAttribute attribute;
 
         private final AtlasType elementType;
@@ -116,16 +122,15 @@ public class GraphMutationContext {
         private  String vertexPropertyKey;
 
 
-        public Builder(AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) {
+        public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) {
+            this.op = op;
             this.attribute = attribute;
             this.elementType = currentElementType;
             this.currentValue = currentValue;
         }
 
-        public Builder(AtlasStructType.AtlasAttribute attribute, Object currentValue) {
-            this.attribute = attribute;
-            this.elementType = null;
-            this.currentValue = currentValue;
+        public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, Object currentValue) {
+           this(op, attribute, null, currentValue);
         }
 
         Builder referringVertex(AtlasVertex referringVertex) {
@@ -192,4 +197,12 @@ public class GraphMutationContext {
     public AtlasStructType.AtlasAttribute getAttribute() {
         return attribute;
     }
+
+    public EntityMutations.EntityOperation getOp() {
+        return op;
+    }
+
+    public void setExistingEdge(final Optional<AtlasEdge> existingEdge) {
+        this.existingEdge = existingEdge;
+    }
 }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java

@@ -105,7 +105,7 @@ public class IDBasedEntityResolver implements EntityResolver {
         if ( vertex != null ) {
             return Optional.of(vertex);
         } else {
-            throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, "Could not find an entity with the specified guid " + typeIdPair.getGuid() + " in Atlas respository");
+            throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, typeIdPair.getGuid());
         }
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java

@@ -97,13 +97,16 @@ public class MapVertexMapper implements InstanceGraphMapper<Map> {
                     String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexPropertyKey(), keyStr);
                     Optional<AtlasEdge> existingEdge = getEdgeIfExists(mapType, currentMap, keyStr);
 
-                    GraphMutationContext mapCtx =  new GraphMutationContext.Builder(ctx.getAttribute(), mapType.getValueType(), entry.getValue())
+                    GraphMutationContext mapCtx =  new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(), mapType.getValueType(), entry.getValue())
                         .referringVertex(ctx.getReferringVertex())
                         .edge(existingEdge)
                         .vertexProperty(propertyNameForKey).build();
 
 
+                    //Add/Update/Remove property value
                     Object newEntry = structVertexMapper.mapCollectionElementsToVertex(mapCtx);
+                    MapVertexMapper.setMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyNameForKey, newEntry);
+
                     newMap.put(keyStr, newEntry);
                 }
             }
@@ -192,7 +195,9 @@ public class MapVertexMapper implements InstanceGraphMapper<Map> {
     private Optional<AtlasEdge> getEdgeIfExists(AtlasMapType mapType, Map<String, Object> currentMap, String keyStr) {
         Optional<AtlasEdge> existingEdge = Optional.absent();
         if ( AtlasGraphUtilsV1.isReference(mapType.getValueType()) ) {
-            existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr));
+            if ( currentMap.get(keyStr) != null) {
+                existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr));
+            }
         }
         
         return existingEdge;

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java

@@ -20,6 +20,7 @@ package org.apache.atlas.repository.store.graph.v1;
 import com.google.common.base.Optional;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasObjectId;
 import org.apache.atlas.model.typedef.AtlasStructDef;
@@ -98,7 +99,12 @@ public class UniqAttrBasedEntityResolver implements EntityResolver {
     }
 
     Optional<AtlasVertex> resolveByUniqueAttribute(AtlasEntity entity) throws AtlasBaseException {
-        AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName());
+        AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
+
+        if (entityType == null) {
+            throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
+        }
+
         for (AtlasStructType.AtlasAttribute attr : entityType.getAllAttributes().values()) {
             if (attr.getAttributeDef().getIsUnique()) {
                 Object attrVal = entity.getAttribute(attr.getAttributeDef().getName());

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEntityFormatConverter.java

@@ -145,8 +145,8 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
 
                 ret = new Referenceable(entity.getGuid(), entity.getTypeName(),
                                         fromV2ToV1(entityType, entity.getAttributes()));
-            } else if (v2Obj instanceof String) { // transient-id
-                ret = new Referenceable((String) v2Obj, type.getTypeName(), null);
+            } else if (v2Obj instanceof AtlasObjectId) { // transient-id
+                ret = new Referenceable(((AtlasObjectId) v2Obj).getGuid(), type.getTypeName(), null);
             } else {
                 throw new AtlasBaseException(AtlasErrorCode.UNEXPECTED_TYPE, "Map or AtlasEntity or String",
                                              v2Obj.getClass().getCanonicalName());

webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java

@@ -31,6 +31,8 @@ import org.apache.atlas.model.instance.AtlasEntityWithAssociations;
 import org.apache.atlas.model.instance.EntityMutationResponse;
 import org.apache.atlas.model.instance.EntityMutations;
 import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.type.AtlasClassificationType;
+import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.IReferenceableInstance;
@@ -106,7 +108,10 @@ public class AtlasInstanceRestAdapters {
 
     public AtlasClassification getClassification(IStruct classification) throws AtlasBaseException {
         AtlasFormatConverter converter          = instanceFormatters.getConverter(TypeCategory.CLASSIFICATION);
-        AtlasType            classificationType = typeRegistry.getType(classification.getTypeName());
+        AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(classification.getTypeName());
+        if (classificationType == null) {
+            throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.CLASSIFICATION.name(), classification.getTypeName());
+        }
         AtlasClassification  ret                = (AtlasClassification)converter.fromV1ToV2(classification, classificationType);
 
         return ret;
@@ -114,7 +119,11 @@ public class AtlasInstanceRestAdapters {
 
     public AtlasEntityWithAssociations getAtlasEntity(IReferenceableInstance referenceable) throws AtlasBaseException {
         AtlasFormatConverter converter  = instanceFormatters.getConverter(TypeCategory.ENTITY);
-        AtlasType            entityType = typeRegistry.getType(referenceable.getTypeName());
+        AtlasEntityType entityType = typeRegistry.getEntityTypeByName(referenceable.getTypeName());
+        if (entityType == null) {
+            throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), referenceable.getTypeName());
+        }
+
         AtlasEntityWithAssociations ret = (AtlasEntityWithAssociations)converter.fromV1ToV2(referenceable, entityType);
 
         return ret;

webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java

@@ -190,12 +190,12 @@ public class TestEntitiesREST {
 
         if ( retrievedDBEntity != null) {
             LOG.info("verifying entity of type {} ", dbEntity.getTypeName());
-            verifyAttributes(dbEntity.getAttributes(), retrievedDBEntity.getAttributes());
+            verifyAttributes(retrievedDBEntity.getAttributes(), dbEntity.getAttributes());
         }
 
         if ( retrievedColumnEntity != null) {
             LOG.info("verifying entity of type {} ", columns.get(0).getTypeName());
-            verifyAttributes(columns.get(0).getAttributes(), retrievedColumnEntity.getAttributes());
+            verifyAttributes(retrievedColumnEntity.getAttributes(), columns.get(0).getAttributes());
         }
 
         if ( retrievedTableEntity != null) {
@@ -216,10 +216,13 @@ public class TestEntitiesREST {
         }
     }
 
-    public static void verifyAttributes(Map<String, Object> sourceAttrs, Map<String, Object> targetAttributes) throws Exception {
-        for (String name : sourceAttrs.keySet() ) {
+    public static void verifyAttributes(Map<String, Object> actual, Map<String, Object> expected) throws Exception {
+        for (String name : actual.keySet() ) {
             LOG.info("verifying attribute {} ", name);
-            Assert.assertEquals(targetAttributes.get(name), sourceAttrs.get(name));
+
+            if ( expected.get(name) != null) {
+                Assert.assertEquals(actual.get(name), expected.get(name));
+            }
         }
     }