ATLAS-1551 auto update of reverse references in V1 API

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1551 auto update of reverse references in V1 API (dkantor)
 ATLAS-1565 Create EntityREST endpoints for delete operations (sarathkumarsubramanian via svimal2106)
 ATLAS-1547 Added tests for hard delete (mneethiraj)
 ATLAS-1546 (ATLAS-1546.3.patch)Hive hook should choose appropriate JAAS config if host uses kerberos ticket-cache (nixonrodrigues via kevalbhat)

repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java

@@ -49,6 +49,7 @@ import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
 import org.apache.atlas.typesystem.types.AttributeInfo;
 import org.apache.atlas.typesystem.types.ClassType;
 import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.atlas.typesystem.types.EnumValue;
 import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.Multiplicity;
@@ -56,6 +57,7 @@ import org.apache.atlas.typesystem.types.ObjectGraphWalker;
 import org.apache.atlas.typesystem.types.TraitType;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.apache.atlas.util.AtlasRepositoryConfiguration;
 import org.apache.atlas.utils.MD5Utils;
 import org.slf4j.Logger;
@@ -249,6 +251,9 @@ public final class TypedInstanceToGraphMapper {
                     deleteHandler.deleteEdgeReference(currentEdge, attributeInfo.dataType().getTypeCategory(),
                             attributeInfo.isComposite, true);
                 }
+                if (attributeInfo.reverseAttributeName != null && newEdge != null) {
+                    addReverseReference(instanceVertex, attributeInfo.reverseAttributeName, newEdge);
+                }
                 break;
 
             case TRAIT:
@@ -421,24 +426,35 @@ public final class TypedInstanceToGraphMapper {
         List<Object> newElementsCreated = new ArrayList<>();
 
         if (!newAttributeEmpty) {
-            if (newElements != null && !newElements.isEmpty()) {
-                int index = 0;
-                for (; index < newElements.size(); index++) {
-                    Object currentElement = (currentElements != null && index < currentElements.size()) ?
-                            currentElements.get(index) : null;
-                    if (LOG.isDebugEnabled()) {
-                        LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index,
-                                currentElement, newElements.get(index));
-                    }
-
-                    Object newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
-                            newElements.get(index), currentElement, propertyName, operation);
-                    newElementsCreated.add(newEntry);
+            int index = 0;
+            for (; index < newElements.size(); index++) {
+                Object currentElement = (currentElements != null && index < currentElements.size()) ?
+                        currentElements.get(index) : null;
+                if (LOG.isDebugEnabled()) {
+                    LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index,
+                            currentElement, newElements.get(index));
                 }
+
+                Object newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
+                        newElements.get(index), currentElement, propertyName, operation);
+                newElementsCreated.add(newEntry);
             }
         }
 
         if(GraphHelper.isReference(elementType)) {
+            if (attributeInfo.reverseAttributeName != null && newElementsCreated.size() > 0) {
+                // Set/add the new reference value(s) on the reverse reference.
+                for (Object newElement : newElementsCreated) {
+                    if ((newElement instanceof AtlasEdge)) {
+                        AtlasEdge newEdge = (AtlasEdge) newElement;
+                        addReverseReference(instanceVertex, attributeInfo.reverseAttributeName, newEdge);
+                    }
+                    else {
+                        throw new AtlasException("Invalid array element type " + newElement.getClass().getName() + " - expected " + AtlasEdge.class.getName() +
+                            " for reference " + GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo) + " on vertex " + GraphHelper.getVertexDetails(instanceVertex));
+                    }
+                }
+            }
 
             List<AtlasEdge> additionalEdges = removeUnusedEntries(instanceVertex, propertyName, (List)currentElements,
                     (List)newElementsCreated, elementType, attributeInfo);
@@ -866,4 +882,67 @@ public final class TypedInstanceToGraphMapper {
         }
         return new GuidMapping(mapping);
     }
+
+
+    private <V,E> void addReverseReference(AtlasVertex<V,E> vertex, String reverseAttributeName, AtlasEdge<V,E> edge)
+        throws AtlasException {
+
+        String typeName = GraphHelper.getTypeName(vertex);
+        Id id = GraphHelper.getIdFromVertex(typeName, vertex);
+
+        AtlasVertex<V, E> reverseVertex = edge.getInVertex();
+        String reverseTypeName = GraphHelper.getTypeName(reverseVertex);
+        Id reverseId = GraphHelper.getIdFromVertex(reverseTypeName, reverseVertex);
+        IDataType reverseType = typeSystem.getDataType(IDataType.class, reverseTypeName);
+        AttributeInfo reverseAttrInfo = TypesUtil.getFieldMapping(reverseType).fields.get(reverseAttributeName);
+        if (reverseAttrInfo.dataType().getTypeCategory() == TypeCategory.MAP) {
+            // If the reverse reference is a map, what would be used as the key?
+            // Not supporting automatic update of reverse map references.
+            LOG.debug("Automatic update of reverse map reference is not supported - reference = {}",
+                GraphHelper.getQualifiedFieldName(reverseType, reverseAttributeName));
+            return;
+        }
+
+        String propertyName = GraphHelper.getQualifiedFieldName(reverseType, reverseAttributeName);
+        String reverseEdgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
+        AtlasEdge<V, E> reverseEdge = graphHelper.getEdgeForLabel(reverseVertex, reverseEdgeLabel);
+
+        AtlasEdge<V, E> newEdge = null;
+        if (reverseEdge != null) {
+            newEdge = updateClassEdge(reverseVertex, reverseEdge, id, vertex, reverseAttrInfo, reverseEdgeLabel);
+        }
+        else {
+            newEdge = addClassEdge(reverseVertex, vertex, reverseEdgeLabel);
+        }
+
+        switch (reverseAttrInfo.dataType().getTypeCategory()) {
+        case CLASS:
+            if (reverseEdge != null && !reverseEdge.getId().toString().equals(newEdge.getId().toString())) {
+                // Disconnect old reference
+                deleteHandler.deleteEdgeReference(reverseEdge, reverseAttrInfo.dataType().getTypeCategory(),
+                    reverseAttrInfo.isComposite, true);
+            }
+            break;
+        case ARRAY:
+            // Add edge ID to property value
+            List<String> elements = reverseVertex.getProperty(propertyName, List.class);
+            if (elements == null) {
+                elements = new ArrayList<>();
+                elements.add(newEdge.getId().toString());
+                reverseVertex.setProperty(propertyName, elements);
+            }
+            else {
+               if (!elements.contains(newEdge.getId().toString())) {
+                    elements.add(newEdge.getId().toString());
+                    reverseVertex.setProperty(propertyName, elements);
+                }
+            }
+            break;
+        }
+
+        RequestContext requestContext = RequestContext.get();
+        GraphHelper.setProperty(reverseVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
+                requestContext.getRequestTime());
+        requestContext.recordEntityUpdate(reverseId._getId());
+    }
 }
\ No newline at end of file

repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java

@@ -24,11 +24,11 @@ import com.google.common.collect.ImmutableSet;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasClient.EntityResult;
 import org.apache.atlas.AtlasException;
-import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.RequestContext;
 import org.apache.atlas.TestUtils;
 import org.apache.atlas.repository.Constants;
+import org.apache.atlas.CreateUpdateEntitiesResult;
 import org.apache.atlas.repository.MetadataRepository;
 import org.apache.atlas.repository.RepositoryException;
 import org.apache.atlas.repository.graphdb.AtlasGraph;
@@ -63,10 +63,6 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -257,7 +253,8 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
     private String createInstance(Referenceable entity) throws Exception {
         ClassType dataType = typeSystem.getDataType(ClassType.class, entity.getTypeName());
         ITypedReferenceableInstance instance = dataType.convert(entity, Multiplicity.REQUIRED);
-        List<String> results = repositoryService.createEntities(instance).getCreatedEntities();
+        CreateUpdateEntitiesResult result = repositoryService.createEntities(instance);
+        List<String> results = result.getCreatedEntities();
         return results.get(results.size() - 1);
     }
 
@@ -434,7 +431,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
         Assert.assertNotNull(modificationTimestampPreUpdate);
 
         ITypedReferenceableInstance jane = repositoryService.getEntityDefinition(nameGuidMap.get("Jane"));
-        Id janeGuid = jane.getId();
+        Id janeId = jane.getId();
 
         // Update max's mentor reference to john.
         ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
@@ -456,7 +453,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
         Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
 
         // Update max's mentor reference to jane.
-        maxEntity.set("mentor", janeGuid);
+        maxEntity.set("mentor", janeId);
         entityResult = updatePartial(maxEntity);
         assertEquals(entityResult.getUpdateEntities().size(), 1);
         assertTrue(entityResult.getUpdateEntities().contains(maxGuid));
@@ -464,7 +461,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
         // Verify the update was applied correctly - jane should now be max's mentor.
         max = repositoryService.getEntityDefinition(maxGuid);
         refTarget = (ITypedReferenceableInstance) max.get("mentor");
-        Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
+        Assert.assertEquals(refTarget.getId()._getId(), janeId._getId());
 
         // Verify modification timestamp was updated.
         vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
@@ -473,21 +470,28 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
         Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
 
         ITypedReferenceableInstance julius = repositoryService.getEntityDefinition(nameGuidMap.get("Julius"));
-        Id juliusGuid = julius.getId();
+        Id juliusId = julius.getId();
         maxEntity = personType.createInstance(max.getId());
-        maxEntity.set("manager", juliusGuid);
+        maxEntity.set("manager", juliusId);
         entityResult = updatePartial(maxEntity);
-        //TODO ATLAS-499 should have updated julius' subordinates
-        assertEquals(entityResult.getUpdateEntities().size(), 2);
+        // Verify julius' subordinates were updated.
+        assertEquals(entityResult.getUpdateEntities().size(), 3);
         assertTrue(entityResult.getUpdateEntities().contains(maxGuid));
-        assertTrue(entityResult.getUpdateEntities().contains(janeGuid._getId()));
+        assertTrue(entityResult.getUpdateEntities().containsAll(Arrays.asList(maxGuid, janeId._getId(), juliusId._getId())));
 
         // Verify the update was applied correctly - julius should now be max's manager.
         max = repositoryService.getEntityDefinition(maxGuid);
         refTarget = (ITypedReferenceableInstance) max.get("manager");
-        Assert.assertEquals(refTarget.getId()._getId(), juliusGuid._getId());
+        Assert.assertEquals(refTarget.getId()._getId(), juliusId._getId());
+        Assert.assertEquals(refTarget.getId()._getId(), juliusId._getId());
+        julius = repositoryService.getEntityDefinition(nameGuidMap.get("Julius"));
+        Object object = julius.get("subordinates");
+        Assert.assertTrue(object instanceof List);
+        List<ITypedReferenceableInstance> refValues = (List<ITypedReferenceableInstance>) object;
+        Assert.assertEquals(refValues.size(), 1);
+        Assert.assertTrue(refValues.contains(max.getId()));
 
-        assertTestUpdateEntity_MultiplicityOneNonCompositeReference(janeGuid._getId());
+        assertTestUpdateEntity_MultiplicityOneNonCompositeReference(janeId._getId());
     }
 
     protected abstract void assertTestUpdateEntity_MultiplicityOneNonCompositeReference(String janeGuid) throws Exception;

repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateHardDeleteTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import java.util.List;
+
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.testng.Assert;
+
+
+/**
+ * Run tests in {@link ReverseReferenceUpdateTestBase} with hard delete enabled.
+ *
+ */
+public class ReverseReferenceUpdateHardDeleteTest extends ReverseReferenceUpdateTestBase {
+
+    @Override
+    DeleteHandler getDeleteHandler(TypeSystem typeSystem) {
+
+        return new HardDeleteHandler(typeSystem);
+    }
+
+    @Override
+    void assertTestOneToOneReference(Object refValue, ITypedReferenceableInstance expectedValue, ITypedReferenceableInstance referencingInstance) throws Exception {
+        // Verify reference was disconnected
+        Assert.assertNull(refValue);
+    }
+
+    @Override
+    void assertTestOneToManyReference(Object object, ITypedReferenceableInstance referencingInstance) {
+        Assert.assertTrue(object instanceof List);
+        List<ITypedReferenceableInstance> refValues = (List<ITypedReferenceableInstance>) object;
+        Assert.assertEquals(refValues.size(), 1);
+    }
+
+}

repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.testng.Assert;
+
+
+/**
+ * Run tests in {@link ReverseReferenceUpdateTestBase} with soft delete enabled.
+ *
+ */
+public class ReverseReferenceUpdateSoftDeleteTest extends ReverseReferenceUpdateTestBase {
+
+    @Override
+    DeleteHandler getDeleteHandler(TypeSystem typeSystem) {
+
+        return new SoftDeleteHandler(typeSystem);
+    }
+
+    @Override
+    void assertTestOneToOneReference(Object actual, ITypedReferenceableInstance expectedValue, ITypedReferenceableInstance referencingInstance) throws Exception {
+        // Verify reference was not disconnected if soft deletes are enabled.
+        Assert.assertNotNull(actual);
+        Assert.assertTrue(actual instanceof ITypedReferenceableInstance);
+        ITypedReferenceableInstance referenceValue = (ITypedReferenceableInstance) actual;
+        Assert.assertEquals(referenceValue.getId()._getId(), expectedValue.getId()._getId());
+
+        //Verify reference edge was marked as DELETED.
+        AtlasVertex vertexForGUID = GraphHelper.getInstance().getVertexForGUID(referencingInstance.getId()._getId());
+        String edgeLabel = GraphHelper.getEdgeLabel(typeB, typeB.fieldMapping.fields.get("a"));
+        AtlasEdge edgeForLabel = GraphHelper.getInstance().getEdgeForLabel(vertexForGUID, edgeLabel);
+        String edgeState = edgeForLabel.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
+        Assert.assertEquals(edgeState, Id.EntityState.DELETED.name());
+    }
+
+    @Override
+ void assertTestOneToManyReference(Object object, ITypedReferenceableInstance referencingInstance) throws Exception {
+        // Verify reference was not disconnected if soft deletes are enabled.
+        Assert.assertTrue(object instanceof List);
+        List<ITypedReferenceableInstance> refValues = (List<ITypedReferenceableInstance>) object;
+        Assert.assertEquals(refValues.size(), 2);
+
+        // Verify that one of the reference edges is marked DELETED.
+        AtlasVertex vertexForGUID = GraphHelper.getInstance().getVertexForGUID(referencingInstance.getId()._getId());
+        String edgeLabel = GraphHelper.getEdgeLabel(typeB, typeB.fieldMapping.fields.get("manyA"));
+        Iterator<AtlasEdge> outGoingEdgesByLabel = GraphHelper.getInstance().getOutGoingEdgesByLabel(vertexForGUID, edgeLabel);
+        boolean found = false;
+        while (outGoingEdgesByLabel.hasNext()) {
+            AtlasEdge edge = outGoingEdgesByLabel.next();
+            String edgeState = edge.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
+            if (edgeState.equals(Id.EntityState.DELETED.name())) {
+                found = true;
+                break;
+            }
+        }
+        Assert.assertTrue(found, "One edge for label " + edgeLabel + " should be marked " + Id.EntityState.DELETED.name());
+    }
+
+}

typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java

@@ -28,9 +28,12 @@ import org.apache.atlas.typesystem.types.AttributeInfo;
 import org.apache.atlas.typesystem.types.ClassType;
 import org.apache.atlas.typesystem.types.EnumTypeDefinition;
 import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.FieldMapping;
+import org.apache.atlas.typesystem.types.HierarchicalType;
 import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
 import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
 import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
 import org.apache.atlas.AtlasConstants;
@@ -129,4 +132,27 @@ public class TypesUtil {
             throw new RuntimeException(e);
         }
     }
+
+
+    /**
+     * Get the field mappings for the specified data type.
+     * Field mappings are only relevant for CLASS, TRAIT, and STRUCT types.
+     *
+     * @param type
+     * @return {@link FieldMapping} for the specified type
+     * @throws IllegalArgumentException if type is not a CLASS, TRAIT, or STRUCT type.
+     */
+    public static FieldMapping getFieldMapping(IDataType type) {
+        switch (type.getTypeCategory()) {
+        case CLASS:
+        case TRAIT:
+            return ((HierarchicalType)type).fieldMapping();
+
+        case STRUCT:
+            return ((StructType)type).fieldMapping();
+
+        default:
+            throw new IllegalArgumentException("Type " + type + " doesn't have any fields!");
+        }
+    }
 }