Skip to content

A
atlas
Commit 852a7118 by Dave Kantor
Options

ATLAS-1551 auto update of reverse references in V1 API

parent 3a0865ad
Showing with 275 additions and 29 deletions
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1551 auto update of reverse references in V1 API (dkantor)
ATLAS-1565 Create EntityREST endpoints for delete operations (sarathkumarsubramanian via svimal2106)
ATLAS-1547 Added tests for hard delete (mneethiraj)
ATLAS-1546 (ATLAS-1546.3.patch)Hive hook should choose appropriate JAAS config if host uses kerberos ticket-cache (nixonrodrigues via kevalbhat)
......
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
......@@ -49,6 +49,7 @@ import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
......@@ -56,6 +57,7 @@ import org.apache.atlas.typesystem.types.ObjectGraphWalker;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.atlas.utils.MD5Utils;
import org.slf4j.Logger;
......@@ -249,6 +251,9 @@ public final class TypedInstanceToGraphMapper {
deleteHandler.deleteEdgeReference(currentEdge, attributeInfo.dataType().getTypeCategory(),
attributeInfo.isComposite, true);
}
if (attributeInfo.reverseAttributeName != null && newEdge != null) {
addReverseReference(instanceVertex, attributeInfo.reverseAttributeName, newEdge);
}
break;
case TRAIT:
......@@ -421,24 +426,35 @@ public final class TypedInstanceToGraphMapper {
List<Object> newElementsCreated = new ArrayList<>();
if (!newAttributeEmpty) {
if (newElements != null && !newElements.isEmpty()) {
int index = 0;
for (; index < newElements.size(); index++) {
Object currentElement = (currentElements != null && index < currentElements.size()) ?
currentElements.get(index) : null;
if (LOG.isDebugEnabled()) {
LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index,
currentElement, newElements.get(index));
}
Object newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
newElements.get(index), currentElement, propertyName, operation);
newElementsCreated.add(newEntry);
int index = 0;
for (; index < newElements.size(); index++) {
Object currentElement = (currentElements != null && index < currentElements.size()) ?
currentElements.get(index) : null;
if (LOG.isDebugEnabled()) {
LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index,
currentElement, newElements.get(index));
}
Object newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
newElements.get(index), currentElement, propertyName, operation);
newElementsCreated.add(newEntry);
}
}
if(GraphHelper.isReference(elementType)) {
if (attributeInfo.reverseAttributeName != null && newElementsCreated.size() > 0) {
// Set/add the new reference value(s) on the reverse reference.
for (Object newElement : newElementsCreated) {
if ((newElement instanceof AtlasEdge)) {
AtlasEdge newEdge = (AtlasEdge) newElement;
addReverseReference(instanceVertex, attributeInfo.reverseAttributeName, newEdge);
}
else {
throw new AtlasException("Invalid array element type " + newElement.getClass().getName() + " - expected " + AtlasEdge.class.getName() +
" for reference " + GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo) + " on vertex " + GraphHelper.getVertexDetails(instanceVertex));
}
}
}
List<AtlasEdge> additionalEdges = removeUnusedEntries(instanceVertex, propertyName, (List)currentElements,
(List)newElementsCreated, elementType, attributeInfo);
......@@ -866,4 +882,67 @@ public final class TypedInstanceToGraphMapper {
}
return new GuidMapping(mapping);
}
private <V,E> void addReverseReference(AtlasVertex<V,E> vertex, String reverseAttributeName, AtlasEdge<V,E> edge)
throws AtlasException {
String typeName = GraphHelper.getTypeName(vertex);
Id id = GraphHelper.getIdFromVertex(typeName, vertex);
AtlasVertex<V, E> reverseVertex = edge.getInVertex();
String reverseTypeName = GraphHelper.getTypeName(reverseVertex);
Id reverseId = GraphHelper.getIdFromVertex(reverseTypeName, reverseVertex);
IDataType reverseType = typeSystem.getDataType(IDataType.class, reverseTypeName);
AttributeInfo reverseAttrInfo = TypesUtil.getFieldMapping(reverseType).fields.get(reverseAttributeName);
if (reverseAttrInfo.dataType().getTypeCategory() == TypeCategory.MAP) {
// If the reverse reference is a map, what would be used as the key?
// Not supporting automatic update of reverse map references.
LOG.debug("Automatic update of reverse map reference is not supported - reference = {}",
GraphHelper.getQualifiedFieldName(reverseType, reverseAttributeName));
return;
}
String propertyName = GraphHelper.getQualifiedFieldName(reverseType, reverseAttributeName);
String reverseEdgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
AtlasEdge<V, E> reverseEdge = graphHelper.getEdgeForLabel(reverseVertex, reverseEdgeLabel);
AtlasEdge<V, E> newEdge = null;
if (reverseEdge != null) {
newEdge = updateClassEdge(reverseVertex, reverseEdge, id, vertex, reverseAttrInfo, reverseEdgeLabel);
}
else {
newEdge = addClassEdge(reverseVertex, vertex, reverseEdgeLabel);
}
switch (reverseAttrInfo.dataType().getTypeCategory()) {
case CLASS:
if (reverseEdge != null && !reverseEdge.getId().toString().equals(newEdge.getId().toString())) {
// Disconnect old reference
deleteHandler.deleteEdgeReference(reverseEdge, reverseAttrInfo.dataType().getTypeCategory(),
reverseAttrInfo.isComposite, true);
}
break;
case ARRAY:
// Add edge ID to property value
List<String> elements = reverseVertex.getProperty(propertyName, List.class);
if (elements == null) {
elements = new ArrayList<>();
elements.add(newEdge.getId().toString());
reverseVertex.setProperty(propertyName, elements);
}
else {
if (!elements.contains(newEdge.getId().toString())) {
elements.add(newEdge.getId().toString());
reverseVertex.setProperty(propertyName, elements);
}
}
break;
}
RequestContext requestContext = RequestContext.get();
GraphHelper.setProperty(reverseVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
requestContext.getRequestTime());
requestContext.recordEntityUpdate(reverseId._getId());
}
}
\ No newline at end of file
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
......@@ -24,11 +24,11 @@ import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasClient.EntityResult;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
......@@ -63,10 +63,6 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
......@@ -257,7 +253,8 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
private String createInstance(Referenceable entity) throws Exception {
ClassType dataType = typeSystem.getDataType(ClassType.class, entity.getTypeName());
ITypedReferenceableInstance instance = dataType.convert(entity, Multiplicity.REQUIRED);
List<String> results = repositoryService.createEntities(instance).getCreatedEntities();
CreateUpdateEntitiesResult result = repositoryService.createEntities(instance);
List<String> results = result.getCreatedEntities();
return results.get(results.size() - 1);
}
......@@ -434,7 +431,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
Assert.assertNotNull(modificationTimestampPreUpdate);
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition(nameGuidMap.get("Jane"));
Id janeGuid = jane.getId();
Id janeId = jane.getId();
// Update max's mentor reference to john.
ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
......@@ -456,7 +453,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
// Update max's mentor reference to jane.
maxEntity.set("mentor", janeGuid);
maxEntity.set("mentor", janeId);
entityResult = updatePartial(maxEntity);
assertEquals(entityResult.getUpdateEntities().size(), 1);
assertTrue(entityResult.getUpdateEntities().contains(maxGuid));
......@@ -464,7 +461,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
// Verify the update was applied correctly - jane should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
refTarget = (ITypedReferenceableInstance) max.get("mentor");
Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
Assert.assertEquals(refTarget.getId()._getId(), janeId._getId());
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
......@@ -473,21 +470,28 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
ITypedReferenceableInstance julius = repositoryService.getEntityDefinition(nameGuidMap.get("Julius"));
Id juliusGuid = julius.getId();
Id juliusId = julius.getId();
maxEntity = personType.createInstance(max.getId());
maxEntity.set("manager", juliusGuid);
maxEntity.set("manager", juliusId);
entityResult = updatePartial(maxEntity);
//TODO ATLAS-499 should have updated julius' subordinates
assertEquals(entityResult.getUpdateEntities().size(), 2);
// Verify julius' subordinates were updated.
assertEquals(entityResult.getUpdateEntities().size(), 3);
assertTrue(entityResult.getUpdateEntities().contains(maxGuid));
assertTrue(entityResult.getUpdateEntities().contains(janeGuid._getId()));
assertTrue(entityResult.getUpdateEntities().containsAll(Arrays.asList(maxGuid, janeId._getId(), juliusId._getId())));
// Verify the update was applied correctly - julius should now be max's manager.
max = repositoryService.getEntityDefinition(maxGuid);
refTarget = (ITypedReferenceableInstance) max.get("manager");
Assert.assertEquals(refTarget.getId()._getId(), juliusGuid._getId());
Assert.assertEquals(refTarget.getId()._getId(), juliusId._getId());
Assert.assertEquals(refTarget.getId()._getId(), juliusId._getId());
julius = repositoryService.getEntityDefinition(nameGuidMap.get("Julius"));
Object object = julius.get("subordinates");
Assert.assertTrue(object instanceof List);
List<ITypedReferenceableInstance> refValues = (List<ITypedReferenceableInstance>) object;
Assert.assertEquals(refValues.size(), 1);
Assert.assertTrue(refValues.contains(max.getId()));
assertTestUpdateEntity_MultiplicityOneNonCompositeReference(janeGuid._getId());
assertTestUpdateEntity_MultiplicityOneNonCompositeReference(janeId._getId());
}
protected abstract void assertTestUpdateEntity_MultiplicityOneNonCompositeReference(String janeGuid) throws Exception;
......
repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateHardDeleteTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import java.util.List;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
/**
* Run tests in {@link ReverseReferenceUpdateTestBase} with hard delete enabled.
*
*/
public class ReverseReferenceUpdateHardDeleteTest extends ReverseReferenceUpdateTestBase {
@Override
DeleteHandler getDeleteHandler(TypeSystem typeSystem) {
return new HardDeleteHandler(typeSystem);
}
@Override
void assertTestOneToOneReference(Object refValue, ITypedReferenceableInstance expectedValue, ITypedReferenceableInstance referencingInstance) throws Exception {
// Verify reference was disconnected
Assert.assertNull(refValue);
}
@Override
void assertTestOneToManyReference(Object object, ITypedReferenceableInstance referencingInstance) {
Assert.assertTrue(object instanceof List);
List<ITypedReferenceableInstance> refValues = (List<ITypedReferenceableInstance>) object;
Assert.assertEquals(refValues.size(), 1);
}
}
repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import java.util.Iterator;
import java.util.List;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
/**
* Run tests in {@link ReverseReferenceUpdateTestBase} with soft delete enabled.
*
*/
public class ReverseReferenceUpdateSoftDeleteTest extends ReverseReferenceUpdateTestBase {
@Override
DeleteHandler getDeleteHandler(TypeSystem typeSystem) {
return new SoftDeleteHandler(typeSystem);
}
@Override
void assertTestOneToOneReference(Object actual, ITypedReferenceableInstance expectedValue, ITypedReferenceableInstance referencingInstance) throws Exception {
// Verify reference was not disconnected if soft deletes are enabled.
Assert.assertNotNull(actual);
Assert.assertTrue(actual instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance referenceValue = (ITypedReferenceableInstance) actual;
Assert.assertEquals(referenceValue.getId()._getId(), expectedValue.getId()._getId());
//Verify reference edge was marked as DELETED.
AtlasVertex vertexForGUID = GraphHelper.getInstance().getVertexForGUID(referencingInstance.getId()._getId());
String edgeLabel = GraphHelper.getEdgeLabel(typeB, typeB.fieldMapping.fields.get("a"));
AtlasEdge edgeForLabel = GraphHelper.getInstance().getEdgeForLabel(vertexForGUID, edgeLabel);
String edgeState = edgeForLabel.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
Assert.assertEquals(edgeState, Id.EntityState.DELETED.name());
}
@Override
void assertTestOneToManyReference(Object object, ITypedReferenceableInstance referencingInstance) throws Exception {
// Verify reference was not disconnected if soft deletes are enabled.
Assert.assertTrue(object instanceof List);
List<ITypedReferenceableInstance> refValues = (List<ITypedReferenceableInstance>) object;
Assert.assertEquals(refValues.size(), 2);
// Verify that one of the reference edges is marked DELETED.
AtlasVertex vertexForGUID = GraphHelper.getInstance().getVertexForGUID(referencingInstance.getId()._getId());
String edgeLabel = GraphHelper.getEdgeLabel(typeB, typeB.fieldMapping.fields.get("manyA"));
Iterator<AtlasEdge> outGoingEdgesByLabel = GraphHelper.getInstance().getOutGoingEdgesByLabel(vertexForGUID, edgeLabel);
boolean found = false;
while (outGoingEdgesByLabel.hasNext()) {
AtlasEdge edge = outGoingEdgesByLabel.next();
String edgeState = edge.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
if (edgeState.equals(Id.EntityState.DELETED.name())) {
found = true;
break;
}
}
Assert.assertTrue(found, "One edge for label " + edgeLabel + " should be marked " + Id.EntityState.DELETED.name());
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
......@@ -28,9 +28,12 @@ import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.AtlasConstants;
......@@ -129,4 +132,27 @@ public class TypesUtil {
throw new RuntimeException(e);
}
}
/**
* Get the field mappings for the specified data type.
* Field mappings are only relevant for CLASS, TRAIT, and STRUCT types.
*
* @param type
* @return {@link FieldMapping} for the specified type
* @throws IllegalArgumentException if type is not a CLASS, TRAIT, or STRUCT type.
*/
public static FieldMapping getFieldMapping(IDataType type) {
switch (type.getTypeCategory()) {
case CLASS:
case TRAIT:
return ((HierarchicalType)type).fieldMapping();
case STRUCT:
return ((StructType)type).fieldMapping();
default:
throw new IllegalArgumentException("Type " + type + " doesn't have any fields!");
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment