Skip to content

A
atlas
Commit 7ebb2013 by Suma Shivaprasad
Options

ATLAS-463 Disconnect inverse references ( dkantor via sumasai)

parent a77d1ab5
Showing with 795 additions and 30 deletions
release-log.txt
...@@ -10,6 +10,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ...@@ -10,6 +10,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES: ALL CHANGES:
ATLAS-463 Disconnect inverse references ( dkantor via sumasai)
ATLAS-479 Add description for different types during create time (guptaneeru via shwethags) ATLAS-479 Add description for different types during create time (guptaneeru via shwethags)
ATLAS-508 Apache nightly build failure - UnsupportedOperationException: Not a single key: __traitNames (shwethags) ATLAS-508 Apache nightly build failure - UnsupportedOperationException: Not a single key: __traitNames (shwethags)
ATLAS-422 JavaDoc NotificationConsumer and NotificationInterface.(tbeerbower via sumasai) ATLAS-422 JavaDoc NotificationConsumer and NotificationInterface.(tbeerbower via sumasai)
......
repository/src/main/java/org/apache/atlas/repository/graph/AtlasEdgeLabel.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
/**
* Represents an edge label used in Atlas.
* The format of an Atlas edge label is EDGE_LABEL_PREFIX<<typeName>>.<<attributeName>>[.mapKey]
*
*/
public class AtlasEdgeLabel {
private final String typeName_;
private final String attributeName_;
private final String mapKey_;
private final String edgeLabel_;
private final String qualifiedMapKey_;
private final String qualifiedAttributeName_;
public AtlasEdgeLabel(String edgeLabel) {
if (!edgeLabel.startsWith(GraphHelper.EDGE_LABEL_PREFIX)) {
throw new IllegalArgumentException("Invalid edge label " + edgeLabel + ": missing required prefix " + GraphHelper.EDGE_LABEL_PREFIX);
}
String labelWithoutPrefix = edgeLabel.substring(GraphHelper.EDGE_LABEL_PREFIX.length());
String[] fields = labelWithoutPrefix.split("\\.", 3);
if (fields.length < 2 || fields.length > 3) {
throw new IllegalArgumentException("Invalid edge label " + edgeLabel +
": expected 2 or 3 label components but found " + fields.length);
}
typeName_ = fields[0];
attributeName_ = fields[1];
if (fields.length == 3) {
mapKey_ = fields[2];
qualifiedMapKey_ = labelWithoutPrefix;
qualifiedAttributeName_ = typeName_ + '.' + attributeName_;
}
else {
mapKey_ = null;
qualifiedMapKey_ = null;
qualifiedAttributeName_ = labelWithoutPrefix;
}
edgeLabel_ = edgeLabel;
}
public String getTypeName() {
return typeName_;
}
public String getAttributeName() {
return attributeName_;
}
public String getMapKey() {
return mapKey_;
}
public String getEdgeLabel() {
return edgeLabel_;
}
public String getQualifiedMapKey() {
return qualifiedMapKey_;
}
public String getQualifiedAttributeName() {
return qualifiedAttributeName_;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append('(').append("typeName: ").append(typeName_);
sb.append(", attributeName: ").append(attributeName_);
if (mapKey_ != null) {
sb.append(", mapKey: ").append(mapKey_);
sb.append(", qualifiedMapKey: ").append(qualifiedMapKey_);
}
sb.append(", edgeLabel: ").append(edgeLabel_).append(')');
return sb.toString();
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
...@@ -21,6 +21,7 @@ import com.thinkaurelius.titan.core.SchemaViolationException; ...@@ -21,6 +21,7 @@ import com.thinkaurelius.titan.core.SchemaViolationException;
import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.RepositoryException;
...@@ -35,13 +36,17 @@ import org.apache.atlas.typesystem.persistence.ReferenceableInstance; ...@@ -35,13 +36,17 @@ import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.EnumValue; import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.IConstructableType;
import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.ObjectGraphWalker; import org.apache.atlas.typesystem.types.ObjectGraphWalker;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.TypeUtils.Pair;
import org.apache.atlas.utils.MD5Utils; import org.apache.atlas.utils.MD5Utils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -64,7 +69,6 @@ public final class TypedInstanceToGraphMapper { ...@@ -64,7 +69,6 @@ public final class TypedInstanceToGraphMapper {
private final List<String> deletedEntityGuids = new ArrayList<>(); private final List<String> deletedEntityGuids = new ArrayList<>();
private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>(); private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>();
private final GraphToTypedInstanceMapper graphToTypedInstanceMapper; private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
private static final GraphHelper graphHelper = GraphHelper.getInstance(); private static final GraphHelper graphHelper = GraphHelper.getInstance();
private final String SIGNATURE_HASH_PROPERTY_KEY = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "signature"; private final String SIGNATURE_HASH_PROPERTY_KEY = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "signature";
...@@ -174,6 +178,7 @@ public final class TypedInstanceToGraphMapper { ...@@ -174,6 +178,7 @@ public final class TypedInstanceToGraphMapper {
void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex, void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation) Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation)
throws AtlasException { throws AtlasException {
LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(), LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(),
instanceVertex); instanceVertex);
for (AttributeInfo attributeInfo : fields.values()) { for (AttributeInfo attributeInfo : fields.values()) {
...@@ -184,12 +189,27 @@ public final class TypedInstanceToGraphMapper { ...@@ -184,12 +189,27 @@ public final class TypedInstanceToGraphMapper {
} }
if (operation == Operation.DELETE) { if (operation == Operation.DELETE) {
// Remove uni-directional references to the deletion candidate.
removeUnidirectionalReferences(instanceVertex);
// Remove vertex for deletion candidate. // Remove vertex for deletion candidate.
graphHelper.removeVertex(instanceVertex); graphHelper.removeVertex(instanceVertex);
} }
} }
private String getInstanceName(Vertex referencingVertex, IConstructableType referencingType) {
if (referencingType.getTypeCategory() == TypeCategory.CLASS) {
Id idFromVertex = GraphHelper.getIdFromVertex(referencingType.getName(), referencingVertex);
String instanceId = referencingType.getName() + ":" + idFromVertex._getId();
return instanceId;
}
else {
return referencingType.getName();
}
}
void mapAttributesToVertex(ITypedInstance typedInstance, Vertex instanceVertex, void mapAttributesToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo, Operation operation) throws AtlasException { AttributeInfo attributeInfo, Operation operation) throws AtlasException {
Object attrValue = typedInstance.get(attributeInfo.name); Object attrValue = typedInstance.get(attributeInfo.name);
...@@ -361,7 +381,6 @@ public final class TypedInstanceToGraphMapper { ...@@ -361,7 +381,6 @@ public final class TypedInstanceToGraphMapper {
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType(); IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
List<String> newEntries = new ArrayList<>(); List<String> newEntries = new ArrayList<>();
if (newElements != null && !newElements.isEmpty()) { if (newElements != null && !newElements.isEmpty()) {
int index = 0; int index = 0;
for (; index < newElements.size(); index++) { for (; index < newElements.size(); index++) {
...@@ -404,7 +423,7 @@ public final class TypedInstanceToGraphMapper { ...@@ -404,7 +423,7 @@ public final class TypedInstanceToGraphMapper {
@SuppressWarnings("unchecked") Map<Object, Object> collection = @SuppressWarnings("unchecked") Map<Object, Object> collection =
(Map<Object, Object>) typedInstance.get(attributeInfo.name); (Map<Object, Object>) typedInstance.get(attributeInfo.name);
boolean empty = (collection == null || collection.isEmpty()); boolean empty = (collection == null || collection.isEmpty());
if (!empty || operation == Operation.UPDATE_FULL) { if (!empty || operation == Operation.UPDATE_FULL || operation == Operation.DELETE) {
String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo); String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType(); IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
...@@ -420,6 +439,7 @@ public final class TypedInstanceToGraphMapper { ...@@ -420,6 +439,7 @@ public final class TypedInstanceToGraphMapper {
//Add/Update/Remove property value //Add/Update/Remove property value
GraphHelper.setProperty(instanceVertex, myPropertyName, newEntry); GraphHelper.setProperty(instanceVertex, myPropertyName, newEntry);
} }
}
//Remove unused key references //Remove unused key references
List<Object> origKeys = instanceVertex.getProperty(propertyName); List<Object> origKeys = instanceVertex.getProperty(propertyName);
...@@ -437,8 +457,6 @@ public final class TypedInstanceToGraphMapper { ...@@ -437,8 +457,6 @@ public final class TypedInstanceToGraphMapper {
} }
} }
}
// for dereference on way out // for dereference on way out
GraphHelper.setProperty(instanceVertex, propertyName, collection == null ? null : new ArrayList(collection.keySet())); GraphHelper.setProperty(instanceVertex, propertyName, collection == null ? null : new ArrayList(collection.keySet()));
} }
...@@ -653,42 +671,241 @@ public final class TypedInstanceToGraphMapper { ...@@ -653,42 +671,241 @@ public final class TypedInstanceToGraphMapper {
} }
private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) throws AtlasException { private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) throws AtlasException {
//Remove edges for property values which do not exist any more TypeCategory typeCategory = elementType.getTypeCategory();
if (typeCategory != TypeCategory.STRUCT && elementType.getTypeCategory() != TypeCategory.CLASS) {
// Only class and struct references have edges.
return null;
}
// Remove edge to disconnect struct or class reference.
// For struct or composite class reference, also delete the target instance.
Edge removedRelation = null; Edge removedRelation = null;
switch (elementType.getTypeCategory()) {
case STRUCT:
removedRelation = graphHelper.removeRelation(edgeId, true);
//Remove the vertex from state so that further processing no longer uses this
break;
case CLASS:
// TODO: disconnect inverse reference if attributeInfo.reverseAttributeName is non-null.
if (attributeInfo.isComposite) {
// Delete contained entity.
TypeUtils.Pair<Edge, Vertex> edgeAndVertex = graphHelper.getEdgeAndTargetVertex(edgeId); TypeUtils.Pair<Edge, Vertex> edgeAndVertex = graphHelper.getEdgeAndTargetVertex(edgeId);
deleteEntity(elementType.getName(), edgeAndVertex.right); if (typeCategory == TypeCategory.STRUCT) {
graphHelper.removeEdge(edgeAndVertex.left); graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left; removedRelation = edgeAndVertex.left;
// Create an empty instance to use for clearing all struct attributes.
StructType structType = (StructType) elementType;
ITypedStruct typedInstance = structType.createInstance();
// Delete target vertex and any underlying structs and composite entities owned by this struct.
mapInstanceToVertex(typedInstance, edgeAndVertex.right, structType.fieldMapping().fields, false, Operation.DELETE);
} }
else { else {
removedRelation = graphHelper.removeRelation(edgeId, false); // Class reference
if (attributeInfo.isComposite) {
// For uni-directional reference, remove the edge.
// For bi-directional reference, the edges are removed
// when the composite entity is deleted.
if (attributeInfo.reverseAttributeName == null) {
graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left;
}
// Delete the contained entity.
if (LOG.isDebugEnabled()) {
Vertex sourceVertex = edgeAndVertex.left.getVertex(Direction.OUT);
String sourceTypeName = GraphHelper.getTypeName(sourceVertex);
LOG.debug("Deleting composite entity {}:{} contained by {}:{} through reference {}",
elementType.getName(), GraphHelper.getIdFromVertex(elementType.getName(), edgeAndVertex.right)._getId(),
sourceTypeName, GraphHelper.getIdFromVertex(sourceTypeName, sourceVertex)._getId(),
attributeInfo.name);
}
deleteEntity(elementType.getName(), edgeAndVertex.right);
}
else {
if (attributeInfo.reverseAttributeName != null) {
// Disconnect both ends of the bi-directional reference
removeReverseReference(edgeAndVertex, attributeInfo);
}
graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left;
} }
break;
} }
return removedRelation; return removedRelation;
} }
/**
* Remove the reverse reference value for the specified edge and vertex.
*
* @param edgeAndVertex
* @param attributeInfo
* @throws AtlasException
*/
private void removeReverseReference(TypeUtils.Pair<Edge, Vertex> edgeAndVertex,
AttributeInfo attributeInfo) throws AtlasException {
Vertex sourceVertex = edgeAndVertex.left.getVertex(Direction.OUT);
String inverseTypeName = GraphHelper.getTypeName(edgeAndVertex.right);
IConstructableType inverseType = typeSystem.getDataType(IConstructableType.class, inverseTypeName);
AttributeInfo inverseAttributeInfo = inverseType.fieldMapping().fields.get(attributeInfo.reverseAttributeName);
String inverseEdgeLabel = GraphHelper.getEdgeLabel(inverseType, inverseAttributeInfo);
TypeCategory inverseTypeCategory = inverseAttributeInfo.dataType().getTypeCategory();
// Find and remove the edge which represents the inverse reference value.
Iterable<Edge> inverseEdges = GraphHelper.getOutGoingEdgesByLabel(edgeAndVertex.right, inverseEdgeLabel);
Edge removedEdge = null;
// Search for the edge which references the source vertex.
for (Edge edge : inverseEdges) {
Vertex vertex = edge.getVertex(Direction.IN);
if (vertex.equals(sourceVertex)) {
// Found the edge which points back at source vertex.
// Disconnect the reference by removing the edge and
// removing the edge ID from the vertex property.
removeReferenceValue(edge, new AtlasEdgeLabel(edge.getLabel()), edgeAndVertex.right, inverseType, inverseTypeCategory);
removedEdge = edge;
break;
}
}
if (removedEdge != null) {
if (LOG.isDebugEnabled()) {
String sourceTypeName = GraphHelper.getTypeName(sourceVertex);
LOG.debug("Removed edge {} for reverse reference {} from {}:{} to {}:{} ", removedEdge,
GraphHelper.getQualifiedFieldName(inverseType, inverseAttributeInfo.name),
inverseTypeName, GraphHelper.getIdFromVertex(inverseTypeName, edgeAndVertex.right)._getId(),
sourceTypeName, GraphHelper.getIdFromVertex(sourceTypeName, sourceVertex)._getId());
}
}
else {
// We didn't find the edge for the inverse reference.
// Since Atlas currently does not automatically set
// the inverse reference when a reference value is updated,
// unbalanced references are not unexpected.
// The presence of inverse reference values depends on
// well behaved client applications which explicitly set
// both ends of the reference.
// TODO: throw an exception as it indicates a unbalanced reference?
String sourceTypeName = GraphHelper.getTypeName(sourceVertex);
LOG.warn("No edge found for inverse reference {} on vertex {} for entity instance {}:{} which points back to vertex {} for {}:{}",
inverseAttributeInfo.name, edgeAndVertex.right,
inverseTypeName, GraphHelper.getIdFromVertex(inverseTypeName, edgeAndVertex.right)._getId(),
sourceVertex, sourceTypeName, GraphHelper.getIdFromVertex(sourceTypeName, sourceVertex)._getId());
}
}
/**
* Remove any unidirectional map or array reference to a class, struct, or trait vertex.
* This involves removing appropriate value from the vertex property which holds the
* reference values.
*
* @param targetVertex a vertex which represents a class, struct, or trait instance
* @throws AtlasException
*/
private void removeUnidirectionalReferences(Vertex targetVertex) throws AtlasException {
// Search for any remaining incoming edges that represent unidirectional references
// to the target vertex.
Iterable<Edge> incomingEdges = targetVertex.getEdges(Direction.IN);
for (Edge edge : incomingEdges) {
String label = edge.getLabel();
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(label);
Vertex referencingVertex = edge.getVertex(Direction.OUT);
String typeName = atlasEdgeLabel.getTypeName();
IConstructableType referencingType = typeSystem.getDataType(IConstructableType.class, typeName);
AttributeInfo attributeInfo = referencingType.fieldMapping().fields.get(atlasEdgeLabel.getAttributeName());
if (attributeInfo == null) {
String instanceId = getInstanceName(referencingVertex, referencingType);
throw new AtlasException("Outgoing edge " + edge.getId().toString()
+ " for " + instanceId + "(vertex " + referencingVertex + "): label " + label
+ " has an attribute name " + atlasEdgeLabel.getAttributeName() + " that is undefined on "
+ referencingType.getTypeCategory() + " " + typeName);
}
// Remove the appropriate value from the vertex property for this reference.
removeReferenceValue(edge, atlasEdgeLabel, referencingVertex, referencingType, attributeInfo.dataType().getTypeCategory());
}
}
private Pair<String, Boolean> removeReferenceValue(Edge edge, AtlasEdgeLabel atlasEdgeLabel,
Vertex referencingVertex, IConstructableType referencingType, TypeCategory attrTypeCategory)
throws AtlasException {
graphHelper.removeEdge(edge);
if (attrTypeCategory != TypeCategory.ARRAY && attrTypeCategory != TypeCategory.MAP) {
// Multiplicity-one reference is represented by the edge,
// there is no vertex property to update. So just remove the edge.
return new Pair<String, Boolean>(edge.getId().toString(), Boolean.TRUE);
}
List<String> currentRefValues = referencingVertex.getProperty(atlasEdgeLabel.getQualifiedAttributeName());
List<String> newRefValues = new ArrayList<>(currentRefValues);
Pair<String, Boolean> refValueRemoved = null;
if (attrTypeCategory == TypeCategory.ARRAY) {
refValueRemoved = removeArrayReferenceValue(atlasEdgeLabel, referencingVertex, edge, newRefValues);
}
else {
refValueRemoved = removeMapReferenceValue(atlasEdgeLabel, referencingVertex, edge, newRefValues);
}
if (refValueRemoved.right) {
if (LOG.isDebugEnabled()) {
String instanceId = getInstanceName(referencingVertex, referencingType);
LOG.debug("Reference value {} removed from reference {} on vertex {} for instance of {} {}",
refValueRemoved.left, atlasEdgeLabel.getAttributeName(), referencingVertex,
referencingType.getTypeCategory(), instanceId);
}
// If the referencing instance is an entity, update the modification timestamp.
if (referencingType instanceof ClassType) {
GraphHelper.setProperty(referencingVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
}
}
else {
// The expected value is missing from the reference property values - log a warning.
String instanceId = getInstanceName(referencingVertex, referencingType);
LOG.warn("Reference value {} expected but not found in array reference {} on vertex {} for instance of {} {}",
refValueRemoved.left, atlasEdgeLabel.getAttributeName(), referencingVertex,
referencingType.getTypeCategory(), instanceId);
}
return refValueRemoved;
}
private TypeUtils.Pair<String, Boolean> removeArrayReferenceValue(AtlasEdgeLabel atlasEdgeLabel, Vertex referencingVertex,
Edge edge, List<String> newRefValues) {
String refValueToRemove = edge.getId().toString();
boolean valueRemoved = newRefValues.remove(refValueToRemove);
if (valueRemoved) {
GraphHelper.setProperty(referencingVertex, atlasEdgeLabel.getQualifiedAttributeName(), newRefValues);
}
return new TypeUtils.Pair<String, Boolean>(refValueToRemove, Boolean.valueOf(valueRemoved));
}
private TypeUtils.Pair<String, Boolean> removeMapReferenceValue(AtlasEdgeLabel atlasEdgeLabel, Vertex referencingVertex,
Edge edge, List<String> newRefValues) throws AtlasException {
String refValueToRemove = atlasEdgeLabel.getMapKey();
if (refValueToRemove == null) {
// Edge label is missing the map key - throw an exception.
String typeName = atlasEdgeLabel.getTypeName();
throw new AtlasException("Outgoing edge " + edge.getId().toString()
+ " for vertex " + referencingVertex + "): label " + atlasEdgeLabel.getEdgeLabel()
+ " for map attribute " + atlasEdgeLabel.getAttributeName() + " on type "
+ typeName + " is missing the map key");
}
boolean valueRemoved = newRefValues.remove(refValueToRemove);
if (valueRemoved) {
GraphHelper.setProperty(referencingVertex, atlasEdgeLabel.getQualifiedAttributeName(), newRefValues);
// For maps, also remove the key-value pair property value.
GraphHelper.setProperty(referencingVertex, atlasEdgeLabel.getQualifiedMapKey(), null);
}
return new TypeUtils.Pair<String, Boolean>(refValueToRemove, Boolean.valueOf(valueRemoved));
}
void deleteEntity(String typeName, Vertex instanceVertex) throws AtlasException { void deleteEntity(String typeName, Vertex instanceVertex) throws AtlasException {
// Check if this entity has already been processed.
Id id = GraphHelper.getIdFromVertex(typeName, instanceVertex);
if (deletedEntityGuids.contains(id._getId())) {
return;
}
deletedEntityGuids.add(id._getId());
// Remove traits owned by this entity. // Remove traits owned by this entity.
deleteAllTraits(instanceVertex); deleteAllTraits(instanceVertex);
// Create an empty instance to use for clearing all attributes. // Create an empty instance to use for clearing all attributes.
Id id = GraphHelper.getIdFromVertex(typeName, instanceVertex);
ClassType classType = typeSystem.getDataType(ClassType.class, typeName); ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
ITypedReferenceableInstance typedInstance = classType.createInstance(id); ITypedReferenceableInstance typedInstance = classType.createInstance(id);
// Remove any underlying structs and composite entities owned by this entity. // Remove any underlying structs and composite entities owned by this entity.
mapInstanceToVertex(typedInstance, instanceVertex, classType.fieldMapping().fields, false, Operation.DELETE); mapInstanceToVertex(typedInstance, instanceVertex, classType.fieldMapping().fields, false, Operation.DELETE);
deletedEntityGuids.add(id._getId());
deletedEntities.add(typedInstance); deletedEntities.add(typedInstance);
} }
......
repository/src/test/java/org/apache/atlas/TestUtils.java
...@@ -170,7 +170,7 @@ public final class TestUtils { ...@@ -170,7 +170,7 @@ public final class TestUtils {
max.set("mentor", julius); max.set("mentor", julius);
john.set("manager", jane); john.set("manager", jane);
john.set("mentor", max);
hrDept.set("employees", ImmutableList.of(john, jane, julius, max)); hrDept.set("employees", ImmutableList.of(john, jane, julius, max));
jane.set("subordinates", ImmutableList.of(john, max)); jane.set("subordinates", ImmutableList.of(john, max));
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
...@@ -18,6 +18,10 @@ ...@@ -18,6 +18,10 @@
package org.apache.atlas.repository.graph; package org.apache.atlas.repository.graph;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup; import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
...@@ -27,13 +31,25 @@ import org.apache.atlas.TestUtils; ...@@ -27,13 +31,25 @@ import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils.Pair; import org.apache.atlas.typesystem.types.TypeUtils.Pair;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
...@@ -44,7 +60,10 @@ import javax.inject.Inject; ...@@ -44,7 +60,10 @@ import javax.inject.Inject;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* Test for GraphBackedMetadataRepository.deleteEntities * Test for GraphBackedMetadataRepository.deleteEntities
...@@ -95,8 +114,12 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { ...@@ -95,8 +114,12 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
/**
* Verify deleting entities with composite references to other entities.
* The composite entities should also be deleted.
*/
@Test @Test
public void testDeleteEntities() throws Exception { public void testDeleteEntitiesWithCompositeArrayReference() throws Exception {
String hrDeptGuid = createHrDeptGraph(); String hrDeptGuid = createHrDeptGraph();
ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid); ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
...@@ -136,23 +159,395 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { ...@@ -136,23 +159,395 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
Assert.assertEquals(vertexCount, 0); Assert.assertEquals(vertexCount, 0);
} }
@Test(dependsOnMethods = "testDeleteEntities") @Test
public void testDeleteContainedEntity() throws Exception { public void testDeleteEntitiesWithCompositeMapReference() throws Exception {
// Define type for map value.
HierarchicalTypeDefinition<ClassType> mapValueDef = TypesUtil.createClassTypeDef("CompositeMapValue",
ImmutableList.<String>of(),
TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE));
// Define type with map where the value is a composite class reference to MapValue.
HierarchicalTypeDefinition<ClassType> mapOwnerDef = TypesUtil.createClassTypeDef("CompositeMapOwner",
ImmutableList.<String>of(),
new AttributeDefinition("map", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"CompositeMapValue"), Multiplicity.OPTIONAL, true, null));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(mapOwnerDef, mapValueDef));
typeSystem.defineTypes(typesDef);
ClassType mapOwnerType = typeSystem.getDataType(ClassType.class, "CompositeMapOwner");
ClassType mapValueType = typeSystem.getDataType(ClassType.class, "CompositeMapValue");
// Create instances of MapOwner and MapValue.
// Set MapOwner.map with one entry that references MapValue instance.
ITypedReferenceableInstance mapOwnerInstance = mapOwnerType.createInstance();
ITypedReferenceableInstance mapValueInstance = mapValueType.createInstance();
mapOwnerInstance.set("map", Collections.singletonMap("value1", mapValueInstance));
List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance);
Assert.assertEquals(createEntitiesResult.size(), 2);
List<String> guids = repositoryService.getEntityList("CompositeMapOwner");
Assert.assertEquals(guids.size(), 1);
String mapOwnerGuid = guids.get(0);
// Verify MapOwner.map attribute has expected value.
mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid);
Object object = mapOwnerInstance.get("map");
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof Map);
Map<String, ITypedReferenceableInstance> map = (Map<String, ITypedReferenceableInstance>)object;
Assert.assertEquals(map.size(), 1);
mapValueInstance = map.get("value1");
Assert.assertNotNull(mapValueInstance);
String mapValueGuid = mapValueInstance.getId()._getId();
String edgeLabel = GraphHelper.getEdgeLabel(mapOwnerType, mapOwnerType.fieldMapping.fields.get("map"));
String mapEntryLabel = edgeLabel + "." + "value1";
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel);
Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey());
Assert.assertNotNull(object);
Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult =
repositoryService.deleteEntities(Arrays.asList(mapOwnerGuid));
Assert.assertEquals(deleteEntitiesResult.left.size(), 2);
Assert.assertTrue(deleteEntitiesResult.left.containsAll(guids));
verifyEntityDoesNotExist(mapOwnerGuid);
verifyEntityDoesNotExist(mapValueGuid);
}
/**
* Verify deleting an entity which is contained by another
* entity through a bi-directional composite reference.
*
* @throws Exception
*/
@Test(dependsOnMethods = "testDeleteEntitiesWithCompositeArrayReference")
public void testDisconnectBidirectionalReferences() throws Exception {
String hrDeptGuid = createHrDeptGraph(); String hrDeptGuid = createHrDeptGraph();
ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid); ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
Object refValue = hrDept.get("employees"); Object refValue = hrDept.get("employees");
Assert.assertTrue(refValue instanceof List); Assert.assertTrue(refValue instanceof List);
List<Object> employees = (List<Object>)refValue; List<Object> employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 4); Assert.assertEquals(employees.size(), 4);
Object listValue = employees.get(2); String employeeGuid = null;
for (Object listValue : employees) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
if (employee.get("name").equals("Max")) {
employeeGuid = employee.getId()._getId();
}
}
Assert.assertNotNull(employeeGuid);
// Verify that Max is one of Jane's subordinates.
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Manager", "name", "Jane");
refValue = jane.get("subordinates");
Assert.assertTrue(refValue instanceof List);
List<Object> subordinates = (List<Object>)refValue;
Assert.assertEquals(subordinates.size(), 2);
List<String> subordinateIds = new ArrayList<>(2);
for (Object listValue : employees) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
String employeeGuid = employee.getId()._getId(); subordinateIds.add(employee.getId()._getId());
}
Assert.assertTrue(subordinateIds.contains(employeeGuid));
Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities = repositoryService.deleteEntities(Arrays.asList(employeeGuid)); Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities = repositoryService.deleteEntities(Arrays.asList(employeeGuid));
Assert.assertTrue(deletedEntities.left.contains(employeeGuid)); Assert.assertTrue(deletedEntities.left.contains(employeeGuid));
verifyEntityDoesNotExist(employeeGuid); verifyEntityDoesNotExist(employeeGuid);
// Verify that the Department.employees reference to the deleted employee
// was disconnected.
hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
refValue = hrDept.get("employees");
Assert.assertTrue(refValue instanceof List);
employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 3);
for (Object listValue : employees) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
Assert.assertNotEquals(employee.getId()._getId(), employeeGuid);
}
// Verify that the Manager.subordinates reference to the deleted employee
// Max was disconnected.
jane = repositoryService.getEntityDefinition("Manager", "name", "Jane");
refValue = jane.get("subordinates");
Assert.assertTrue(refValue instanceof List);
subordinates = (List<Object>)refValue;
Assert.assertEquals(subordinates.size(), 1);
Object listValue = subordinates.get(0);
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance subordinate = (ITypedReferenceableInstance) listValue;
String subordinateGuid = subordinate.getId()._getId();
Assert.assertNotEquals(subordinateGuid, employeeGuid);
// Verify that max's Person.mentor unidirectional reference to john was disconnected.
ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Manager", "name", "John");
refValue = john.get("mentor");
Assert.assertNull(refValue);
// Now delete jane - this should disconnect the manager reference from her
// subordinate.
String janeGuid = jane.getId()._getId();
deletedEntities = repositoryService.deleteEntities(Arrays.asList(janeGuid));
Assert.assertTrue(deletedEntities.left.contains(janeGuid));
verifyEntityDoesNotExist(janeGuid);
subordinate = repositoryService.getEntityDefinition(subordinateGuid);
Assert.assertNull(subordinate.get("manager"));
}
/**
* Verify deleting entity that is the target of a unidirectional class array reference
* from a class instance.
*/
@Test
public void testDisconnectUnidirectionalArrayReferenceFromClassType() throws Exception {
createDbTableGraph();
// Get the guid for one of the table's columns.
ITypedReferenceableInstance table = repositoryService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", TestUtils.TABLE_NAME);
String tableGuid = table.getId()._getId();
Object refValues = table.get("columns");
Assert.assertTrue(refValues instanceof List);
List<Object> refList = (List<Object>) refValues;
Assert.assertEquals(refList.size(), 5);
Assert.assertTrue(refList.get(0) instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance column = (ITypedReferenceableInstance) refList.get(0);
String columnGuid = column.getId()._getId();
// Delete the column.
Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities =
repositoryService.deleteEntities(Arrays.asList(columnGuid));
Assert.assertEquals(deletedEntities.left.size(), 1);
Assert.assertEquals(deletedEntities.right.size(), 1);
verifyEntityDoesNotExist(columnGuid);
// Verify table.columns reference to the deleted column has been disconnected.
table = repositoryService.getEntityDefinition(tableGuid);
refList = (List<Object>) table.get("columns");
Assert.assertEquals(refList.size(), 4);
for (Object refValue : refList) {
Assert.assertTrue(refValue instanceof ITypedReferenceableInstance);
column = (ITypedReferenceableInstance)refValue;
Assert.assertFalse(column.getId()._getId().equals(columnGuid));
}
}
/**
* Verify deleting entities that are the target of a unidirectional class array reference
* from a struct or trait instance.
*/
@Test
public void testDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes() throws Exception {
// Define class types.
HierarchicalTypeDefinition<ClassType> structTargetDef = TypesUtil.createClassTypeDef("StructTarget",
ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> traitTargetDef = TypesUtil.createClassTypeDef("TraitTarget",
ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> structContainerDef = TypesUtil.createClassTypeDef("StructContainer",
ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("struct", "TestStruct"));
// Define struct and trait types which have a unidirectional array reference
// to a class type.
StructTypeDefinition structDef = TypesUtil.createStructTypeDef("TestStruct",
new AttributeDefinition("target", DataTypes.arrayTypeName("StructTarget"), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("nestedStructs", DataTypes.arrayTypeName("NestedStruct"), Multiplicity.OPTIONAL, false, null));
StructTypeDefinition nestedStructDef = TypesUtil.createStructTypeDef("NestedStruct",
TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<TraitType> traitDef = TypesUtil.createTraitTypeDef("TestTrait", ImmutableList.<String>of(),
new AttributeDefinition("target", DataTypes.arrayTypeName("TraitTarget"), Multiplicity.OPTIONAL, false, null));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(structDef, nestedStructDef),
ImmutableList.of(traitDef), ImmutableList.of(structTargetDef, traitTargetDef, structContainerDef));
typeSystem.defineTypes(typesDef);
// Create instances of class, struct, and trait types.
Referenceable structTargetEntity = new Referenceable("StructTarget");
Referenceable traitTargetEntity = new Referenceable("TraitTarget");
Referenceable structContainerEntity = new Referenceable("StructContainer");
Referenceable structInstance = new Referenceable("TestStruct");
Referenceable nestedStructInstance = new Referenceable("NestedStruct");
Referenceable traitInstance = new Referenceable("TestTrait");
structContainerEntity.set("struct", structInstance);
structInstance.set("target", ImmutableList.of(structTargetEntity));
structInstance.set("nestedStructs", ImmutableList.of(nestedStructInstance));
ClassType structTargetType = typeSystem.getDataType(ClassType.class, "StructTarget");
ClassType traitTargetType = typeSystem.getDataType(ClassType.class, "TraitTarget");
ClassType structContainerType = typeSystem.getDataType(ClassType.class, "StructContainer");
ITypedReferenceableInstance structTargetConvertedEntity =
structTargetType.convert(structTargetEntity, Multiplicity.REQUIRED);
ITypedReferenceableInstance traitTargetConvertedEntity =
traitTargetType.convert(traitTargetEntity, Multiplicity.REQUIRED);
ITypedReferenceableInstance structContainerConvertedEntity =
structContainerType.convert(structContainerEntity, Multiplicity.REQUIRED);
List<String> guids = repositoryService.createEntities(
structTargetConvertedEntity, traitTargetConvertedEntity, structContainerConvertedEntity);
Assert.assertEquals(guids.size(), 3);
guids = repositoryService.getEntityList("StructTarget");
Assert.assertEquals(guids.size(), 1);
String structTargetGuid = guids.get(0);
guids = repositoryService.getEntityList("TraitTarget");
Assert.assertEquals(guids.size(), 1);
String traitTargetGuid = guids.get(0);
guids = repositoryService.getEntityList("StructContainer");
Assert.assertEquals(guids.size(), 1);
String structContainerGuid = guids.get(0);
// Add TestTrait to StructContainer instance
traitInstance.set("target", ImmutableList.of(new Id(traitTargetGuid, 0, "TraitTarget")));
TraitType traitType = typeSystem.getDataType(TraitType.class, "TestTrait");
ITypedStruct convertedTrait = traitType.convert(traitInstance, Multiplicity.REQUIRED);
repositoryService.addTrait(structContainerGuid, convertedTrait);
// Verify that the unidirectional references from the struct and trait instances
// are pointing at the target entities.
structContainerConvertedEntity = repositoryService.getEntityDefinition(structContainerGuid);
Object object = structContainerConvertedEntity.get("struct");
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof ITypedStruct);
ITypedStruct struct = (ITypedStruct) object;
object = struct.get("target");
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof List);
List<ITypedReferenceableInstance> refList = (List<ITypedReferenceableInstance>)object;
Assert.assertEquals(refList.size(), 1);
Assert.assertEquals(refList.get(0).getId()._getId(), structTargetGuid);
IStruct trait = structContainerConvertedEntity.getTrait("TestTrait");
Assert.assertNotNull(trait);
object = trait.get("target");
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof List);
refList = (List<ITypedReferenceableInstance>)object;
Assert.assertEquals(refList.size(), 1);
Assert.assertEquals(refList.get(0).getId()._getId(), traitTargetGuid);
// Delete the entities that are targets of the struct and trait instances.
Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult =
repositoryService.deleteEntities(Arrays.asList(structTargetGuid, traitTargetGuid));
verifyEntityDoesNotExist(structTargetGuid);
verifyEntityDoesNotExist(traitTargetGuid);
Assert.assertEquals(deleteEntitiesResult.left.size(), 2);
Assert.assertTrue(deleteEntitiesResult.left.containsAll(Arrays.asList(structTargetGuid, traitTargetGuid)));
// Verify that the unidirectional references from the struct and trait instances
// to the deleted entities were disconnected.
structContainerConvertedEntity = repositoryService.getEntityDefinition(structContainerGuid);
object = structContainerConvertedEntity.get("struct");
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof ITypedStruct);
struct = (ITypedStruct) object;
Assert.assertNull(struct.get("target"));
trait = structContainerConvertedEntity.getTrait("TestTrait");
Assert.assertNotNull(trait);
Assert.assertNull(trait.get("target"));
// Delete the entity which contains nested structs and has the TestTrait trait.
deleteEntitiesResult =
repositoryService.deleteEntities(Arrays.asList(structContainerGuid));
verifyEntityDoesNotExist(structContainerGuid);
Assert.assertEquals(deleteEntitiesResult.left.size(), 1);
Assert.assertTrue(deleteEntitiesResult.left.contains(structContainerGuid));
// Verify all TestStruct struct vertices were removed.
int vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "TestStruct");
Assert.assertEquals(vertexCount, 0);
// Verify all NestedStruct struct vertices were removed.
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "NestedStruct");
Assert.assertEquals(vertexCount, 0);
// Verify all TestTrait trait vertices were removed.
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "TestTrait");
Assert.assertEquals(vertexCount, 0);
}
/**
* Verify deleting entities that are the target of class map references.
*/
@Test
public void testDisconnectMapReferenceFromClassType() throws Exception {
// Define type for map value.
HierarchicalTypeDefinition<ClassType> mapValueDef = TypesUtil.createClassTypeDef("MapValue",
ImmutableList.<String>of(),
new AttributeDefinition("biMapOwner", "MapOwner", Multiplicity.OPTIONAL, false, "biMap"));
// Define type with unidirectional and bidirectional map references,
// where the map value is a class reference to MapValue.
HierarchicalTypeDefinition<ClassType> mapOwnerDef = TypesUtil.createClassTypeDef("MapOwner",
ImmutableList.<String>of(),
new AttributeDefinition("map", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"MapValue"), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("biMap", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"MapValue"), Multiplicity.OPTIONAL, false, "biMapOwner"));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(mapOwnerDef, mapValueDef));
typeSystem.defineTypes(typesDef);
ClassType mapOwnerType = typeSystem.getDataType(ClassType.class, "MapOwner");
ClassType mapValueType = typeSystem.getDataType(ClassType.class, "MapValue");
// Create instances of MapOwner and MapValue.
// Set MapOwner.map and MapOwner.biMap with one entry that references MapValue instance.
ITypedReferenceableInstance mapOwnerInstance = mapOwnerType.createInstance();
ITypedReferenceableInstance mapValueInstance = mapValueType.createInstance();
mapOwnerInstance.set("map", Collections.singletonMap("value1", mapValueInstance));
mapOwnerInstance.set("biMap", Collections.singletonMap("value1", mapValueInstance));
// Set biMapOwner reverse reference on MapValue.
mapValueInstance.set("biMapOwner", mapOwnerInstance);
List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance);
Assert.assertEquals(createEntitiesResult.size(), 2);
List<String> guids = repositoryService.getEntityList("MapOwner");
Assert.assertEquals(guids.size(), 1);
String mapOwnerGuid = guids.get(0);
String edgeLabel = GraphHelper.getEdgeLabel(mapOwnerType, mapOwnerType.fieldMapping.fields.get("map"));
String mapEntryLabel = edgeLabel + "." + "value1";
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel);
edgeLabel = GraphHelper.getEdgeLabel(mapOwnerType, mapOwnerType.fieldMapping.fields.get("biMap"));
mapEntryLabel = edgeLabel + "." + "value1";
AtlasEdgeLabel biMapAtlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel);
// Verify MapOwner.map attribute has expected value.
String mapValueGuid = null;
Vertex mapOwnerVertex = null;
mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid);
for (String mapAttrName : Arrays.asList("map", "biMap")) {
Object object = mapOwnerInstance.get(mapAttrName);
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof Map);
Map<String, ITypedReferenceableInstance> map = (Map<String, ITypedReferenceableInstance>)object;
Assert.assertEquals(map.size(), 1);
mapValueInstance = map.get("value1");
Assert.assertNotNull(mapValueInstance);
mapValueGuid = mapValueInstance.getId()._getId();
mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey());
Assert.assertNotNull(object);
}
// Delete the map value instance.
// This should disconnect the references from the map owner instance.
Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult =
repositoryService.deleteEntities(Arrays.asList(mapValueGuid));
verifyEntityDoesNotExist(mapValueGuid);
// Verify map references from mapOwner were disconnected.
mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid);
Assert.assertNull(mapOwnerInstance.get("map"));
Assert.assertNull(mapOwnerInstance.get("biMap"));
mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
Object object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey());
Assert.assertNull(object);
object = mapOwnerVertex.getProperty(biMapAtlasEdgeLabel.getQualifiedMapKey());
Assert.assertNull(object);
} }
private String createHrDeptGraph() throws Exception { private String createHrDeptGraph() throws Exception {
...@@ -170,6 +565,36 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { ...@@ -170,6 +565,36 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
return entityList.get(0); return entityList.get(0);
} }
private void createDbTableGraph() throws Exception {
Referenceable databaseInstance = new Referenceable(TestUtils.DATABASE_TYPE);
databaseInstance.set("name", TestUtils.DATABASE_NAME);
databaseInstance.set("description", "foo database");
ClassType dbType = typeSystem.getDataType(ClassType.class, TestUtils.DATABASE_TYPE);
ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
Referenceable tableInstance = new Referenceable(TestUtils.TABLE_TYPE, TestUtils.CLASSIFICATION);
tableInstance.set("name", TestUtils.TABLE_NAME);
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
Struct traitInstance = (Struct) tableInstance.getTrait(TestUtils.CLASSIFICATION);
traitInstance.set("tag", "foundation_etl");
tableInstance.set("tableType", 1); // enum
tableInstance.set("database", databaseInstance);
ArrayList<Referenceable> columns = new ArrayList<>();
for (int index = 0; index < 5; index++) {
Referenceable columnInstance = new Referenceable("column_type");
final String name = "column_" + index;
columnInstance.set("name", name);
columnInstance.set("type", "string");
columns.add(columnInstance);
}
tableInstance.set("columns", columns);
ClassType tableType = typeSystem.getDataType(ClassType.class, TestUtils.TABLE_TYPE);
ITypedReferenceableInstance table = tableType.convert(tableInstance, Multiplicity.REQUIRED);
repositoryService.createEntities(db, table);
}
private int countVertices(String propertyName, Object value) { private int countVertices(String propertyName, Object value) {
Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value); Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value);
int vertexCount = 0; int vertexCount = 0;
...@@ -179,9 +604,9 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { ...@@ -179,9 +604,9 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
return vertexCount; return vertexCount;
} }
private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException { private void verifyEntityDoesNotExist(String guid) throws RepositoryException {
try { try {
repositoryService.getEntityDefinition(hrDeptGuid); repositoryService.getEntityDefinition(guid);
Assert.fail("EntityNotFoundException was expected but none thrown"); Assert.fail("EntityNotFoundException was expected but none thrown");
} catch(EntityNotFoundException e) { } catch(EntityNotFoundException e) {
// good // good
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
...@@ -534,7 +534,6 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -534,7 +534,6 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate); Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
// Update max's mentor reference to jane. // Update max's mentor reference to jane.
instance = personType.createInstance(max.getId());
instance.set("mentor", janeGuid); instance.set("mentor", janeGuid);
repositoryService.updatePartial(instance); repositoryService.updatePartial(instance);
...@@ -550,6 +549,30 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -550,6 +549,30 @@ public class GraphBackedMetadataRepositoryTest {
Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(modificationTimestampPost2ndUpdate); Assert.assertNotNull(modificationTimestampPost2ndUpdate);
Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate); Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
ITypedReferenceableInstance julius = repositoryService.getEntityDefinition("Person", "name", "Julius");
Id juliusGuid = julius.getId();
instance = personType.createInstance(max.getId());
instance.set("manager", juliusGuid);
repositoryService.updatePartial(instance);
// Verify the update was applied correctly - julius should now be max's manager.
max = repositoryService.getEntityDefinition(maxGuid);
object = max.get("manager");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), juliusGuid._getId());
// Verify that max is no longer a subordinate of jane.
jane = repositoryService.getEntityDefinition(janeGuid._getId());
Object refValue = jane.get("subordinates");
Assert.assertTrue(refValue instanceof List);
List<Object> subordinates = (List<Object>)refValue;
Assert.assertEquals(subordinates.size(), 1);
Object listValue = subordinates.get(0);
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance subordinate = (ITypedReferenceableInstance) listValue;
Assert.assertNotEquals(subordinate.getId()._getId(), maxGuid);
} }
private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception { private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {
......
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
...@@ -788,7 +788,8 @@ public class DefaultMetadataServiceTest { ...@@ -788,7 +788,8 @@ public class DefaultMetadataServiceTest {
for (ITypedReferenceableInstance deletedEntity : deletedEntitiesFromListener) { for (ITypedReferenceableInstance deletedEntity : deletedEntitiesFromListener) {
deletedGuidsFromListener.add(deletedEntity.getId()._getId()); deletedGuidsFromListener.add(deletedEntity.getId()._getId());
} }
Assert.assertEquals(deletedGuidsFromListener, deletedGuids); Assert.assertEquals(deletedGuidsFromListener.size(), deletedGuids.size());
Assert.assertTrue(deletedGuidsFromListener.containsAll(deletedGuids));
} }
private static class DeleteEntitiesChangeListener implements EntityChangeListener { private static class DeleteEntitiesChangeListener implements EntityChangeListener {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment