Skip to content

A
atlas
Commit a46711c5 by Shwetha GS
Options

ATLAS-370 Implement deleteEntities at repository level (dkantor via shwethags)

parent c4eebe0e
Showing with 375 additions and 21 deletions
.gitignore
......@@ -39,6 +39,9 @@ test-output
#Python
*.pyc
# review board
.reviewboardrc
# other files
.DS_Store
*.swp
release-log.txt
......@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-370 Implement deleteEntities at repository level (dkantor via shwethags)
ATLAS-406 Resizing lineage window – should be an anchor on a corner – like ppt for graphic (sanjayp via shwethags)
ATLAS-432 QuickStart lineage is broken (yhemanth via shwethags)
ATLAS-421 typo in Architecture.twiki (dbist13 via shwethags)
......
repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
......@@ -104,13 +104,15 @@ public interface MetadataRepository {
List<String> getEntityList(String entityType) throws RepositoryException;
/**
* Deletes an entity definition (instance) corresponding to a given type.
* Deletes entities for the specified guids.
*
* @param guid globally unique identifier for the entity
* @return true if deleted else false
* @param guids globally unique identifiers for the deletion candidate entities
* @return guids of deleted entities
* @throws RepositoryException
*/
// boolean deleteEntity(String guid) throws RepositoryException;
List <String> deleteEntities(String... guids) throws RepositoryException;
// Trait management functions
/**
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -24,18 +24,22 @@ import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.TypedInstanceToGraphMapper.Operation;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.slf4j.Logger;
......@@ -43,6 +47,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
......@@ -258,7 +263,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
titanGraph.removeEdge(traitEdge);
if (traitVertex != null) { // remove the trait instance from the repository
titanGraph.removeVertex(traitVertex);
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
instanceToGraphMapper.deleteTraitVertex(traitNameToBeDeleted, traitVertex);
// update the traits in entity once trait removal is successful
traitNames.remove(traitNameToBeDeleted);
......@@ -270,6 +276,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
private void updateTraits(Vertex instanceVertex, List<String> traitNames) {
// remove the key
instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
......@@ -304,4 +311,34 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throw new RepositoryException(e);
}
}
@Override
@GraphTransaction
public List<String> deleteEntities(String... guids) throws RepositoryException {
if (guids == null || guids.length == 0) {
throw new IllegalArgumentException("guids must be non-null and non-empty");
}
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
for (String guid : guids) {
if (guid == null) {
LOG.warn("deleteEntities: Ignoring null guid");
continue;
}
try {
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
String typeName = GraphHelper.getTypeName(instanceVertex);
instanceToGraphMapper.deleteEntity(typeName, instanceVertex);
} catch (EntityNotFoundException e) {
// Entity does not exist - treat as non-error, since the caller
// wanted to delete the entity and it's already gone.
LOG.info("Deletion request ignored for non-existent entity with guid " + guid);
continue;
} catch (AtlasException e) {
throw new RepositoryException(e);
}
}
return instanceToGraphMapper.getDeletedEntities();
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -26,6 +26,7 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.IReferenceableInstance;
......@@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -175,11 +177,44 @@ public final class GraphHelper {
LOG.info("Removed edge {}", edge);
if (cascade) {
Vertex referredVertex = edge.getVertex(Direction.IN);
titanGraph.removeVertex(referredVertex);
LOG.info("Removed vertex {}", referredVertex);
removeVertex(referredVertex);
}
return edge;
}
/**
* Remove the specified edge from the graph.
*
* @param edge
*/
public void removeEdge(Edge edge) {
LOG.debug("Removing edge {}", edge);
titanGraph.removeEdge(edge);
LOG.info("Removed edge {}", edge);
}
/**
* Return the edge and target vertex for the specified edge ID.
*
* @param edgeId
* @return edge and target vertex
*/
public Pair<Edge, Vertex> getEdgeAndTargetVertex(String edgeId) {
final Edge edge = titanGraph.getEdge(edgeId);
Vertex referredVertex = edge.getVertex(Direction.IN);
return Pair.of(edge, referredVertex);
}
/**
* Remove the specified vertex from the graph.
*
* @param vertex
*/
public void removeVertex(Vertex vertex) {
LOG.debug("Removing vertex {}", vertex);
titanGraph.removeVertex(vertex);
LOG.info("Removed vertex {}", vertex);
}
public Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid);
......@@ -269,7 +304,6 @@ public final class GraphHelper {
return result;
}
public static void dumpToLog(final Graph graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
......
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
......@@ -21,6 +21,7 @@ import com.thinkaurelius.titan.core.SchemaViolationException;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
......@@ -61,6 +62,7 @@ public final class TypedInstanceToGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class);
private final Map<Id, Vertex> idToVertexMap = new HashMap<>();
private final TypeSystem typeSystem = TypeSystem.getInstance();
private final List<String> deletedEntities = new ArrayList<>();
private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
......@@ -108,7 +110,7 @@ public final class TypedInstanceToGraphMapper {
addFullTextProperty(instancesPair.right);
break;
case DELETE:
default:
throw new UnsupportedOperationException("Not handled - " + operation);
}
}
......@@ -169,8 +171,8 @@ public final class TypedInstanceToGraphMapper {
return getId(typedInstance)._getId();
}
private void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation)
void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation)
throws AtlasException {
LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(),
instanceVertex);
......@@ -180,20 +182,26 @@ public final class TypedInstanceToGraphMapper {
}
mapAttributesToVertex(typedInstance, instanceVertex, attributeInfo, operation);
}
if (operation == Operation.DELETE) {
// Remove vertex for deletion candidate.
graphHelper.removeVertex(instanceVertex);
}
}
void mapAttributesToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
AttributeInfo attributeInfo, Operation operation) throws AtlasException {
Object attrValue = typedInstance.get(attributeInfo.name);
LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
final String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
if (attrValue != null || operation == Operation.UPDATE_FULL) {
if (attrValue != null || operation == Operation.UPDATE_FULL || operation == Operation.DELETE) {
switch (attributeInfo.dataType().getTypeCategory()) {
case PRIMITIVE:
case ENUM:
mapPrimitiveOrEnumToVertex(typedInstance, instanceVertex, attributeInfo);
if (operation != Operation.DELETE) {
mapPrimitiveOrEnumToVertex(typedInstance, instanceVertex, attributeInfo);
}
break;
case ARRAY:
......@@ -206,6 +214,8 @@ public final class TypedInstanceToGraphMapper {
case STRUCT:
case CLASS:
final String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
Iterator<Edge> outGoingEdgesIterator =
GraphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel).iterator();
String currentEntry =
......@@ -345,7 +355,7 @@ public final class TypedInstanceToGraphMapper {
attributeInfo.name);
List newElements = (List) typedInstance.get(attributeInfo.name);
boolean empty = (newElements == null || newElements.isEmpty());
if (!empty || operation == Operation.UPDATE_FULL) {
if (!empty || operation == Operation.UPDATE_FULL || operation == Operation.DELETE) {
String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
List<String> currentEntries = instanceVertex.getProperty(propertyName);
......@@ -371,6 +381,14 @@ public final class TypedInstanceToGraphMapper {
}
}
}
else if (operation == Operation.UPDATE_FULL || operation == Operation.DELETE) {
// Clear all existing entries
if (currentEntries != null) {
for (String edgeId : currentEntries) {
removeUnusedReference(edgeId, attributeInfo, elementType);
}
}
}
// for dereference on way out
GraphHelper.setProperty(instanceVertex, propertyName, newEntries);
......@@ -525,10 +543,10 @@ public final class TypedInstanceToGraphMapper {
}
private Id getId(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
Id id = null;
if (typedReference != null) {
id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
if (typedReference == null) {
throw new IllegalArgumentException("typedReference must be non-null");
}
Id id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
if (id.isUnassigned()) {
Vertex classVertex = idToVertexMap.get(id);
......@@ -634,7 +652,7 @@ public final class TypedInstanceToGraphMapper {
GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
}
private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) {
private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) throws AtlasException {
//Remove edges for property values which do not exist any more
Edge removedRelation = null;
switch (elementType.getTypeCategory()) {
......@@ -643,9 +661,79 @@ public final class TypedInstanceToGraphMapper {
//Remove the vertex from state so that further processing no longer uses this
break;
case CLASS:
removedRelation = graphHelper.removeRelation(edgeId, attributeInfo.isComposite);
// TODO: disconnect inverse reference if attributeInfo.reverseAttributeName is non-null.
if (attributeInfo.isComposite) {
// Delete contained entity.
TypeUtils.Pair<Edge, Vertex> edgeAndVertex = graphHelper.getEdgeAndTargetVertex(edgeId);
deleteEntity(elementType.getName(), edgeAndVertex.right);
graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left;
}
else {
removedRelation = graphHelper.removeRelation(edgeId, false);
}
break;
}
return removedRelation;
}
void deleteEntity(String typeName, Vertex instanceVertex) throws AtlasException {
// Remove traits owned by this entity.
deleteAllTraits(instanceVertex);
// Create an empty instance to use for clearing all attributes.
Id id = GraphHelper.getIdFromVertex(typeName, instanceVertex);
ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
ITypedReferenceableInstance typedInstance = classType.createInstance(id);
// Remove any underlying structs and composite entities owned by this entity.
mapInstanceToVertex(typedInstance, instanceVertex, classType.fieldMapping().fields, false, Operation.DELETE);
deletedEntities.add(id._getId());
}
/**
* Delete all traits from the specified vertex.
*
* @param instanceVertex
* @throws AtlasException
*/
private void deleteAllTraits(Vertex instanceVertex) throws AtlasException {
List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
final String entityTypeName = GraphHelper.getTypeName(instanceVertex);
for (String traitNameToBeDeleted : traitNames) {
String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted);
Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
if (results.hasNext()) { // there should only be one edge for this label
final Edge traitEdge = results.next();
final Vertex traitVertex = traitEdge.getVertex(Direction.IN);
// remove the edge to the trait instance from the repository
graphHelper.removeEdge(traitEdge);
if (traitVertex != null) { // remove the trait instance from the repository
deleteTraitVertex(traitNameToBeDeleted, traitVertex);
}
}
}
}
void deleteTraitVertex(String traitName, final Vertex traitVertex) throws AtlasException {
TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
ITypedStruct traitStruct = traitType.createInstance();
// Remove trait vertex along with any struct and class attributes owned by this trait.
mapInstanceToVertex(traitStruct, traitVertex, traitType.fieldMapping().fields, false, Operation.DELETE);
}
/**
* Get the IDs of entities that have been deleted.
*
* @return
*/
List<String> getDeletedEntities() {
return Collections.unmodifiableList(deletedEntities);
}
}
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Vertex;
/**
* Test for GraphBackedMetadataRepository.deleteEntities
*
* Guice loads the dependencies and injects the necessary objects
*
*/
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
@Inject
private GraphBackedDiscoveryService discoveryService;
private TypeSystem typeSystem;
@BeforeClass
public void setUp() throws Exception {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
new GraphBackedSearchIndexer(graphProvider);
TestUtils.defineDeptEmployeeTypes(typeSystem);
TestUtils.createHiveTypes(typeSystem);
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void testDeleteEntities() throws Exception {
String hrDeptGuid = createHrDeptGraph();
ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
Object refValue = hrDept.get("employees");
Assert.assertTrue(refValue instanceof List);
List<Object> employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 4);
List<String> employeeGuids = new ArrayList<String>(4);
for (Object listValue : employees) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
employeeGuids.add(employee.getId()._getId());
}
// There should be 4 vertices for Address structs (one for each Person.address attribute value).
int vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
Assert.assertEquals(vertexCount, 4);
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
Assert.assertEquals(vertexCount, 1);
List<String> deletedEntities = repositoryService.deleteEntities(hrDeptGuid);
Assert.assertTrue(deletedEntities.contains(hrDeptGuid));
// Verify Department entity and its contained Person entities were deleted.
verifyEntityDoesNotExist(hrDeptGuid);
for (String employeeGuid : employeeGuids) {
verifyEntityDoesNotExist(employeeGuid);
}
// Verify all Person.address struct vertices were removed.
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
Assert.assertEquals(vertexCount, 0);
// Verify all SecurityClearance trait vertices were removed.
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
Assert.assertEquals(vertexCount, 0);
}
@Test(dependsOnMethods = "testDeleteEntities")
public void testDeleteContainedEntity() throws Exception {
String hrDeptGuid = createHrDeptGraph();
ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
Object refValue = hrDept.get("employees");
Assert.assertTrue(refValue instanceof List);
List<Object> employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 4);
Object listValue = employees.get(2);
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
String employeeGuid = employee.getId()._getId();
List<String> deletedEntities = repositoryService.deleteEntities(employeeGuid);
Assert.assertTrue(deletedEntities.contains(employeeGuid));
verifyEntityDoesNotExist(employeeGuid);
}
private String createHrDeptGraph() throws Exception {
Referenceable deptEg1 = TestUtils.createDeptEg1(typeSystem);
ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(deptEg1, Multiplicity.REQUIRED);
List<String> guids = repositoryService.createEntities(hrDept2);
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 5);
List<String> entityList = repositoryService.getEntityList("Department");
Assert.assertNotNull(entityList);
Assert.assertEquals(entityList.size(), 1);
String hrDeptGuid = entityList.get(0);
return hrDeptGuid;
}
private int countVertices(String propertyName, Object value) {
Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value);
int vertexCount = 0;
for (Vertex vertex : vertices) {
vertexCount++;
}
return vertexCount;
}
private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException {
try {
repositoryService.getEntityDefinition(hrDeptGuid);
Assert.fail("EntityNotFoundException was expected but none thrown");
}
catch(EntityNotFoundException e) {
// good
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment