Commit 20608f02 by Shwetha GS

ATLAS-372 Expose entity deletion through REST API (dkantor via shwethags)

parent d2b9b99f
...@@ -166,6 +166,8 @@ public class AtlasClient { ...@@ -166,6 +166,8 @@ public class AtlasClient {
UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT, Response.Status.OK), UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT, Response.Status.OK),
UPDATE_ENTITY_PARTIAL(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK), UPDATE_ENTITY_PARTIAL(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK),
LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
DELETE_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK),
//Trait operations //Trait operations
ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED), ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED),
...@@ -379,6 +381,23 @@ public class AtlasClient { ...@@ -379,6 +381,23 @@ public class AtlasClient {
} }
/** /**
* Delete the specified entities from the repository
*
* @param guids guids of entities to delete
* @return List of deleted entity guids
* @throws AtlasServiceException
*/
public List<String> deleteEntities(String ... guids) throws AtlasServiceException {
API api = API.DELETE_ENTITIES;
WebResource resource = getResource(api);
for (String guid : guids) {
resource = resource.queryParam(GUID.toLowerCase(), guid);
}
JSONObject jsonResponse = callAPIWithResource(API.DELETE_ENTITIES, resource);
return extractResults(jsonResponse, GUID);
}
/**
* Get an entity given the entity id * Get an entity given the entity id
* @param guid entity id * @param guid entity id
* @return result object * @return result object
......
...@@ -33,6 +33,7 @@ public interface EntityNotification { ...@@ -33,6 +33,7 @@ public interface EntityNotification {
enum OperationType { enum OperationType {
ENTITY_CREATE, ENTITY_CREATE,
ENTITY_UPDATE, ENTITY_UPDATE,
ENTITY_DELETE,
TRAIT_ADD, TRAIT_ADD,
TRAIT_DELETE TRAIT_DELETE
} }
......
...@@ -76,9 +76,15 @@ public class NotificationEntityChangeListener implements EntityChangeListener { ...@@ -76,9 +76,15 @@ public class NotificationEntityChangeListener implements EntityChangeListener {
notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.TRAIT_DELETE); notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.TRAIT_DELETE);
} }
@Override
public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_DELETE);
}
// ----- helper methods ------------------------------------------------- // ----- helper methods -------------------------------------------------
// send notification of entity change // send notification of entity change
private void notifyOfEntityEvent(Collection<ITypedReferenceableInstance> entityDefinitions, private void notifyOfEntityEvent(Collection<ITypedReferenceableInstance> entityDefinitions,
EntityNotification.OperationType operationType) throws AtlasException { EntityNotification.OperationType operationType) throws AtlasException {
......
...@@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ...@@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES: ALL CHANGES:
ATLAS-372 Expose entity deletion through REST API (dkantor via shwethags)
ATLAS-452 Exceptions while running HiveHookIT#testAlterTableRename (shwethags) ATLAS-452 Exceptions while running HiveHookIT#testAlterTableRename (shwethags)
ATLAS-388 UI : On creating Tag, the page to be reset for creating new Tag (Anilg via shwethags) ATLAS-388 UI : On creating Tag, the page to be reset for creating new Tag (Anilg via shwethags)
ATLAS-199 webapp build fails (grunt + tests) (sanjayp via shwethags) ATLAS-199 webapp build fails (grunt + tests) (sanjayp via shwethags)
......
...@@ -110,7 +110,7 @@ public interface MetadataRepository { ...@@ -110,7 +110,7 @@ public interface MetadataRepository {
* @return guids of deleted entities * @return guids of deleted entities
* @throws RepositoryException * @throws RepositoryException
*/ */
List <String> deleteEntities(String... guids) throws RepositoryException; TypeUtils.Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntities(List<String> guids) throws RepositoryException;
// Trait management functions // Trait management functions
......
...@@ -30,16 +30,13 @@ import org.apache.atlas.GraphTransaction; ...@@ -30,16 +30,13 @@ import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.TypedInstanceToGraphMapper.Operation;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.exception.EntityExistsException; import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.TypeUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
...@@ -316,9 +313,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -316,9 +313,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override @Override
@GraphTransaction @GraphTransaction
public List<String> deleteEntities(String... guids) throws RepositoryException { public TypeUtils.Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntities(List<String> guids) throws RepositoryException {
if (guids == null || guids.length == 0) { if (guids == null || guids.size() == 0) {
throw new IllegalArgumentException("guids must be non-null and non-empty"); throw new IllegalArgumentException("guids must be non-null and non-empty");
} }
...@@ -341,6 +338,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -341,6 +338,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throw new RepositoryException(e); throw new RepositoryException(e);
} }
} }
return instanceToGraphMapper.getDeletedEntities(); return new TypeUtils.Pair<>(
instanceToGraphMapper.getDeletedEntityGuids(), instanceToGraphMapper.getDeletedEntities());
} }
} }
...@@ -61,8 +61,8 @@ public final class TypedInstanceToGraphMapper { ...@@ -61,8 +61,8 @@ public final class TypedInstanceToGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class); private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class);
private final Map<Id, Vertex> idToVertexMap = new HashMap<>(); private final Map<Id, Vertex> idToVertexMap = new HashMap<>();
private final TypeSystem typeSystem = TypeSystem.getInstance(); private final TypeSystem typeSystem = TypeSystem.getInstance();
private final List<String> deletedEntities = new ArrayList<>(); private final List<String> deletedEntityGuids = new ArrayList<>();
private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>();
private final GraphToTypedInstanceMapper graphToTypedInstanceMapper; private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
private static final GraphHelper graphHelper = GraphHelper.getInstance(); private static final GraphHelper graphHelper = GraphHelper.getInstance();
...@@ -688,7 +688,8 @@ public final class TypedInstanceToGraphMapper { ...@@ -688,7 +688,8 @@ public final class TypedInstanceToGraphMapper {
// Remove any underlying structs and composite entities owned by this entity. // Remove any underlying structs and composite entities owned by this entity.
mapInstanceToVertex(typedInstance, instanceVertex, classType.fieldMapping().fields, false, Operation.DELETE); mapInstanceToVertex(typedInstance, instanceVertex, classType.fieldMapping().fields, false, Operation.DELETE);
deletedEntities.add(id._getId()); deletedEntityGuids.add(id._getId());
deletedEntities.add(typedInstance);
} }
/** /**
...@@ -728,12 +729,31 @@ public final class TypedInstanceToGraphMapper { ...@@ -728,12 +729,31 @@ public final class TypedInstanceToGraphMapper {
/** /**
* Get the IDs of entities that have been deleted. * Get the GUIDs of entities that have been deleted.
* *
* @return * @return
*/ */
List<String> getDeletedEntities() { List<String> getDeletedEntityGuids() {
return Collections.unmodifiableList(deletedEntities); if (deletedEntityGuids.size() == 0) {
return Collections.emptyList();
}
else {
return Collections.unmodifiableList(deletedEntityGuids);
}
}
/**
* Get the entities that have been deleted.
*
* @return
*/
List<ITypedReferenceableInstance> getDeletedEntities() {
if (deletedEntities.size() == 0) {
return Collections.emptyList();
}
else {
return Collections.unmodifiableList(deletedEntities);
}
} }
} }
...@@ -21,6 +21,7 @@ package org.apache.atlas.services; ...@@ -21,6 +21,7 @@ package org.apache.atlas.services;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.inject.Provider; import com.google.inject.Provider;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.classification.InterfaceAudience;
...@@ -54,6 +55,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition; ...@@ -54,6 +55,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.TypeUtils.Pair;
import org.apache.atlas.typesystem.types.ValueConversionException; import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.utils.ParamChecker;
...@@ -65,6 +67,7 @@ import org.slf4j.LoggerFactory; ...@@ -65,6 +67,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject; import javax.inject.Inject;
import javax.inject.Singleton; import javax.inject.Singleton;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
...@@ -680,4 +683,23 @@ public class DefaultMetadataService implements MetadataService { ...@@ -680,4 +683,23 @@ public class DefaultMetadataService implements MetadataService {
public void unregisterListener(EntityChangeListener listener) { public void unregisterListener(EntityChangeListener listener) {
entityChangeListeners.remove(listener); entityChangeListeners.remove(listener);
} }
/* (non-Javadoc)
* @see org.apache.atlas.services.MetadataService#deleteEntities(java.lang.String)
*/
@Override
public List<String> deleteEntities(List<String> deleteCandidateGuids) throws AtlasException {
ParamChecker.notEmpty(deleteCandidateGuids, "delete candidate guids cannot be empty");
Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult = repository.deleteEntities(deleteCandidateGuids);
if (deleteEntitiesResult.right.size() > 0) {
onEntitiesDeleted(deleteEntitiesResult.right);
}
return deleteEntitiesResult.left;
}
private void onEntitiesDeleted(List<ITypedReferenceableInstance> entities) throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntitiesDeleted(entities);
}
}
} }
...@@ -23,6 +23,7 @@ import com.thinkaurelius.titan.core.TitanGraph; ...@@ -23,6 +23,7 @@ import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
...@@ -187,6 +188,7 @@ public final class TestUtils { ...@@ -187,6 +188,7 @@ public final class TestUtils {
public static final String DATABASE_TYPE = "hive_database"; public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo"; public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table"; public static final String TABLE_TYPE = "hive_table";
public static final String COLUMN_TYPE = "column_type";
public static final String TABLE_NAME = "bar"; public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification"; public static final String CLASSIFICATION = "classification";
public static final String PII = "PII"; public static final String PII = "PII";
...@@ -220,7 +222,7 @@ public final class TestUtils { ...@@ -220,7 +222,7 @@ public final class TestUtils {
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values); EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
HierarchicalTypeDefinition<ClassType> columnsDefinition = HierarchicalTypeDefinition<ClassType> columnsDefinition =
createClassTypeDef("column_type", ImmutableList.<String>of(), createClassTypeDef(COLUMN_TYPE, ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE)); createRequiredAttrDef("type", DataTypes.STRING_TYPE));
...@@ -228,7 +230,7 @@ public final class TestUtils { ...@@ -228,7 +230,7 @@ public final class TestUtils {
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),}); new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),});
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("cols", String.format("array<%s>", "column_type"), new AttributeDefinition("cols", String.format("array<%s>", COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null), Multiplicity.OPTIONAL, true, null),
new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null), null),
...@@ -256,7 +258,7 @@ public final class TestUtils { ...@@ -256,7 +258,7 @@ public final class TestUtils {
null), null),
new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true, new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true,
null), null),
new AttributeDefinition("columns", DataTypes.arrayTypeName("column_type"), new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null), Multiplicity.OPTIONAL, true, null),
new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),}; new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),};
...@@ -277,7 +279,7 @@ public final class TestUtils { ...@@ -277,7 +279,7 @@ public final class TestUtils {
String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL,
false, null), false, null),
// array of classes // array of classes
new AttributeDefinition("columns", String.format("array<%s>", "column_type"), new AttributeDefinition("columns", String.format("array<%s>", COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null), Multiplicity.OPTIONAL, true, null),
// array of structs // array of structs
new AttributeDefinition("partitions", String.format("array<%s>", "partition_struct_type"), new AttributeDefinition("partitions", String.format("array<%s>", "partition_struct_type"),
...@@ -289,7 +291,7 @@ public final class TestUtils { ...@@ -289,7 +291,7 @@ public final class TestUtils {
//map of classes - //map of classes -
new AttributeDefinition("columnsMap", new AttributeDefinition("columnsMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"column_type"), COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null), Multiplicity.OPTIONAL, true, null),
//map of structs //map of structs
new AttributeDefinition("partitionsMap", new AttributeDefinition("partitionsMap",
......
...@@ -21,6 +21,7 @@ package org.apache.atlas.repository.graph; ...@@ -21,6 +21,7 @@ package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup; import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils; import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
...@@ -32,6 +33,7 @@ import org.apache.atlas.typesystem.exception.EntityNotFoundException; ...@@ -32,6 +33,7 @@ import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils.Pair;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
...@@ -39,7 +41,9 @@ import org.testng.annotations.Guice; ...@@ -39,7 +41,9 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import javax.inject.Inject; import javax.inject.Inject;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.List; import java.util.List;
/** /**
...@@ -114,8 +118,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { ...@@ -114,8 +118,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance"); vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
Assert.assertEquals(vertexCount, 1); Assert.assertEquals(vertexCount, 1);
List<String> deletedEntities = repositoryService.deleteEntities(hrDeptGuid); Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities = repositoryService.deleteEntities(Arrays.asList(hrDeptGuid));
Assert.assertTrue(deletedEntities.contains(hrDeptGuid)); Assert.assertTrue(deletedEntities.left.contains(hrDeptGuid));
// Verify Department entity and its contained Person entities were deleted. // Verify Department entity and its contained Person entities were deleted.
verifyEntityDoesNotExist(hrDeptGuid); verifyEntityDoesNotExist(hrDeptGuid);
...@@ -145,8 +149,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { ...@@ -145,8 +149,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
String employeeGuid = employee.getId()._getId(); String employeeGuid = employee.getId()._getId();
List<String> deletedEntities = repositoryService.deleteEntities(employeeGuid); Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities = repositoryService.deleteEntities(Arrays.asList(employeeGuid));
Assert.assertTrue(deletedEntities.contains(employeeGuid)); Assert.assertTrue(deletedEntities.left.contains(employeeGuid));
verifyEntityDoesNotExist(employeeGuid); verifyEntityDoesNotExist(employeeGuid);
} }
......
...@@ -22,14 +22,20 @@ import com.google.common.collect.ImmutableList; ...@@ -22,14 +22,20 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup; import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.typesystem.exception.TypeNotFoundException; import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils; import org.apache.atlas.TestUtils;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct; import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.TypesDef;
...@@ -49,6 +55,7 @@ import org.testng.annotations.Test; ...@@ -49,6 +55,7 @@ import org.testng.annotations.Test;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -152,6 +159,12 @@ public class DefaultMetadataServiceTest { ...@@ -152,6 +159,12 @@ public class DefaultMetadataServiceTest {
return entity; return entity;
} }
private Referenceable createColumnEntity() {
Referenceable entity = new Referenceable(TestUtils.COLUMN_TYPE);
entity.set("name", RandomStringUtils.randomAlphanumeric(10));
entity.set("type", "VARCHAR(32)");
return entity;
}
@Test @Test
public void testCreateEntityWithUniqueAttribute() throws Exception { public void testCreateEntityWithUniqueAttribute() throws Exception {
//name is the unique attribute //name is the unique attribute
...@@ -685,4 +698,121 @@ public class DefaultMetadataServiceTest { ...@@ -685,4 +698,121 @@ public class DefaultMetadataServiceTest {
//expected //expected
} }
} }
@Test
public void testDeleteEntities() throws Exception {
// Create 2 table entities, each with 3 composite column entities
Referenceable dbEntity = createDBEntity();
String dbGuid = createInstance(dbEntity);
Id dbId = new Id(dbGuid, 0, TestUtils.DATABASE_TYPE);
Referenceable table1Entity = createTableEntity(dbId);
Referenceable table2Entity = createTableEntity(dbId);
Referenceable col1 = createColumnEntity();
Referenceable col2 = createColumnEntity();
Referenceable col3 = createColumnEntity();
table1Entity.set("columns", ImmutableList.of(col1, col2, col3));
table2Entity.set("columns", ImmutableList.of(col1, col2, col3));
createInstance(table1Entity);
createInstance(table2Entity);
// Retrieve the table entities from the repository,
// to get their guids and the composite column guids.
String entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE,
"name", (String)table1Entity.get("name"));
Assert.assertNotNull(entityJson);
table1Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true);
Object val = table1Entity.get("columns");
Assert.assertTrue(val instanceof List);
List<IReferenceableInstance> table1Columns = (List<IReferenceableInstance>) val;
entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE,
"name", (String)table2Entity.get("name"));
Assert.assertNotNull(entityJson);
table2Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true);
val = table2Entity.get("columns");
Assert.assertTrue(val instanceof List);
List<IReferenceableInstance> table2Columns = (List<IReferenceableInstance>) val;
// Register an EntityChangeListener to verify the notification mechanism
// is working for deleteEntities().
DeleteEntitiesChangeListener listener = new DeleteEntitiesChangeListener();
metadataService.registerListener(listener);
// Delete the table entities. The deletion should cascade
// to their composite columns.
JSONArray deleteCandidateGuids = new JSONArray();
deleteCandidateGuids.put(table1Entity.getId()._getId());
deleteCandidateGuids.put(table2Entity.getId()._getId());
List<String> deletedGuids = metadataService.deleteEntities(
Arrays.asList(table1Entity.getId()._getId(), table2Entity.getId()._getId()));
// Verify that deleteEntities() response has guids for tables and their composite columns.
Assert.assertTrue(deletedGuids.contains(table1Entity.getId()._getId()));
Assert.assertTrue(deletedGuids.contains(table2Entity.getId()._getId()));
for (IReferenceableInstance column : table1Columns) {
Assert.assertTrue(deletedGuids.contains(column.getId()._getId()));
}
for (IReferenceableInstance column : table2Columns) {
Assert.assertTrue(deletedGuids.contains(column.getId()._getId()));
}
// Verify that tables and their composite columns have been deleted from the repository.
for (String guid : deletedGuids) {
try {
metadataService.getEntityDefinition(guid);
Assert.fail(EntityNotFoundException.class.getSimpleName() +
" expected but not thrown. The entity with guid " + guid +
" still exists in the repository after being deleted." );
}
catch(EntityNotFoundException e) {
// The entity does not exist in the repository, so deletion was successful.
}
}
// Verify that the listener was notified about the deleted entities.
Collection<ITypedReferenceableInstance> deletedEntitiesFromListener = listener.getDeletedEntities();
Assert.assertNotNull(deletedEntitiesFromListener);
Assert.assertEquals(deletedEntitiesFromListener.size(), deletedGuids.size());
List<String> deletedGuidsFromListener = new ArrayList<>(deletedGuids.size());
for (ITypedReferenceableInstance deletedEntity : deletedEntitiesFromListener) {
deletedGuidsFromListener.add(deletedEntity.getId()._getId());
}
Assert.assertEquals(deletedGuidsFromListener, deletedGuids);
}
private static class DeleteEntitiesChangeListener implements EntityChangeListener {
private Collection<ITypedReferenceableInstance> deletedEntities_;
@Override
public void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities)
throws AtlasException {
}
@Override
public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities)
throws AtlasException {
}
@Override
public void onTraitAdded(ITypedReferenceableInstance entity, IStruct trait)
throws AtlasException {
}
@Override
public void onTraitDeleted(ITypedReferenceableInstance entity, String traitName)
throws AtlasException {
}
@Override
public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities)
throws AtlasException {
deletedEntities_ = entities;
}
public Collection<ITypedReferenceableInstance> getDeletedEntities() {
return deletedEntities_;
}
}
} }
...@@ -66,4 +66,12 @@ public interface EntityChangeListener { ...@@ -66,4 +66,12 @@ public interface EntityChangeListener {
* @throws AtlasException if the listener notification fails * @throws AtlasException if the listener notification fails
*/ */
void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) throws AtlasException; void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) throws AtlasException;
/**
* This is upon deleting entities from the repository.
*
* @param entities the deleted entities
* @throws AtlasException
*/
void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities) throws AtlasException;
} }
...@@ -175,6 +175,15 @@ public interface MetadataService { ...@@ -175,6 +175,15 @@ public interface MetadataService {
void deleteTrait(String guid, String traitNameToBeDeleted) throws AtlasException; void deleteTrait(String guid, String traitNameToBeDeleted) throws AtlasException;
/** /**
* Delete the specified entities from the repository
*
* @param guids entity guids to be deleted
* @return List of guids for deleted entities
* @throws AtlasException
*/
List<String> deleteEntities(List<String> guids) throws AtlasException;
/**
* Register a listener for entity change. * Register a listener for entity change.
* *
* @param listener the listener to register * @param listener the listener to register
......
...@@ -21,14 +21,14 @@ package org.apache.atlas.web.resources; ...@@ -21,14 +21,14 @@ package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.exception.EntityExistsException; import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException; import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.types.ValueConversionException; import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.web.util.Servlets; import org.apache.atlas.web.util.Servlets;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
...@@ -250,7 +250,7 @@ public class EntityResource { ...@@ -250,7 +250,7 @@ public class EntityResource {
return updateEntityAttributeByGuid(guid, attribute, request); return updateEntityAttributeByGuid(guid, attribute, request);
} }
} }
private Response updateEntityPartialByGuid(String guid, HttpServletRequest request) { private Response updateEntityPartialByGuid(String guid, HttpServletRequest request) {
try { try {
ParamChecker.notEmpty(guid, "Guid property cannot be null"); ParamChecker.notEmpty(guid, "Guid property cannot be null");
...@@ -312,6 +312,37 @@ public class EntityResource { ...@@ -312,6 +312,37 @@ public class EntityResource {
} }
/** /**
* Delete entities from the repository
*
* @param guids deletion candidate guids
* @param request
* @return response payload as json
*/
@DELETE
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response deleteEntities(@QueryParam("guid") List<String> guids, @Context HttpServletRequest request) {
try {
List<String> deletedGuids = metadataService.deleteEntities(guids);
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
JSONArray guidArray = new JSONArray(deletedGuids.size());
for (String guid : deletedGuids) {
guidArray.put(guid);
}
response.put(AtlasClient.GUID, guidArray);
return Response.ok(response).build();
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to delete entities {}", guids, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to delete entities {}", guids, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Fetch the complete definition of an entity given its GUID. * Fetch the complete definition of an entity given its GUID.
* *
* @param guid GUID for the entity * @param guid GUID for the entity
......
...@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; ...@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.WebResource;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException; import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.notification.NotificationConsumer; import org.apache.atlas.notification.NotificationConsumer;
...@@ -58,6 +59,7 @@ import org.testng.annotations.Test; ...@@ -58,6 +59,7 @@ import org.testng.annotations.Test;
import javax.ws.rs.HttpMethod; import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -698,4 +700,85 @@ public class EntityJerseyResourceIT extends BaseResourceIT { ...@@ -698,4 +700,85 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertTrue(refs.get(0).equalsContents(columns.get(0))); Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
Assert.assertTrue(refs.get(1).equalsContents(columns.get(1))); Assert.assertTrue(refs.get(1).equalsContents(columns.get(1)));
} }
@Test
public void testDeleteEntitiesViaRestApi() throws Exception {
// Create 2 database entities
Referenceable db1 = new Referenceable(DATABASE_TYPE);
db1.set("name", randomString());
db1.set("description", randomString());
Id db1Id = createInstance(db1);
Referenceable db2 = new Referenceable(DATABASE_TYPE);
db2.set("name", randomString());
db2.set("description", randomString());
Id db2Id = createInstance(db2);
// Delete the database entities
ClientResponse clientResponse = service.path(ENTITIES).
queryParam(AtlasClient.GUID.toLowerCase(), db1Id._getId()).
queryParam(AtlasClient.GUID.toLowerCase(), db2Id._getId()).
accept(Servlets.JSON_MEDIA_TYPE).type(Servlets.JSON_MEDIA_TYPE).method(HttpMethod.DELETE, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
// Verify that response has guids for both database entities
JSONObject response = new JSONObject(clientResponse.getEntity(String.class));
final String deletedGuidsJson = response.getString(AtlasClient.GUID);
Assert.assertNotNull(deletedGuidsJson);
JSONArray guidsArray = new JSONArray(deletedGuidsJson);
Assert.assertEquals(guidsArray.length(), 2);
List<String> deletedGuidsList = new ArrayList<>(2);
for (int index = 0; index < guidsArray.length(); index++) {
deletedGuidsList.add(guidsArray.getString(index));
}
Assert.assertTrue(deletedGuidsList.contains(db1Id._getId()));
Assert.assertTrue(deletedGuidsList.contains(db2Id._getId()));
// Verify entities were deleted from the repository.
for (String guid : deletedGuidsList) {
try {
serviceClient.getEntity(guid);
Assert.fail(AtlasServiceException.class.getSimpleName() +
" was expected but not thrown. The entity with guid " + guid +
" still exists in the repository after being deleted.");
}
catch (AtlasServiceException e) {
Assert.assertTrue(e.getMessage().contains(Integer.toString(Response.Status.NOT_FOUND.getStatusCode())));
}
}
}
@Test
public void testDeleteEntitiesViaClientApi() throws Exception {
// Create 2 database entities
Referenceable db1 = new Referenceable(DATABASE_TYPE);
db1.set("name", randomString());
db1.set("description", randomString());
Id db1Id = createInstance(db1);
Referenceable db2 = new Referenceable(DATABASE_TYPE);
db2.set("name", randomString());
db2.set("description", randomString());
Id db2Id = createInstance(db2);
// Delete the database entities
List<String> deletedGuidsList = serviceClient.deleteEntities(db1Id._getId(), db2Id._getId());
// Verify that deleteEntities() response has database entity guids
Assert.assertEquals(deletedGuidsList.size(), 2);
Assert.assertTrue(deletedGuidsList.contains(db1Id._getId()));
Assert.assertTrue(deletedGuidsList.contains(db2Id._getId()));
// Verify entities were deleted from the repository.
for (String guid : deletedGuidsList) {
try {
serviceClient.getEntity(guid);
Assert.fail(AtlasServiceException.class.getSimpleName() +
" was expected but not thrown. The entity with guid " + guid +
" still exists in the repository after being deleted.");
}
catch (AtlasServiceException e) {
Assert.assertTrue(e.getMessage().contains(Integer.toString(Response.Status.NOT_FOUND.getStatusCode())));
}
}
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment