Skip to content

A
atlas
Commit aa67f8ae by Suma Shivaprasad
Options

ATLAS-1547 Add tests for DeleteHandlerV1 (sumasai via mneethiraj)

parent c7540b38
Showing with 1303 additions and 113 deletions
addons/models/0030-hive_model.json
......@@ -320,7 +320,7 @@
}
],
"isIndexable": false,
"isOptional": false,
"isOptional": true,
"isUnique": false
},
{
......@@ -483,7 +483,7 @@
}
],
"isIndexable": false,
"isOptional": false,
"isOptional": true,
"isUnique": false
}
]
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java
......@@ -33,6 +33,8 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
......@@ -189,4 +191,10 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
super(list, startIndex, pageSize, totalCount, sortType, sortBy);
}
}
@JsonIgnore
public AtlasObjectId getAtlasObjectId() {
return new AtlasObjectId(getGuid(), getTypeName());
}
}
intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
......@@ -104,7 +104,7 @@ public class AtlasStruct implements Serializable {
public boolean hasAttribute(String name) {
Map<String, Object> a = this.attributes;
return a != null ? a.containsKey(name) : null;
return a != null ? a.containsKey(name) : false;
}
public Object getAttribute(String name) {
......
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
......@@ -54,7 +54,7 @@ public class EntityMutationResponse {
this.entitiesMutated = opVsEntityMap;
}
public Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> getEntitiesMutated() {
public Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> getMutatedEntities() {
return entitiesMutated;
}
......@@ -69,6 +69,27 @@ public class EntityMutationResponse {
return null;
}
public List<AtlasEntityHeader> getCreatedEntities() {
if ( entitiesMutated != null) {
return entitiesMutated.get(EntityMutations.EntityOperation.CREATE);
}
return null;
}
public List<AtlasEntityHeader> getUpdatedEntities() {
if ( entitiesMutated != null) {
return entitiesMutated.get(EntityMutations.EntityOperation.UPDATE);
}
return null;
}
public List<AtlasEntityHeader> getDeletedEntities() {
if ( entitiesMutated != null) {
return entitiesMutated.get(EntityMutations.EntityOperation.DELETE);
}
return null;
}
@JsonIgnore
public AtlasEntityHeader getFirstEntityCreated() {
final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
......@@ -91,30 +112,42 @@ public class EntityMutationResponse {
@JsonIgnore
public AtlasEntityHeader getFirstCreatedEntityByTypeName(String typeName) {
final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
for (AtlasEntityHeader header : entitiesByOperation) {
if ( header.getTypeName().equals(typeName)) {
return header;
return getFirstEntityByType(getEntitiesByOperation(EntityMutations.EntityOperation.CREATE), typeName);
}
@JsonIgnore
public AtlasEntityHeader getFirstDeletedEntityByTypeName(String typeName) {
return getFirstEntityByType(getEntitiesByOperation(EntityMutations.EntityOperation.DELETE), typeName);
}
@JsonIgnore
public List<AtlasEntityHeader> getCreatedEntitiesByTypeName(String typeName) {
return getEntitiesByType(getEntitiesByOperation(EntityMutations.EntityOperation.CREATE), typeName);
}
return null;
@JsonIgnore
public AtlasEntityHeader getCreatedEntityByTypeNameAndAttribute(String typeName, String attrName, String attrVal) {
return getEntityByTypeAndUniqueAttribute(getEntitiesByOperation(EntityMutations.EntityOperation.CREATE), typeName, attrName, attrVal);
}
@JsonIgnore
public AtlasEntityHeader getFirstUpdatedEntityByTypeName(String typeName) {
final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE);
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
for (AtlasEntityHeader header : entitiesByOperation) {
if ( header.getTypeName().equals(typeName)) {
return header;
public AtlasEntityHeader getUpdatedEntityByTypeNameAndAttribute(String typeName, String attrName, String attrVal) {
return getEntityByTypeAndUniqueAttribute(getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE), typeName, attrName, attrVal);
}
@JsonIgnore
public List<AtlasEntityHeader> getUpdatedEntitiesByTypeName(String typeName) {
return getEntitiesByType(getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE), typeName);
}
@JsonIgnore
public List<AtlasEntityHeader> getDeletedEntitiesByTypeName(String typeName) {
return getEntitiesByType(getEntitiesByOperation(EntityMutations.EntityOperation.DELETE), typeName);
}
return null;
@JsonIgnore
public AtlasEntityHeader getFirstUpdatedEntityByTypeName(String typeName) {
return getFirstEntityByType(getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE), typeName);
}
public void addEntity(EntityMutations.EntityOperation op, AtlasEntityHeader header) {
......@@ -162,6 +195,42 @@ public class EntityMutationResponse {
return toString(new StringBuilder()).toString();
}
private AtlasEntityHeader getFirstEntityByType(List<AtlasEntityHeader> entitiesByOperation, String typeName) {
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
for (AtlasEntityHeader header : entitiesByOperation) {
if ( header.getTypeName().equals(typeName)) {
return header;
}
}
}
return null;
}
private List<AtlasEntityHeader> getEntitiesByType(List<AtlasEntityHeader> entitiesByOperation, String typeName) {
List<AtlasEntityHeader> ret = new ArrayList<>();
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
for (AtlasEntityHeader header : entitiesByOperation) {
if ( header.getTypeName().equals(typeName)) {
ret.add(header);
}
}
}
return ret;
}
private AtlasEntityHeader getEntityByTypeAndUniqueAttribute(List<AtlasEntityHeader> entitiesByOperation, String typeName, String attrName, String attrVal) {
if (entitiesByOperation != null && entitiesByOperation.size() > 0) {
for (AtlasEntityHeader header : entitiesByOperation) {
if (header.getTypeName().equals(typeName)) {
if (attrVal != null && attrVal.equals(header.getAttribute(attrName))) {
return header;
}
}
}
}
return null;
}
public void setGuidAssignments(Map<String,String> guidAssignments) {
this.guidAssignments = guidAssignments;
}
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
......@@ -390,8 +390,6 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
cDefs = new ArrayList<>();
this.constraints = cDefs;
} else {
cDefs = new ArrayList<>(cDefs);
}
cDefs.add(constraintDef);
......
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
......@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableSet;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
......@@ -45,6 +46,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.atlas.type.AtlasTypeUtil.createStructTypeDef;
......@@ -56,6 +58,8 @@ public final class TestUtilsV2 {
public static final long TEST_DATE_IN_LONG = 1418265358440L;
private static AtomicInteger seq = new AtomicInteger();
private TestUtilsV2() {
}
......@@ -87,7 +91,9 @@ public final class TestUtilsV2 {
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false,
new ArrayList<AtlasConstraintDef>()));
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}}));
AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
......@@ -111,7 +117,12 @@ public final class TestUtilsV2 {
new AtlasAttributeDef("manager", "Manager", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasConstraintDef>emptyList()),
new ArrayList<AtlasConstraintDef>() {{
add(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "subordinates");
}}));
}}),
new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
......@@ -162,7 +173,9 @@ public final class TestUtilsV2 {
AtlasTypeUtil.createOptionalAttrDef("dep-code", "string"),
new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false,
Collections.<AtlasConstraintDef>emptyList()));
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}}));
AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description,
ImmutableSet.<String>of(),
......@@ -190,7 +203,12 @@ public final class TestUtilsV2 {
new AtlasAttributeDef("manager", "Manager", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasConstraintDef>emptyList()),
new ArrayList<AtlasConstraintDef>() {{
add(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "subordinates");
}}));
}}),
new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
......@@ -239,7 +257,9 @@ public final class TestUtilsV2 {
AtlasTypeUtil.createRequiredAttrDef("dep-code", "string"),
new AtlasAttributeDef("employees", String.format("array<%s>", "Person"), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false,
Collections.<AtlasConstraintDef>emptyList()));
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}}));
AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(),
AtlasTypeUtil.createRequiredAttrDef("name", "string"),
......@@ -253,7 +273,12 @@ public final class TestUtilsV2 {
new AtlasAttributeDef("manager", "Manager", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasConstraintDef>emptyList()),
new ArrayList<AtlasConstraintDef>() {{
add(new AtlasConstraintDef(
AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "subordinates");
}}));
}}),
new AtlasAttributeDef("mentor", "Person", true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
......@@ -524,6 +549,7 @@ public final class TestUtilsV2 {
ImmutableList.of(newSuperTypeDefinition));
}
public static AtlasTypesDef defineHiveTypes() {
String _description = "_description";
AtlasEntityDef superTypeDefinition =
......@@ -557,8 +583,8 @@ public final class TestUtilsV2 {
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("type", "string"),
new AtlasAttributeDef("table", TABLE_TYPE,
false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(
......@@ -604,8 +630,8 @@ public final class TestUtilsV2 {
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("table", TABLE_TYPE, false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
new AtlasAttributeDef("table", TABLE_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasConstraintDef>emptyList()),
new AtlasAttributeDef("createTime", "long", true,
......@@ -686,17 +712,6 @@ public final class TestUtilsV2 {
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()
/* TODO - Fix map validation in type store and enable this
*
new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
add(new AtlasStructDef.AtlasConstraintDef(
AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF,
new HashMap<String, Object>() {{
put(AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, "table");
}}));
}}
*
*/
),
//map of structs
new AtlasAttributeDef("partitionsMap",
......@@ -748,8 +763,12 @@ public final class TestUtilsV2 {
}
public static AtlasEntity createDBEntity() {
AtlasEntity entity = new AtlasEntity(DATABASE_TYPE);
String dbName = RandomStringUtils.randomAlphanumeric(10);
return createDBEntity(dbName);
}
public static AtlasEntity createDBEntity(String dbName) {
AtlasEntity entity = new AtlasEntity(DATABASE_TYPE);
entity.setAttribute(NAME, dbName);
entity.setAttribute("description", "us db");
......@@ -766,9 +785,13 @@ public final class TestUtilsV2 {
}
public static AtlasEntity createTableEntity(AtlasEntity dbEntity) {
AtlasEntity entity = new AtlasEntity(TABLE_TYPE);
String tableName = RandomStringUtils.randomAlphanumeric(10);
entity.setAttribute(NAME, tableName);
return createTableEntity(dbEntity, tableName);
}
public static AtlasEntity createTableEntity(AtlasEntity dbEntity, String name) {
AtlasEntity entity = new AtlasEntity(TABLE_TYPE);
entity.setAttribute(NAME, name);
entity.setAttribute("description", "random table");
entity.setAttribute("type", "type");
entity.setAttribute("tableType", "MANAGED");
......@@ -813,14 +836,26 @@ public final class TestUtilsV2 {
}
public static AtlasEntity createColumnEntity(AtlasEntity tableEntity) {
return createColumnEntity(tableEntity, "col" + seq.addAndGet(1));
}
public static AtlasEntity createColumnEntity(AtlasEntity tableEntity, String colName) {
AtlasEntity entity = new AtlasEntity(COLUMN_TYPE);
entity.setAttribute(NAME, RandomStringUtils.randomAlphanumeric(10));
entity.setAttribute(NAME, colName);
entity.setAttribute("type", "VARCHAR(32)");
entity.setAttribute("table", tableEntity.getAtlasObjectId());
return entity;
}
public static AtlasEntity createProcessEntity(List<AtlasObjectId> inputs, List<AtlasObjectId> outputs) {
AtlasEntity entity = new AtlasEntity(PROCESS_TYPE);
entity.setAttribute(NAME, RandomStringUtils.randomAlphanumeric(10));
entity.setAttribute("inputs", inputs);
entity.setAttribute("outputs", outputs);
return entity;
}
public static List<AtlasClassificationDef> getClassificationWithValidSuperType() {
AtlasClassificationDef securityClearanceTypeDef =
AtlasTypeUtil.createTraitTypeDef("SecurityClearance1", "SecurityClearance_description", ImmutableSet.<String>of(),
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -540,8 +540,7 @@ public final class GraphHelper {
*
* @return propertyValue to AtlasVertex map with the result.
*/
public Map<String, AtlasVertex> getVerticesForPropertyValues(String property, List<String> values)
throws RepositoryException {
public Map<String, AtlasVertex> getVerticesForPropertyValues(String property, List<String> values) {
if(values.isEmpty()) {
return Collections.emptyMap();
......@@ -581,8 +580,7 @@ public final class GraphHelper {
*
* @return GUID to AtlasVertex map with the result.
*/
public Map<String, AtlasVertex> getVerticesForGUIDs(List<String> guids)
throws RepositoryException {
public Map<String, AtlasVertex> getVerticesForGUIDs(List<String> guids) {
return getVerticesForPropertyValues(Constants.GUID_PROPERTY_KEY, guids);
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
......@@ -21,24 +21,35 @@ package org.apache.atlas.repository.store.graph.v1;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RequestContext;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
......@@ -47,7 +58,6 @@ import java.util.Map;
public class AtlasEntityStoreV1 implements AtlasEntityStore {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV1.class);
private final DeleteHandlerV1 deleteHandler;
private final AtlasTypeRegistry typeRegistry;
......@@ -101,6 +111,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
@GraphTransaction
public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getByUniqueAttribute({}, {})", entityType.getTypeName(), uniqAttributes);
}
......@@ -157,23 +168,73 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "updateByUniqueAttributes() not implemented yet");
}
@GraphTransaction
public EntityMutationResponse deleteById(final String guid) throws AtlasBaseException {
if (StringUtils.isEmpty(guid)) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
}
// Retrieve vertices for requested guids.
AtlasVertex vertex = AtlasGraphUtilsV1.findByGuid(guid);
if (LOG.isDebugEnabled()) {
if (vertex == null) {
// Entity does not exist - treat as non-error, since the caller
// wanted to delete the entity and it's already gone.
LOG.debug("Deletion request ignored for non-existent entity with guid " + guid);
}
}
Collection<AtlasVertex> deletionCandidates = new ArrayList<AtlasVertex>();
deletionCandidates.add(vertex);
return deleteVertices(deletionCandidates);
}
@Override
@GraphTransaction
public EntityMutationResponse deleteById(String guid) throws AtlasBaseException {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "deleteById() not implemented yet");
public EntityMutationResponse deleteByIds(final List<String> guids) throws AtlasBaseException {
if (CollectionUtils.isEmpty(guids)) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guids);
}
Collection<AtlasVertex> deletionCandidates = new ArrayList<>();
for (String guid : guids) {
// Retrieve vertices for requested guids.
AtlasVertex vertex = AtlasGraphUtilsV1.findByGuid(guid);
if (LOG.isDebugEnabled()) {
if (vertex == null) {
// Entity does not exist - treat as non-error, since the caller
// wanted to delete the entity and it's already gone.
LOG.debug("Deletion request ignored for non-existent entity with guid " + guid);
}
}
deletionCandidates.add(vertex);
}
if (deletionCandidates.isEmpty()) {
LOG.info("No deletion candidate entities were found for guids %s", guids);
}
return deleteVertices(deletionCandidates);
}
@Override
@GraphTransaction
public EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
throws AtlasBaseException {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "deleteByUniqueAttributes() not implemented yet");
if (MapUtils.isEmpty(uniqAttributes)) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, uniqAttributes.toString());
}
@Override
@GraphTransaction
public EntityMutationResponse deleteByIds(List<String> guids) throws AtlasBaseException {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "deleteByIds() not implemented yet");
final AtlasVertex vertex = AtlasGraphUtilsV1.findByUniqueAttributes(entityType, uniqAttributes);
Collection<AtlasVertex> deletionCandidates = new ArrayList<>();
deletionCandidates.add(vertex);
return deleteVertices(deletionCandidates);
}
@Override
......@@ -216,8 +277,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
}
context.addUpdated(guid, entity, entityType, vertex);
RequestContextV1.get().recordEntityUpdate(entity.getAtlasObjectId());
}
} else {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
......@@ -233,10 +292,24 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
context.addCreated(guid, entity, entityType, vertex);
RequestContextV1.get().recordEntityCreate(entity.getAtlasObjectId());
}
}
return context;
}
private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCandidates) throws AtlasBaseException {
EntityMutationResponse response = new EntityMutationResponse();
deleteHandler.deleteEntities(deletionCandidates);
RequestContextV1 req = RequestContextV1.get();
for (AtlasObjectId id : req.getDeletedEntityIds()) {
response.addEntity(EntityMutations.EntityOperation.DELETE, EntityGraphMapper.constructHeader(id));
}
for (AtlasObjectId id : req.getUpdatedEntityIds()) {
response.addEntity(EntityMutations.EntityOperation.UPDATE, EntityGraphMapper.constructHeader(id));
}
return response;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -25,7 +25,6 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.AtlasEdgeLabel;
......@@ -34,10 +33,10 @@ import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.slf4j.Logger;
......@@ -48,6 +47,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.Stack;
......@@ -60,14 +60,14 @@ public abstract class DeleteHandlerV1 {
public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class);
private AtlasTypeRegistry typeRegistry;
private boolean shouldUpdateReverseAttribute;
private boolean shouldUpdateInverseReferences;
private boolean softDelete;
protected static final GraphHelper graphHelper = GraphHelper.getInstance();
public DeleteHandlerV1(AtlasTypeRegistry typeRegistry, boolean shouldUpdateReverseAttribute, boolean softDelete) {
public DeleteHandlerV1(AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete) {
this.typeRegistry = typeRegistry;
this.shouldUpdateReverseAttribute = shouldUpdateReverseAttribute;
this.shouldUpdateInverseReferences = shouldUpdateInverseReference;
this.softDelete = softDelete;
}
......@@ -79,21 +79,30 @@ public abstract class DeleteHandlerV1 {
* @param instanceVertices
* @throws AtlasException
*/
public void deleteEntities(List<AtlasVertex> instanceVertices) throws AtlasBaseException {
public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasBaseException {
RequestContextV1 requestContext = RequestContextV1.get();
Set<AtlasVertex> deletionCandidateVertices = new HashSet<>();
for (AtlasVertex instanceVertex : instanceVertices) {
String guid = GraphHelper.getGuid(instanceVertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(instanceVertex);
AtlasEntity.Status state = AtlasGraphUtilsV1.getState(instanceVertex);
if (requestContext.getDeletedEntityIds().contains(guid) || state == AtlasEntity.Status.DELETED) {
if (state == AtlasEntity.Status.DELETED) {
LOG.debug("Skipping deletion of {} as it is already deleted", guid);
continue;
}
String typeName = AtlasGraphUtilsV1.getTypeName(instanceVertex);
AtlasObjectId objId = new AtlasObjectId(guid, typeName);
if (requestContext.getDeletedEntityIds().contains(objId)) {
LOG.debug("Skipping deletion of {} as it is already deleted", guid);
continue;
}
// Get GUIDs and vertices for all deletion candidates.
Set<GraphHelper.VertexInfo> compositeVertices = getCompositeVertices(instanceVertex);
Set<GraphHelper.VertexInfo> compositeVertices = getOwnedVertices(instanceVertex);
// Record all deletion candidate GUIDs in RequestContext
// and gather deletion candidate vertices.
......@@ -118,19 +127,22 @@ public abstract class DeleteHandlerV1 {
* @return set of VertexInfo for all composite entities
* @throws AtlasException
*/
public Set<GraphHelper.VertexInfo> getCompositeVertices(AtlasVertex entityVertex) throws AtlasBaseException {
Set<GraphHelper.VertexInfo> result = new HashSet<>();
public Set<GraphHelper.VertexInfo> getOwnedVertices(AtlasVertex entityVertex) throws AtlasBaseException {
Set<GraphHelper.VertexInfo> result = new LinkedHashSet<>();
Stack<AtlasVertex> vertices = new Stack<>();
vertices.push(entityVertex);
while (vertices.size() > 0) {
AtlasVertex vertex = vertices.pop();
String typeName = GraphHelper.getTypeName(vertex);
String guid = GraphHelper.getGuid(vertex);
AtlasEntity.Status state = AtlasGraphUtilsV1.getState(vertex);
if (state == AtlasEntity.Status.DELETED) {
//If the reference vertex is marked for deletion, skip it
continue;
}
String typeName = GraphHelper.getTypeName(vertex);
String guid = GraphHelper.getGuid(vertex);
result.add(new GraphHelper.VertexInfo(guid, vertex, typeName));
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
......@@ -143,7 +155,7 @@ public abstract class DeleteHandlerV1 {
continue;
}
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(entityType, attributeInfo.getName());
AtlasType attrType = typeRegistry.getType(attributeInfo.getTypeName());
AtlasType attrType = attributeInfo.getAttributeType();
switch (attrType.getTypeCategory()) {
case OBJECT_ID_TYPE:
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
......@@ -199,19 +211,19 @@ public abstract class DeleteHandlerV1 {
* Force delete is used to remove struct/trait in case of entity updates
* @param edge
* @param typeCategory
* @param isComposite
* @param isOwned
* @param forceDeleteStructTrait
* @return returns true if the edge reference is hard deleted
* @throws AtlasException
*/
public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isComposite,
public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned,
boolean forceDeleteStructTrait) throws AtlasBaseException {
LOG.debug("Deleting {}", string(edge));
boolean forceDelete =
(typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait;
if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION
|| (typeCategory == TypeCategory.OBJECT_ID_TYPE && isComposite)) {
|| (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) {
//If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities.
//If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled
//through this delete, hence delete the edge and the reference vertex.
......@@ -232,18 +244,19 @@ public abstract class DeleteHandlerV1 {
}
protected void deleteEdge(AtlasEdge edge, boolean updateReverseAttribute, boolean force) throws AtlasBaseException {
//update reverse attribute
//update inverse attribute
if (updateReverseAttribute) {
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edge.getLabel());
AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName());
if (parentType instanceof AtlasStructType) {
AtlasStructType parentStructType = (AtlasStructType) parentType;
//TODO - Fix this later
// if (parentStructType.isForeignKeyAttribute(atlasEdgeLabel.getAttributeName())) {
// deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), atlasEdgeLabel.getAttributeName());
// }
if (parentType instanceof AtlasEntityType) {
AtlasEntityType parentEntityType = (AtlasEntityType) parentType;
AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName());
if (attribute.getInverseRefAttribute() != null) {
deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute());
}
}
}
......@@ -286,16 +299,16 @@ public abstract class DeleteHandlerV1 {
for (AtlasStructType.AtlasAttribute attributeInfo : structType.getAllAttributes().values()) {
LOG.debug("Deleting attribute {} for {}", attributeInfo.getName(), string(instanceVertex));
boolean isComposite = isEntityType && attributeInfo.isOwnedRef();
boolean isOwned = isEntityType && attributeInfo.isOwnedRef();
AtlasType attrType = typeRegistry.getType(attributeInfo.getTypeName());
AtlasType attrType = attributeInfo.getAttributeType();
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(structType, attributeInfo.getName());
switch (attrType.getTypeCategory()) {
case OBJECT_ID_TYPE:
//If its class attribute, delete the reference
deleteEdgeReference(instanceVertex, edgeLabel, attrType.getTypeCategory(), isComposite);
deleteEdgeReference(instanceVertex, edgeLabel, attrType.getTypeCategory(), isOwned);
break;
case STRUCT:
......@@ -312,7 +325,7 @@ public abstract class DeleteHandlerV1 {
if (edges != null) {
while (edges.hasNext()) {
AtlasEdge edge = edges.next();
deleteEdgeReference(edge, elemType.getTypeCategory(), isComposite, false);
deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false);
}
}
}
......@@ -330,7 +343,7 @@ public abstract class DeleteHandlerV1 {
if (keys != null) {
for (Object key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, (String) key);
deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, isComposite);
deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, isOwned);
}
}
}
......@@ -342,10 +355,10 @@ public abstract class DeleteHandlerV1 {
}
public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory,
boolean isComposite) throws AtlasBaseException {
boolean isOwned) throws AtlasBaseException {
AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (edge != null) {
deleteEdgeReference(edge, typeCategory, isComposite, false);
deleteEdgeReference(edge, typeCategory, isOwned, false);
}
}
......@@ -390,8 +403,11 @@ public abstract class DeleteHandlerV1 {
attributeName);
String typeName = GraphHelper.getTypeName(outVertex);
String outId = GraphHelper.getGuid(outVertex);
AtlasObjectId objId = new AtlasObjectId(outId, typeName);
AtlasEntity.Status state = AtlasGraphUtilsV1.getState(outVertex);
if ((outId != null && RequestContextV1.get().isDeletedEntity(outId)) || state == AtlasEntity.Status.DELETED) {
if (state == AtlasEntity.Status.DELETED || (outId != null && RequestContextV1.get().isDeletedEntity(objId))) {
//If the reference vertex is marked for deletion, skip updating the reference
return;
}
......@@ -401,21 +417,22 @@ public abstract class DeleteHandlerV1 {
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
AtlasEdge edge = null;
AtlasAttribute attribute = parentType.getAttribute(attributeName);
AtlasAttributeDef attrDef = parentType.getAttributeDef(attributeName);
AtlasType attrType = typeRegistry.getType(attrDef.getTypeName());
AtlasType attrType = attribute.getAttributeType();
switch (attrType.getTypeCategory()) {
case OBJECT_ID_TYPE:
//If its class attribute, its the only edge between two vertices
if (attrDef.getIsOptional()) {
edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (shouldUpdateReverseAttribute) {
if (shouldUpdateInverseReferences) {
GraphHelper.setProperty(outVertex, propertyName, null);
}
} else {
// Cannot unset a required attribute.
throw new AtlasBaseException("Cannot unset required attribute " + propertyName +
" on " + GraphHelper.getVertexDetails(outVertex) + " edge = " + edgeLabel);
" on " + GraphHelper.vertexString(outVertex) + " edge = " + edgeLabel);
}
break;
......@@ -444,7 +461,7 @@ public abstract class DeleteHandlerV1 {
+ GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge));
}
if (shouldUpdateReverseAttribute) {
if (shouldUpdateInverseReferences) {
//if composite attribute, remove the reference as well. else, just remove the edge
//for example, when table is deleted, process still references the table
//but when column is deleted, table will not reference the deleted column
......@@ -485,7 +502,7 @@ public abstract class DeleteHandlerV1 {
propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge));
}
if (shouldUpdateReverseAttribute) {
if (shouldUpdateInverseReferences) {
//remove this key
LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key,
attributeName);
......@@ -506,7 +523,7 @@ public abstract class DeleteHandlerV1 {
default:
throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to "
+ GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attributeName + " which is not class/array/map attribute");
+ GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attributeName + " which is not class/array/map attribute. found " + attrType.getTypeCategory().name());
}
if (edge != null) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
......@@ -139,6 +139,10 @@ public class EntityGraphMapper {
resp.addEntity(DELETE, constructHeader(id));
}
for (AtlasObjectId id : req.getUpdatedEntityIds()) {
resp.addEntity(UPDATE, constructHeader(id));
}
return resp;
}
......@@ -744,7 +748,7 @@ public class EntityGraphMapper {
return header;
}
private AtlasEntityHeader constructHeader(AtlasObjectId id) {
public static AtlasEntityHeader constructHeader(AtlasObjectId id) {
AtlasEntityHeader entity = new AtlasEntityHeader(id.getTypeName());
entity.setGuid(id.getGuid());
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java
......@@ -28,7 +28,7 @@ public class HardDeleteHandlerV1 extends DeleteHandlerV1 {
@Inject
public HardDeleteHandlerV1(AtlasTypeRegistry typeRegistry) {
super(typeRegistry, false, true);
super(typeRegistry, true, false);
}
@Override
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.repository.store.graph.v1;
import com.google.inject.Inject;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.graph.GraphHelper;
......@@ -46,10 +47,10 @@ public class SoftDeleteHandlerV1 extends DeleteHandlerV1 {
} else {
AtlasEntity.Status state = AtlasGraphUtilsV1.getState(instanceVertex);
if (state != AtlasEntity.Status.DELETED) {
GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, AtlasEntity.Status.DELETED.name());
GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
GraphHelper.setProperty(instanceVertex, MODIFIED_BY_KEY, RequestContext.get().getUser());
RequestContextV1.get().getRequestTime());
GraphHelper.setProperty(instanceVertex, MODIFIED_BY_KEY, RequestContextV1.get().getUser());
}
}
}
......@@ -59,12 +60,12 @@ public class SoftDeleteHandlerV1 extends DeleteHandlerV1 {
if (force) {
graphHelper.removeEdge(edge);
} else {
Id.EntityState state = GraphHelper.getState(edge);
if (state != Id.EntityState.DELETED) {
GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
AtlasEntity.Status state = AtlasGraphUtilsV1.getState(edge);
if (state != AtlasEntity.Status.DELETED) {
GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, AtlasEntity.Status.DELETED.name());
GraphHelper
.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
GraphHelper.setProperty(edge, MODIFIED_BY_KEY, RequestContext.get().getUser());
.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContextV1.get().getRequestTime());
GraphHelper.setProperty(edge, MODIFIED_BY_KEY, RequestContextV1.get().getUser());
}
}
}
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestUtils;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtils.COLUMN_TYPE;
import static org.apache.atlas.TestUtils.DEPARTMENT_TYPE;
import static org.apache.atlas.TestUtils.NAME;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@Guice(modules = RepositoryMetadataModule.class)
public abstract class AtlasDeleteHandlerV1Test {
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
AtlasTypeDefStore typeDefStore;
AtlasEntityStore entityStore;
@Inject
MetadataService metadataService;
private AtlasEntityType compositeMapOwnerType;
private AtlasEntityType compositeMapValueType;
private TypeSystem typeSystem = TypeSystem.getInstance();
@BeforeClass
public void setUp() throws Exception {
metadataService = TestUtils.addSessionCleanupWrapper(metadataService);
new GraphBackedSearchIndexer(typeRegistry);
final AtlasTypesDef deptTypesDef = TestUtilsV2.defineDeptEmployeeTypes();
typeDefStore.createTypesDef(deptTypesDef);
final AtlasTypesDef hiveTypesDef = TestUtilsV2.defineHiveTypes();
typeDefStore.createTypesDef(hiveTypesDef);
// Define type for map value.
AtlasEntityDef mapValueDef = AtlasTypeUtil.createClassTypeDef("CompositeMapValue", "CompositeMapValue" + "_description", "1.0",
ImmutableSet.<String>of(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string")
);
// Define type with map where the value is a composite class reference to MapValue.
AtlasEntityDef mapOwnerDef = AtlasTypeUtil.createClassTypeDef("CompositeMapOwner", "CompositeMapOwner_description",
ImmutableSet.<String>of(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createOptionalAttrDef("map", "map<string,string>")
);
final AtlasTypesDef typesDef = AtlasTypeUtil.getTypesDef(ImmutableList.<AtlasEnumDef>of(),
ImmutableList.<AtlasStructDef>of(),
ImmutableList.<AtlasClassificationDef>of(),
ImmutableList.of(mapValueDef, mapOwnerDef));
typeDefStore.createTypesDef(typesDef);
compositeMapOwnerType = typeRegistry.getEntityTypeByName("CompositeMapOwner");
compositeMapValueType = typeRegistry.getEntityTypeByName("CompositeMapValue");
}
@BeforeTest
public void init() throws Exception {
final Class<? extends DeleteHandlerV1> deleteHandlerImpl = AtlasRepositoryConfiguration.getDeleteHandlerV1Impl();
final Constructor<? extends DeleteHandlerV1> deleteHandlerImplConstructor = deleteHandlerImpl.getConstructor(AtlasTypeRegistry.class);
DeleteHandlerV1 deleteHandler = deleteHandlerImplConstructor.newInstance(typeRegistry);
entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry);
RequestContextV1.clear();
}
@AfterClass
public void clear() {
AtlasGraphProvider.cleanup();
}
abstract DeleteHandlerV1 getDeleteHandler(AtlasTypeRegistry typeRegistry);
@Test
public void testDeleteAndCreate() throws Exception {
init();
final AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity));
init();
//delete entity should mark it as deleted
EntityMutationResponse deleteResponse = entityStore.deleteById(response.getFirstEntityCreated().getGuid());
AtlasEntityHeader dbEntityCreated = response.getFirstEntityCreated();
assertEquals(deleteResponse.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE).get(0).getGuid(), dbEntityCreated.getGuid());
//get entity by unique attribute should throw EntityNotFoundException
try {
metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "name", (String) response.getFirstEntityCreated().getAttribute("name"));
fail("Expected EntityNotFoundException");
} catch(EntityNotFoundException e) {
//expected
}
init();
//Create the same entity again, should create new entity
AtlasEntity newDBEntity = TestUtilsV2.createDBEntity((String) dbEntity.getAttribute(NAME));
EntityMutationResponse newCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(newDBEntity));
assertNotEquals(newCreationResponse.getFirstEntityCreated().getGuid(), response.getFirstEntityCreated().getGuid());
//TODO - Enable after GET is ready
//get by unique attribute should return the new entity
ITypedReferenceableInstance instance = metadataService.getEntityDefinitionReference(TestUtils.DATABASE_TYPE, "name", (String) dbEntity.getAttribute("name"));
assertEquals(instance.getId()._getId(), newCreationResponse.getFirstEntityCreated().getGuid());
}
@Test
public void testDeleteReference() throws Exception {
//Deleting column should update table
final AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
init();
EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity));
final AtlasEntity tableEntity = TestUtilsV2.createTableEntity(dbEntity);
final AtlasEntity columnEntity = TestUtilsV2.createColumnEntity(tableEntity);
tableEntity.setAttribute(COLUMNS_ATTR_NAME, Arrays.asList(columnEntity.getAtlasObjectId()));
AtlasEntity.AtlasEntityWithExtInfo input = new AtlasEntity.AtlasEntityWithExtInfo(tableEntity);
input.addReferredEntity(columnEntity);
init();
EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(input));
final AtlasEntityHeader columnCreated = tblCreationResponse.getFirstCreatedEntityByTypeName(COLUMN_TYPE);
final AtlasEntityHeader tableCreated = tblCreationResponse.getFirstCreatedEntityByTypeName(TABLE_TYPE);
init();
EntityMutationResponse deletionResponse = entityStore.deleteById(columnCreated.getGuid());
assertEquals(deletionResponse.getDeletedEntities().size(), 1);
assertEquals(deletionResponse.getDeletedEntities().get(0).getGuid(), columnCreated.getGuid());
assertEquals(deletionResponse.getUpdatedEntities().size(), 1);
assertEquals(deletionResponse.getUpdatedEntities().get(0).getGuid(), tableCreated.getGuid());
assertEntityDeleted(columnCreated.getGuid());
//TODO - Fix after GET is ready
// ITypedReferenceableInstance tableInstance = repositoryService.getEntityDefinition(tableId);
// assertColumnForTestDeleteReference(tableInstance);
//Deleting table should update process
AtlasEntity process = TestUtilsV2.createProcessEntity(null, Arrays.asList(tableCreated.getAtlasObjectId()));
init();
final EntityMutationResponse processCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(process));
init();
entityStore.deleteById(tableCreated.getGuid());
assertEntityDeleted(tableCreated.getGuid());
assertTableForTestDeleteReference(tableCreated.getGuid());
assertProcessForTestDeleteReference(processCreationResponse.getFirstEntityCreated());
}
@Test
public void testDeleteEntities() throws Exception {
// Create a table entity, with 3 composite column entities
init();
final AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity));
final AtlasEntity tableEntity = TestUtilsV2.createTableEntity(dbEntity);
AtlasEntity.AtlasEntitiesWithExtInfo entitiesInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(tableEntity);
final AtlasEntity columnEntity1 = TestUtilsV2.createColumnEntity(tableEntity);
entitiesInfo.addReferredEntity(columnEntity1);
final AtlasEntity columnEntity2 = TestUtilsV2.createColumnEntity(tableEntity);
entitiesInfo.addReferredEntity(columnEntity2);
final AtlasEntity columnEntity3 = TestUtilsV2.createColumnEntity(tableEntity);
entitiesInfo.addReferredEntity(columnEntity3);
tableEntity.setAttribute(COLUMNS_ATTR_NAME, Arrays.asList(columnEntity1.getAtlasObjectId(), columnEntity2.getAtlasObjectId(), columnEntity3.getAtlasObjectId()));
init();
final EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo));
final AtlasEntityHeader column1Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity1.getAttribute(NAME));
final AtlasEntityHeader column2Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity2.getAttribute(NAME));
final AtlasEntityHeader column3Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity3.getAttribute(NAME));
// Retrieve the table entities from the Repository, to get their guids and the composite column guids.
ITypedReferenceableInstance tableInstance = metadataService.getEntityDefinitionReference(TestUtils.TABLE_TYPE, NAME, (String) tableEntity.getAttribute(NAME));
List<IReferenceableInstance> columns = (List<IReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME);
//Delete column
String colId = columns.get(0).getId()._getId();
String tableId = tableInstance.getId()._getId();
init();
EntityMutationResponse deletionResponse = entityStore.deleteById(colId);
assertEquals(deletionResponse.getDeletedEntities().size(), 1);
assertEquals(deletionResponse.getDeletedEntities().get(0).getGuid(), colId);
assertEquals(deletionResponse.getUpdatedEntities().size(), 1);
assertEquals(deletionResponse.getUpdatedEntities().get(0).getGuid(), tableId);
assertEntityDeleted(colId);
tableInstance = metadataService.getEntityDefinitionReference(TestUtils.TABLE_TYPE, NAME, (String) tableEntity.getAttribute(NAME));
assertDeletedColumn(tableInstance);
assertTestDisconnectUnidirectionalArrayReferenceFromClassType(
(List<ITypedReferenceableInstance>) tableInstance.get("columns"), colId);
//update by removing a column - col1
final AtlasEntity tableEntity1 = TestUtilsV2.createTableEntity(dbEntity, (String) tableEntity.getAttribute(NAME));
AtlasEntity.AtlasEntitiesWithExtInfo entitiesInfo1 = new AtlasEntity.AtlasEntitiesWithExtInfo(tableEntity1);
final AtlasEntity columnEntity3New = TestUtilsV2.createColumnEntity(tableEntity1, (String) column3Created.getAttribute(NAME));
tableEntity1.setAttribute(COLUMNS_ATTR_NAME, Arrays.asList(columnEntity3New.getAtlasObjectId()));
entitiesInfo1.addReferredEntity(columnEntity3New);
init();
deletionResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo1));
//TODO - enable after fixing unique atribute resolver
assertEquals(deletionResponse.getDeletedEntities().size(), 1);
assertEquals(deletionResponse.getDeletedEntities().get(0).getGuid(), column2Created.getGuid());
assertEntityDeleted(colId);
// Delete the table entities. The deletion should cascade to their composite columns.
tableInstance = metadataService.getEntityDefinitionReference(TestUtils.TABLE_TYPE, NAME, (String) tableEntity.getAttribute(NAME));
init();
EntityMutationResponse tblDeletionResponse = entityStore.deleteById(tableInstance.getId()._getId());
assertEquals(tblDeletionResponse.getDeletedEntities().size(), 2);
final AtlasEntityHeader tableDeleted = tblDeletionResponse.getFirstDeletedEntityByTypeName(TABLE_TYPE);
final AtlasEntityHeader colDeleted = tblDeletionResponse.getFirstDeletedEntityByTypeName(COLUMN_TYPE);
// Verify that deleteEntities() response has guids for tables and their composite columns.
Assert.assertTrue(tableDeleted.getGuid().equals(tableInstance.getId()._getId()));
Assert.assertTrue(colDeleted.getGuid().equals(column3Created.getGuid()));
// Verify that tables and their composite columns have been deleted from the graph Repository.
assertEntityDeleted(tableDeleted.getGuid());
assertEntityDeleted(colDeleted.getGuid());
assertTestDeleteEntities(tableInstance);
}
protected abstract void assertDeletedColumn(ITypedReferenceableInstance tableInstance) throws AtlasException;
protected abstract void assertTestDeleteEntities(ITypedReferenceableInstance tableInstance) throws Exception;
protected abstract void assertTableForTestDeleteReference(String tableId) throws Exception;
protected abstract void assertColumnForTestDeleteReference(AtlasEntity tableInstance)
throws AtlasException;
protected abstract void assertProcessForTestDeleteReference(AtlasEntityHeader processInstance) throws Exception;
protected abstract void assertEntityDeleted(String id) throws Exception;
String getFirstGuid(Map<String, AtlasEntity> entityMap) {
return entityMap.keySet().iterator().next();
}
@Test
public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
AtlasEntity.AtlasEntitiesWithExtInfo hrDept = TestUtilsV2.createDeptEg2();
init();
RequestContextV1.clear();
final EntityMutationResponse hrDeptCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(hrDept));
final AtlasEntityHeader deptCreated = hrDeptCreationResponse.getFirstUpdatedEntityByTypeName(DEPARTMENT_TYPE);
final AtlasEntityHeader maxEmployeeCreated = hrDeptCreationResponse.getCreatedEntityByTypeNameAndAttribute(TestUtilsV2.EMPLOYEE_TYPE, NAME, "Max");
final AtlasEntityHeader johnEmployeeCreated = hrDeptCreationResponse.getUpdatedEntityByTypeNameAndAttribute(TestUtilsV2.EMPLOYEE_TYPE, NAME, "John");
final AtlasEntityHeader janeEmployeeCreated = hrDeptCreationResponse.getCreatedEntityByTypeNameAndAttribute(TestUtilsV2.MANAGER_TYPE, NAME, "Jane");
final AtlasEntityHeader juliusEmployeeCreated = hrDeptCreationResponse.getUpdatedEntityByTypeNameAndAttribute(TestUtilsV2.MANAGER_TYPE, NAME, "Julius");
// ITypedReferenceableInstance hrDeptInstance = metadataService.getEntityDefinition(hrDeptCreationResponse.getFirstCreatedEntityByTypeName(DEPARTMENT_TYPE).getGuid());
// Map<String, String> nameGuidMap = getEmployeeNameGuidMap(hrDeptInstance);
ITypedReferenceableInstance max = metadataService.getEntityDefinition(maxEmployeeCreated.getGuid());
String maxGuid = max.getId()._getId();
AtlasVertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long creationTimestamp = GraphHelper.getSingleValuedProperty(vertex, Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(creationTimestamp);
Long modificationTimestampPreUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPreUpdate);
AtlasEntity maxEmployee = getEmployeeByName(hrDept, "Max");
maxEmployee.setAttribute("mentor", johnEmployeeCreated.getAtlasObjectId());
maxEmployee.setAttribute("department", deptCreated.getAtlasObjectId());
maxEmployee.setAttribute("manager", janeEmployeeCreated.getAtlasObjectId());
init();
EntityMutationResponse entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee));
assertEquals(entityResult.getUpdatedEntities().size(), 1);
assertTrue(extractGuids(entityResult.getUpdatedEntities()).contains(maxGuid));
// Verify the update was applied correctly - john should now be max's mentor.
max = metadataService.getEntityDefinition(maxGuid);
ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) max.get("mentor");
Assert.assertEquals(refTarget.getId()._getId(), johnEmployeeCreated.getGuid());
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPostUpdate);
Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
// Update max's mentor reference to jane.
maxEmployee.setAttribute("mentor", janeEmployeeCreated.getAtlasObjectId());
init();
entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee));
assertEquals(entityResult.getUpdatedEntities().size(), 1);
assertTrue(extractGuids(entityResult.getUpdatedEntities()).contains(maxGuid));
// Verify the update was applied correctly - jane should now be max's mentor.
max = metadataService.getEntityDefinition(maxGuid);
refTarget = (ITypedReferenceableInstance) max.get("mentor");
Assert.assertEquals(refTarget.getId()._getId(), janeEmployeeCreated.getGuid());
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long modificationTimestampPost2ndUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPost2ndUpdate);
Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
ITypedReferenceableInstance julius = metadataService.getEntityDefinition(juliusEmployeeCreated.getGuid());
Id juliusGuid = julius.getId();
init();
maxEmployee.setAttribute("manager", juliusEmployeeCreated.getAtlasObjectId());
entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee));
//TODO ATLAS-499 should have updated julius' subordinates
assertEquals(entityResult.getUpdatedEntities().size(), 2);
assertTrue(extractGuids(entityResult.getUpdatedEntities()).contains(maxGuid));
assertTrue(extractGuids(entityResult.getUpdatedEntities()).contains(janeEmployeeCreated.getGuid()));
// Verify the update was applied correctly - julius should now be max's manager.
max = metadataService.getEntityDefinition(maxGuid);
refTarget = (ITypedReferenceableInstance) max.get("manager");
Assert.assertEquals(refTarget.getId()._getId(), juliusGuid._getId());
assertTestUpdateEntity_MultiplicityOneNonCompositeReference(janeEmployeeCreated.getGuid());
}
private Map<String, String> getEmployeeNameGuidMap(final ITypedReferenceableInstance hrDept) throws AtlasException {
Object refValue = hrDept.get("employees");
Assert.assertTrue(refValue instanceof List);
List<Object> employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 4);
Map<String, String> nameGuidMap = new HashMap<String, String>() {{
put("hr", hrDept.getId()._getId());
}};
for (Object listValue : employees) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
nameGuidMap.put((String)employee.get("name"), employee.getId()._getId());
}
return nameGuidMap;
}
private AtlasEntity getEmployeeByName(AtlasEntity.AtlasEntitiesWithExtInfo hrDept, String name) {
for (AtlasEntity entity : hrDept.getEntities()) {
if ( name.equals(entity.getAttribute(NAME))) {
return entity;
}
}
return null;
}
//
protected abstract void assertTestUpdateEntity_MultiplicityOneNonCompositeReference(String janeGuid) throws Exception;
/**
* Verify deleting an entity which is contained by another
* entity through a bi-directional composite reference.
*
* @throws Exception
*/
@Test
public void testDisconnectBidirectionalReferences() throws Exception {
AtlasEntity.AtlasEntitiesWithExtInfo hrDept = TestUtilsV2.createDeptEg2();
init();
final EntityMutationResponse hrDeptCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(hrDept));
final AtlasEntityHeader deptCreated = hrDeptCreationResponse.getFirstCreatedEntityByTypeName(DEPARTMENT_TYPE);
final AtlasEntityHeader maxEmployee = hrDeptCreationResponse.getCreatedEntityByTypeNameAndAttribute(TestUtilsV2.EMPLOYEE_TYPE, NAME, "Max");
final AtlasEntityHeader johnEmployee = hrDeptCreationResponse.getCreatedEntityByTypeNameAndAttribute(TestUtilsV2.EMPLOYEE_TYPE, NAME, "John");
final AtlasEntityHeader janeEmployee = hrDeptCreationResponse.getCreatedEntityByTypeNameAndAttribute(TestUtilsV2.MANAGER_TYPE, NAME, "Jane");
final AtlasEntityHeader juliusEmployee = hrDeptCreationResponse.getCreatedEntityByTypeNameAndAttribute(TestUtilsV2.MANAGER_TYPE, NAME, "Julius");
ITypedReferenceableInstance hrDeptInstance = metadataService.getEntityDefinition(deptCreated.getGuid());
Map<String, String> nameGuidMap = getEmployeeNameGuidMap(hrDeptInstance);
// Verify that Max is one of Jane's subordinates.
ITypedReferenceableInstance jane = metadataService.getEntityDefinition(janeEmployee.getGuid());
Object refValue = jane.get("subordinates");
Assert.assertTrue(refValue instanceof List);
List<Object> subordinates = (List<Object>)refValue;
Assert.assertEquals(subordinates.size(), 2);
List<String> subordinateIds = new ArrayList<>(2);
for (Object listValue : subordinates) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
subordinateIds.add(employee.getId()._getId());
}
Assert.assertTrue(subordinateIds.contains(maxEmployee.getGuid()));
init();
EntityMutationResponse entityResult = entityStore.deleteById(maxEmployee.getGuid());
ITypedReferenceableInstance john = metadataService.getEntityDefinitionReference(TestUtilsV2.EMPLOYEE_TYPE, NAME, "John");
assertEquals(entityResult.getDeletedEntities().size(), 1);
assertEquals(entityResult.getDeletedEntities().get(0).getGuid(), maxEmployee.getGuid());
assertEquals(entityResult.getUpdatedEntities().size(), 3);
assertEquals(extractGuids(entityResult.getUpdatedEntities()), Arrays.asList(janeEmployee.getGuid(), deptCreated.getGuid(), johnEmployee.getGuid()));
assertEntityDeleted(maxEmployee.getGuid());
assertMaxForTestDisconnectBidirectionalReferences(nameGuidMap);
// Now delete jane - this should disconnect the manager reference from her
// subordinate.
init();
entityResult = entityStore.deleteById(janeEmployee.getGuid());
assertEquals(entityResult.getDeletedEntities().size(), 1);
assertEquals(entityResult.getDeletedEntities().get(0).getGuid(), janeEmployee.getGuid());
assertEquals(entityResult.getUpdatedEntities().size(), 2);
assertEquals(extractGuids(entityResult.getUpdatedEntities()), Arrays.asList(deptCreated.getGuid(), johnEmployee.getGuid()));
assertEntityDeleted(janeEmployee.getGuid());
john = metadataService.getEntityDefinitionReference(TestUtilsV2.EMPLOYEE_TYPE, NAME, "John");
assertJohnForTestDisconnectBidirectionalReferences(john, janeEmployee.getGuid());
}
protected List<String> extractGuids(final List<AtlasEntityHeader> updatedEntities) {
List<String> guids = new ArrayList<>();
for (AtlasEntityHeader header : updatedEntities ) {
guids.add(header.getGuid());
}
return guids;
}
protected abstract void assertJohnForTestDisconnectBidirectionalReferences(ITypedReferenceableInstance john,
String janeGuid) throws Exception;
protected abstract void assertMaxForTestDisconnectBidirectionalReferences(Map<String, String> nameGuidMap)
throws Exception;
protected abstract void assertTestDisconnectUnidirectionalArrayReferenceFromClassType(
List<ITypedReferenceableInstance> columns, String columnGuid);
/**
* Verify deleting entities that are the target of a unidirectional class array reference
* from a struct or trait instance.
*/
@Test
public void testDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes() throws Exception {
// Define class types.
AtlasStructDef.AtlasAttributeDef[] structTargetAttributes = new AtlasStructDef.AtlasAttributeDef[]{
new AtlasStructDef.AtlasAttributeDef("attr1", "string",
true,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList())};
AtlasEntityDef structTargetDef =
new AtlasEntityDef("StructTarget", "StructTarget_description", "1.0",
Arrays.asList(structTargetAttributes), Collections.<String>emptySet());
AtlasStructDef.AtlasAttributeDef[] traitTargetAttributes = new AtlasStructDef.AtlasAttributeDef[]{
new AtlasStructDef.AtlasAttributeDef("attr1", "string",
true,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList())};
AtlasEntityDef traitTargetDef =
new AtlasEntityDef("TraitTarget", "TraitTarget_description", "1.0",
Arrays.asList(traitTargetAttributes), Collections.<String>emptySet());
AtlasStructDef.AtlasAttributeDef[] structContainerAttributes = new AtlasStructDef.AtlasAttributeDef[]{
new AtlasStructDef.AtlasAttributeDef("struct", "TestStruct",
true,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList())};
AtlasEntityDef structContainerDef =
new AtlasEntityDef("StructContainer", "StructContainer_description", "1.0",
Arrays.asList(structContainerAttributes), Collections.<String>emptySet());
// Define struct and trait types which have a unidirectional array reference
// to a class type.
AtlasStructDef.AtlasAttributeDef[] structDefAttributes = new AtlasStructDef.AtlasAttributeDef[] {
new AtlasStructDef.AtlasAttributeDef("target", "array<StructTarget>",
true,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()),
new AtlasStructDef.AtlasAttributeDef("nestedStructs", "array<NestedStruct>",
true,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()) };
AtlasStructDef structDef = new AtlasStructDef("TestStruct", "TestStruct_desc", "1.0", Arrays.asList(structDefAttributes));
// Define struct and trait types which have a unidirectional array reference
// to a class type.
AtlasStructDef.AtlasAttributeDef[] nestedStructDefAttributes = new AtlasStructDef.AtlasAttributeDef[] {
new AtlasStructDef.AtlasAttributeDef("attr1", "string",
true,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()),
new AtlasStructDef.AtlasAttributeDef("target", "array<TraitTarget>",
true,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()) };
AtlasStructDef nestedStructDef = new AtlasStructDef("NestedStruct", "NestedStruct_desc", "1.0", Arrays.asList(nestedStructDefAttributes));
AtlasStructDef.AtlasAttributeDef[] traitDefAttributes = new AtlasStructDef.AtlasAttributeDef[] {
new AtlasStructDef.AtlasAttributeDef("target", "array<TraitTarget>",
true,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList())
};
AtlasClassificationDef traitDef = new AtlasClassificationDef("TestTrait", "TestTrait_desc", "1.0", Arrays.asList(traitDefAttributes));
AtlasTypesDef typesDef = AtlasTypeUtil.getTypesDef(ImmutableList.<AtlasEnumDef>of(),
ImmutableList.<AtlasStructDef>of(structDef, nestedStructDef),
ImmutableList.<AtlasClassificationDef>of(traitDef),
ImmutableList.<AtlasEntityDef>of(structTargetDef, traitTargetDef, structContainerDef));
typeDefStore.createTypesDef(typesDef);
// Create instances of class, struct, and trait types.
final AtlasEntity structTargetEntity = new AtlasEntity("StructTarget");
final AtlasEntity traitTargetEntity = new AtlasEntity("TraitTarget");
final AtlasEntity structContainerEntity = new AtlasEntity("StructContainer");
AtlasStruct structInstance = new AtlasStruct("TestStruct");
AtlasStruct nestedStructInstance = new AtlasStruct("NestedStruct");
Struct traitInstance = new Struct("TestTrait");
structContainerEntity.setAttribute("struct", structInstance);
structInstance.setAttribute("target", ImmutableList.of(structTargetEntity.getAtlasObjectId()));
structInstance.setAttribute("nestedStructs", ImmutableList.of(nestedStructInstance));
AtlasEntity.AtlasEntitiesWithExtInfo structCreationObj = new AtlasEntity.AtlasEntitiesWithExtInfo();
structCreationObj.addEntity(structContainerEntity);
structCreationObj.addEntity(traitTargetEntity);
structCreationObj.addReferredEntity(structTargetEntity);
init();
AtlasEntityStream entityStream = new AtlasEntityStream(structCreationObj);
EntityMutationResponse response = entityStore.createOrUpdate(entityStream);
Assert.assertEquals(response.getCreatedEntities().size(), 3);
final List<String> structTarget = metadataService.getEntityList("StructTarget");
Assert.assertEquals(structTarget.size(), 1);
final String structTargetGuid = structTarget.get(0);
final List<String> traitTarget = metadataService.getEntityList("TraitTarget");
Assert.assertEquals(traitTarget.size(), 1);
final String traitTargetGuid = traitTarget.get(0);
final List<String> structContainerTarget = metadataService.getEntityList("StructContainer");
Assert.assertEquals(structContainerTarget.size(), 1);
String structContainerGuid = structContainerTarget.get(0);
// Add TestTrait to StructContainer instance
traitInstance.set("target", ImmutableList.of(new Id(traitTargetGuid, 0, "TraitTarget")));
TraitType traitType = typeSystem.getDataType(TraitType.class, "TestTrait");
ITypedStruct convertedTrait = traitType.convert(traitInstance, Multiplicity.REQUIRED);
metadataService.addTrait(structContainerGuid, convertedTrait);
// Verify that the unidirectional references from the struct and trait instances
// are pointing at the target entities.
final ITypedReferenceableInstance structContainerConvertedEntity = metadataService.getEntityDefinition(structContainerGuid);
Object object = structContainerConvertedEntity.get("struct");
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof ITypedStruct);
ITypedStruct struct = (ITypedStruct) object;
object = struct.get("target");
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof List);
List<ITypedReferenceableInstance> refList = (List<ITypedReferenceableInstance>)object;
Assert.assertEquals(refList.size(), 1);
Assert.assertEquals(refList.get(0).getId()._getId(), structTargetGuid);
IStruct trait = structContainerConvertedEntity.getTrait("TestTrait");
Assert.assertNotNull(trait);
object = trait.get("target");
Assert.assertNotNull(object);
Assert.assertTrue(object instanceof List);
refList = (List<ITypedReferenceableInstance>)object;
Assert.assertEquals(refList.size(), 1);
Assert.assertEquals(refList.get(0).getId()._getId(), traitTargetGuid);
init();
// Delete the entities that are targets of the struct and trait instances.
EntityMutationResponse entityResult = entityStore.deleteByIds(new ArrayList<String>() {{
add(structTargetGuid);
add(traitTargetGuid);
}});
Assert.assertEquals(entityResult.getDeletedEntities().size(), 2);
Assert.assertTrue(extractGuids(entityResult.getDeletedEntities()).containsAll(Arrays.asList(structTargetGuid, traitTargetGuid)));
assertEntityDeleted(structTargetGuid);
assertEntityDeleted(traitTargetGuid);
assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes(structContainerGuid);
init();
// Delete the entity which contains nested structs and has the TestTrait trait.
entityResult = entityStore.deleteById(structContainerGuid);
Assert.assertEquals(entityResult.getDeletedEntities().size(), 1);
Assert.assertTrue(extractGuids(entityResult.getDeletedEntities()).contains(structContainerGuid));
assertEntityDeleted(structContainerGuid);
// Verify all TestStruct struct vertices were removed.
assertVerticesDeleted(getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "TestStruct"));
// Verify all NestedStruct struct vertices were removed.
assertVerticesDeleted(getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "NestedStruct"));
// Verify all TestTrait trait vertices were removed.
assertVerticesDeleted(getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "TestTrait"));
}
@Test
public void testDeleteByUniqueAttribute() throws Exception {
// Create a table entity, with 3 composite column entities
init();
final AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity));
final AtlasEntity tableEntity = TestUtilsV2.createTableEntity(dbEntity);
AtlasEntity.AtlasEntitiesWithExtInfo entitiesInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(tableEntity);
final AtlasEntity columnEntity1 = TestUtilsV2.createColumnEntity(tableEntity);
entitiesInfo.addReferredEntity(columnEntity1);
final AtlasEntity columnEntity2 = TestUtilsV2.createColumnEntity(tableEntity);
entitiesInfo.addReferredEntity(columnEntity2);
final AtlasEntity columnEntity3 = TestUtilsV2.createColumnEntity(tableEntity);
entitiesInfo.addReferredEntity(columnEntity3);
tableEntity.setAttribute(COLUMNS_ATTR_NAME, Arrays.asList(columnEntity1.getAtlasObjectId(), columnEntity2.getAtlasObjectId(), columnEntity3.getAtlasObjectId()));
init();
final EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo));
final AtlasEntityHeader column1Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity1.getAttribute(NAME));
final AtlasEntityHeader column2Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity2.getAttribute(NAME));
final AtlasEntityHeader column3Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity3.getAttribute(NAME));
// Retrieve the table entities from the Repository, to get their guids and the composite column guids.
ITypedReferenceableInstance tableInstance = metadataService.getEntityDefinitionReference(TestUtils.TABLE_TYPE, NAME, (String) tableEntity.getAttribute(NAME));
List<IReferenceableInstance> columns = (List<IReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME);
//Delete column
String colId = columns.get(0).getId()._getId();
String tableId = tableInstance.getId()._getId();
init();
Map<String, Object> uniqueAttrs = new HashMap<>();
uniqueAttrs.put(NAME, column1Created.getAttribute(NAME));
AtlasEntityType columnType = typeRegistry.getEntityTypeByName(COLUMN_TYPE);
EntityMutationResponse deletionResponse = entityStore.deleteByUniqueAttributes(columnType, uniqueAttrs);
assertEquals(deletionResponse.getDeletedEntities().size(), 1);
assertEquals(deletionResponse.getDeletedEntities().get(0).getGuid(), colId);
assertEquals(deletionResponse.getUpdatedEntities().size(), 1);
assertEquals(deletionResponse.getUpdatedEntities().get(0).getGuid(), tableId);
assertEntityDeleted(colId);
tableInstance = metadataService.getEntityDefinitionReference(TestUtils.TABLE_TYPE, NAME, (String) tableEntity.getAttribute(NAME));
assertDeletedColumn(tableInstance);
}
protected abstract void assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes(
String structContainerGuid) throws Exception;
protected abstract void assertVerticesDeleted(List<AtlasVertex> vertices);
protected List<AtlasVertex> getVertices(String propertyName, Object value) {
AtlasGraph graph = TestUtils.getGraph();
Iterable<AtlasVertex> vertices = graph.getVertices(propertyName, value);
List<AtlasVertex> list = new ArrayList<>();
for (AtlasVertex vertex : vertices) {
list.add(vertex);
}
return list;
}
}
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
......@@ -119,13 +119,11 @@ public class AtlasEntityStoreV1Test {
@AfterClass
public void clear() {
AtlasGraphProvider.cleanup();
TestUtils.resetRequestContext();
}
@BeforeTest
public void init() throws Exception {
entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry);
RequestContextV1.clear();
}
......@@ -138,7 +136,7 @@ public class AtlasEntityStoreV1Test {
AtlasEntityHeader dept1 = response.getFirstCreatedEntityByTypeName(TestUtilsV2.DEPARTMENT_TYPE);
validateEntity(deptEntity, getEntityFromStore(dept1), deptEntity.getEntities().get(0));
final Map<EntityOperation, List<AtlasEntityHeader>> entitiesMutated = response.getEntitiesMutated();
final Map<EntityOperation, List<AtlasEntityHeader>> entitiesMutated = response.getMutatedEntities();
List<AtlasEntityHeader> entitiesCreated = entitiesMutated.get(EntityOperation.CREATE);
Assert.assertTrue(entitiesCreated.size() >= deptEntity.getEntities().size());
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1Test.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
import org.testng.Assert;
import javax.inject.Inject;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtils.NAME;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
public class SoftDeleteHandlerV1Test extends AtlasDeleteHandlerV1Test {
@Inject
MetadataService metadataService;
@Override
DeleteHandlerV1 getDeleteHandler(final AtlasTypeRegistry typeRegistry) {
return new SoftDeleteHandlerV1(typeRegistry);
}
@Override
protected void assertDeletedColumn(final ITypedReferenceableInstance tableInstance) throws AtlasException {
}
@Override
protected void assertTestDeleteEntities(final ITypedReferenceableInstance tableInstance) throws Exception {
}
@Override
protected void assertTableForTestDeleteReference(final String tableId) throws Exception {
//TODO - Fix after GET is ready
ITypedReferenceableInstance table = metadataService.getEntityDefinition(tableId);
assertNotNull(table.get(NAME));
assertNotNull(table.get("description"));
assertNotNull(table.get("type"));
assertNotNull(table.get("tableType"));
assertNotNull(table.get("created"));
Id dbId = (Id) table.get("database");
assertNotNull(dbId);
ITypedReferenceableInstance db = metadataService.getEntityDefinition(dbId.getId()._getId());
assertNotNull(db);
assertEquals(db.getId().getState(), Id.EntityState.ACTIVE);
}
@Override
protected void assertColumnForTestDeleteReference(final AtlasEntity tableInstance) throws AtlasException {
List<AtlasObjectId> columns = (List<AtlasObjectId>) tableInstance.getAttribute(COLUMNS_ATTR_NAME);
assertEquals(columns.size(), 1);
//TODO - Enable after GET is ready
ITypedReferenceableInstance colInst = metadataService.getEntityDefinition(columns.get(0).getGuid());
assertEquals(colInst.getId().getState(), Id.EntityState.DELETED);
}
@Override
protected void assertProcessForTestDeleteReference(final AtlasEntityHeader processInstance) throws Exception {
//
ITypedReferenceableInstance process = metadataService.getEntityDefinition(processInstance.getGuid());
List<ITypedReferenceableInstance> outputs =
(List<ITypedReferenceableInstance>) process.get(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS);
List<ITypedReferenceableInstance> expectedOutputs =
(List<ITypedReferenceableInstance>) process.get(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS);
assertEquals(outputs.size(), expectedOutputs.size());
}
@Override
protected void assertEntityDeleted(final String id) throws Exception {
// ITypedReferenceableInstance entity = metadataService.getEntityDefinition(id);
// assertEquals(entity.getId().getState(), Id.EntityState.DELETED);
final AtlasEntity.AtlasEntityWithExtInfo byId = entityStore.getById(id);
assertEquals(byId.getEntity().getStatus(), AtlasEntity.Status.DELETED);
}
@Override
protected void assertTestUpdateEntity_MultiplicityOneNonCompositeReference(final String janeGuid) throws Exception {
// Verify Jane's subordinates reference cardinality is still 2.
ITypedReferenceableInstance jane = metadataService.getEntityDefinition(janeGuid);
List<ITypedReferenceableInstance> subordinates = (List<ITypedReferenceableInstance>) jane.get("subordinates");
Assert.assertEquals(subordinates.size(), 2);
}
@Override
protected void assertJohnForTestDisconnectBidirectionalReferences(final ITypedReferenceableInstance john, final String janeGuid) throws Exception {
Id mgr = (Id) john.get("manager");
assertNotNull(mgr);
assertEquals(mgr._getId(), janeGuid);
assertEquals(mgr.getState(), Id.EntityState.DELETED);
}
@Override
protected void assertMaxForTestDisconnectBidirectionalReferences(final Map<String, String> nameGuidMap) throws Exception {
// Verify that the Department.employees reference to the deleted employee
// was disconnected.
ITypedReferenceableInstance hrDept = metadataService.getEntityDefinition(nameGuidMap.get("hr"));
List<ITypedReferenceableInstance> employees = (List<ITypedReferenceableInstance>) hrDept.get("employees");
Assert.assertEquals(employees.size(), 4);
String maxGuid = nameGuidMap.get("Max");
for (ITypedReferenceableInstance employee : employees) {
if (employee.getId()._getId().equals(maxGuid)) {
assertEquals(employee.getId().getState(), Id.EntityState.DELETED);
}
}
// Verify that the Manager.subordinates still references deleted employee
ITypedReferenceableInstance jane = metadataService.getEntityDefinition(nameGuidMap.get("Jane"));
List<ITypedReferenceableInstance> subordinates = (List<ITypedReferenceableInstance>) jane.get("subordinates");
assertEquals(subordinates.size(), 2);
for (ITypedReferenceableInstance subordinate : subordinates) {
if (subordinate.getId()._getId().equals(maxGuid)) {
assertEquals(subordinate.getId().getState(), Id.EntityState.DELETED);
}
}
// Verify that max's Person.mentor unidirectional reference to john was disconnected.
ITypedReferenceableInstance john = metadataService.getEntityDefinition(nameGuidMap.get("John"));
Id mentor = (Id) john.get("mentor");
assertEquals(mentor._getId(), maxGuid);
assertEquals(mentor.getState(), Id.EntityState.DELETED);
}
@Override
protected void assertTestDisconnectUnidirectionalArrayReferenceFromClassType(final List<ITypedReferenceableInstance> columns, final String columnGuid) {
Assert.assertEquals(columns.size(), 3);
for (ITypedReferenceableInstance column : columns) {
if (column.getId()._getId().equals(columnGuid)) {
assertEquals(column.getId().getState(), Id.EntityState.DELETED);
} else {
assertEquals(column.getId().getState(), Id.EntityState.ACTIVE);
}
}
}
@Override
protected void assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes(final String structContainerGuid) throws Exception {
// Verify that the unidirectional references from the struct and trait instances
// to the deleted entities were not disconnected.
ITypedReferenceableInstance structContainerConvertedEntity =
metadataService.getEntityDefinition(structContainerGuid);
ITypedStruct struct = (ITypedStruct) structContainerConvertedEntity.get("struct");
assertNotNull(struct.get("target"));
IStruct trait = structContainerConvertedEntity.getTrait("TestTrait");
assertNotNull(trait);
assertNotNull(trait.get("target"));
}
@Override
protected void assertVerticesDeleted(List<AtlasVertex> vertices) {
for (AtlasVertex vertex : vertices) {
assertEquals(GraphHelper.getSingleValuedProperty(vertex, Constants.STATE_PROPERTY_KEY, String.class), Id.EntityState.DELETED.name());
}
}
}
server-api/src/main/java/org/apache/atlas/RequestContextV1.java
......@@ -115,8 +115,8 @@ public class RequestContextV1 {
return requestTime;
}
public boolean isDeletedEntity(String entityGuid) {
return deletedEntityIds.contains(entityGuid);
public boolean isDeletedEntity(AtlasObjectId entityId) {
return deletedEntityIds.contains(entityId);
}
public static Metrics getMetrics() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment