Skip to content

A
atlas
Commit c76afbe2 by apoorvnaik
Options

ATLAS-2582: Hard delete of internal types

Change-Id: Id5dfd66840f9916958cc91a7230ab086151c4232
parent 801aea9a
Showing with 77 additions and 52 deletions
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -50,6 +50,8 @@ public class AtlasEntityType extends AtlasStructType {
private final AtlasEntityDef entityDef;
private final String typeQryStr;
private static final String INTERNAL_TYPENAME = "__internal";
private List<AtlasEntityType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> subTypes = Collections.emptySet();
......@@ -59,6 +61,7 @@ public class AtlasEntityType extends AtlasStructType {
private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap();
private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
private String typeAndAllSubTypesQryStr = "";
private boolean isInternalType = false;
public AtlasEntityType(AtlasEntityDef entityDef) {
......@@ -145,6 +148,10 @@ public class AtlasEntityType extends AtlasStructType {
}
for (String superTypeName : allSuperTypes) {
if (INTERNAL_TYPENAME.equals(superTypeName)) {
isInternalType = true;
}
AtlasEntityType superType = typeRegistry.getEntityTypeByName(superTypeName);
Map<String, AtlasAttribute> superTypeRelationshipAttributes = superType.getRelationshipAttributes();
......@@ -206,6 +213,10 @@ public class AtlasEntityType extends AtlasStructType {
return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName);
}
public boolean isInternalType() {
return isInternalType;
}
public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; }
public AtlasAttribute getRelationshipAttribute(String attributeName) { return relationshipAttributes.get(attributeName); }
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -647,8 +647,8 @@ public final class GraphHelper {
public static <T> T getSingleValuedProperty(AtlasElement element, String propertyName, Class<T> clazz) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (LOG.isDebugEnabled()) {
LOG.debug("Reading property {} from {}", actualPropertyName, string(element));
if (LOG.isTraceEnabled()) {
LOG.trace("Reading property {} from {}", actualPropertyName, string(element));
}
return element.getProperty(actualPropertyName, clazz);
......@@ -658,8 +658,8 @@ public final class GraphHelper {
public static Object getProperty(AtlasVertex<?,?> vertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (LOG.isDebugEnabled()) {
LOG.debug("Reading property {} from {}", actualPropertyName, string(vertex));
if (LOG.isTraceEnabled()) {
LOG.trace("Reading property {} from {}", actualPropertyName, string(vertex));
}
if(AtlasGraphProvider.getGraphInstance().isMultiProperty(actualPropertyName)) {
......@@ -673,8 +673,8 @@ public final class GraphHelper {
public static Object getProperty(AtlasEdge<?,?> edge, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (LOG.isDebugEnabled()) {
LOG.debug("Reading property {} from {}", actualPropertyName, string(edge));
if (LOG.isTraceEnabled()) {
LOG.trace("Reading property {} from {}", actualPropertyName, string(edge));
}
return edge.getProperty(actualPropertyName, Object.class);
......@@ -736,18 +736,14 @@ public final class GraphHelper {
* @param edge
*/
public void removeEdge(AtlasEdge edge) {
String edgeString = null;
if (LOG.isDebugEnabled()) {
edgeString = string(edge);
LOG.debug("Removing {}", edgeString);
LOG.debug("==> removeEdge({})", string(edge));
}
graph.removeEdge(edge);
if (LOG.isDebugEnabled()) {
LOG.info("Removed {}", edgeString);
LOG.info("<== removeEdge()");
}
}
......@@ -757,18 +753,14 @@ public final class GraphHelper {
* @param vertex
*/
public void removeVertex(AtlasVertex vertex) {
String vertexString = null;
if (LOG.isDebugEnabled()) {
vertexString = string(vertex);
LOG.debug("Removing {}", vertexString);
LOG.debug("==> GraphHelper.removeVertex({})", string(vertex));
}
graph.removeVertex(vertex);
if (LOG.isDebugEnabled()) {
LOG.info("Removed {}", vertexString);
LOG.debug("<== GraphHelper.removeVertex()");
}
}
......@@ -1580,7 +1572,7 @@ public final class GraphHelper {
}
// newly added
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(AtlasGraphUtilsV1.isReference(elementType)) {
return (List)instanceVertex.getListProperty(encodedPropertyName, AtlasEdge.class);
......
repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AbstractGlossaryDTO.java
......@@ -24,7 +24,6 @@ import org.apache.atlas.model.glossary.relations.AtlasRelatedCategoryHeader;
import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader;
import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader;
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.repository.ogm.AbstractDataTransferObject;
import org.apache.atlas.type.AtlasTypeRegistry;
......@@ -151,9 +150,7 @@ public abstract class AbstractGlossaryDTO<T extends AtlasBaseModelObject> extend
ret = new HashSet<>();
for (Object t : (Collection) relatedObjectIds) {
if (t instanceof AtlasRelatedObjectId) {
if (((AtlasRelatedObjectId) t).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.add(constructRelatedTermId((AtlasRelatedObjectId) t));
}
ret.add(constructRelatedTermId((AtlasRelatedObjectId) t));
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryCategoryDTO.java
......@@ -60,17 +60,13 @@ public class AtlasGlossaryCategoryDTO extends AbstractGlossaryDTO<AtlasGlossaryC
Object anchor = entity.getRelationshipAttribute("anchor");
if (anchor instanceof AtlasRelatedObjectId) {
LOG.debug("Processing anchor");
if (((AtlasRelatedObjectId) anchor).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor));
}
ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor));
}
Object parentCategory = entity.getRelationshipAttribute("parentCategory");
if (parentCategory instanceof AtlasRelatedObjectId) {
LOG.debug("Processing parentCategory");
if (((AtlasRelatedObjectId) parentCategory).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.setParentCategory(constructRelatedCategoryId((AtlasRelatedObjectId) parentCategory));
}
ret.setParentCategory(constructRelatedCategoryId((AtlasRelatedObjectId) parentCategory));
}
Object childrenCategories = entity.getRelationshipAttribute("childrenCategories");
......@@ -90,9 +86,7 @@ public class AtlasGlossaryCategoryDTO extends AbstractGlossaryDTO<AtlasGlossaryC
LOG.debug("Processing terms");
for (Object term : (Collection) terms) {
if (term instanceof AtlasRelatedObjectId) {
if (((AtlasRelatedObjectId) term).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) term));
}
ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) term));
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryDTO.java
......@@ -21,7 +21,6 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.glossary.AtlasGlossary;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -69,9 +68,7 @@ public class AtlasGlossaryDTO extends AbstractGlossaryDTO<AtlasGlossary> {
if (categoriesAttr instanceof Collection) {
for (Object o : (Collection) categoriesAttr) {
if (o instanceof AtlasRelatedObjectId) {
if (((AtlasRelatedObjectId) o).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.addCategory(constructRelatedCategoryId((AtlasRelatedObjectId) o));
}
ret.addCategory(constructRelatedCategoryId((AtlasRelatedObjectId) o));
}
}
}
......@@ -83,9 +80,7 @@ public class AtlasGlossaryDTO extends AbstractGlossaryDTO<AtlasGlossary> {
if (termsAttr instanceof Collection) {
for (Object o : (Collection) termsAttr) {
if (o instanceof AtlasRelatedObjectId) {
if (((AtlasRelatedObjectId) o).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) o));
}
ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) o));
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryTermDTO.java
......@@ -21,7 +21,6 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
......@@ -65,9 +64,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm>
Object anchor = entity.getRelationshipAttribute("anchor");
if (anchor instanceof AtlasRelatedObjectId) {
LOG.debug("Processing anchor");
if (((AtlasRelatedObjectId) anchor).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor));
}
ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor));
}
Object categories = entity.getRelationshipAttribute("categories");
......@@ -75,9 +72,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm>
LOG.debug("Processing categories");
for (Object category : (Collection) categories) {
if (category instanceof AtlasRelatedObjectId) {
if (((AtlasRelatedObjectId) category).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.addCategory(constructTermCategorizationId((AtlasRelatedObjectId) category));
}
ret.addCategory(constructTermCategorizationId((AtlasRelatedObjectId) category));
}
}
}
......@@ -90,9 +85,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm>
for (Object assignedEntity : (Collection) assignedEntities) {
if (assignedEntity instanceof AtlasRelatedObjectId) {
AtlasRelatedObjectId id = (AtlasRelatedObjectId) assignedEntity;
if (id.getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
ret.addAssignedEntity(id);
}
ret.addAssignedEntity(id);
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java
......@@ -30,6 +30,8 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.v1.model.instance.Referenceable;
import org.apache.atlas.v1.model.instance.Struct;
import org.apache.atlas.repository.Constants;
......@@ -48,6 +50,7 @@ import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import static org.apache.atlas.util.AtlasRepositoryConfiguration.isV2EntityNotificationEnabled;
......@@ -61,15 +64,20 @@ public class AtlasEntityChangeNotifier {
private final Set<EntityChangeListenerV2> entityChangeListenersV2;
private final AtlasInstanceConverter instanceConverter;
private final FullTextMapperV2 fullTextMapperV2;
private final AtlasTypeRegistry atlasTypeRegistry;
@Inject
public AtlasEntityChangeNotifier(Set<EntityChangeListener> entityChangeListeners, Set<EntityChangeListenerV2> entityChangeListenersV2,
AtlasInstanceConverter instanceConverter, final FullTextMapperV2 fullTextMapperV2) {
public AtlasEntityChangeNotifier(Set<EntityChangeListener> entityChangeListeners,
Set<EntityChangeListenerV2> entityChangeListenersV2,
AtlasInstanceConverter instanceConverter,
FullTextMapperV2 fullTextMapperV2,
AtlasTypeRegistry atlasTypeRegistry) {
this.entityChangeListeners = entityChangeListeners;
this.entityChangeListenersV2 = entityChangeListenersV2;
this.instanceConverter = instanceConverter;
this.fullTextMapperV2 = fullTextMapperV2;
this.atlasTypeRegistry = atlasTypeRegistry;
}
public void onEntitiesMutated(EntityMutationResponse entityMutationResponse, boolean isImport) throws AtlasBaseException {
......@@ -282,6 +290,17 @@ public class AtlasEntityChangeNotifier {
if (CollectionUtils.isNotEmpty(entityHeaders)) {
for (AtlasEntityHeader entityHeader : entityHeaders) {
String entityGuid = entityHeader.getGuid();
String typeName = entityHeader.getTypeName();
// Skip all internal types as the HARD DELETE will cause lookup errors
AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName(typeName);
if (Objects.nonNull(entityType) && entityType.isInternalType()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping internal type = {}", typeName);
}
continue;
}
AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid);
if (entityWithExtInfo != null) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
......@@ -683,6 +683,13 @@ public final class EntityGraphRetriever {
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
for (Object element : arrayElements) {
// When internal types are deleted, sometimes the collection type attribute will contain a null value
// Graph layer does erroneous mapping of the null element, hence avoiding the processing of the null element
if (element == null) {
LOG.debug("Skipping null arrayElement");
continue;
}
Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel,
entityExtInfo, isOwnedAttribute, edgeDirection);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasTypeRegistry;
......@@ -38,11 +39,19 @@ public class HardDeleteHandlerV1 extends DeleteHandlerV1 {
@Override
protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> SoftDeleteHandlerV1._deleteVertex({}, {})", GraphHelper.string(instanceVertex), force);
}
graphHelper.removeVertex(instanceVertex);
}
@Override
protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HardDeleteHandlerV1.deleteEdge({}, {})", GraphHelper.string(edge), force);
}
graphHelper.removeEdge(edge);
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java
......@@ -45,6 +45,10 @@ public class SoftDeleteHandlerV1 extends DeleteHandlerV1 {
@Override
protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> SoftDeleteHandlerV1._deleteVertex({}, {})", GraphHelper.string(instanceVertex), force);
}
if (force) {
graphHelper.removeVertex(instanceVertex);
} else {
......@@ -60,6 +64,10 @@ public class SoftDeleteHandlerV1 extends DeleteHandlerV1 {
@Override
protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> SoftDeleteHandlerV1.deleteEdge({}, {})",GraphHelper.string(edge), force);
}
if (force) {
graphHelper.removeEdge(edge);
} else {
......
repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
......@@ -332,7 +332,7 @@ public class GlossaryServiceTest {
try {
glossaryService.getGlossary(bankGlossary.getGuid());
} catch (AtlasBaseException e) {
assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INSTANCE_GUID_DELETED);
assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INSTANCE_GUID_NOT_FOUND);
}
} catch (AtlasBaseException e) {
fail("Glossary delete should've succeeded", e);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment