Skip to content

A
atlas
Commit 5c844562 by Madhan Neethiraj
Options

ATLAS-3065: added type-patch to remove legacy attributes

parent efb6a365
Showing with 168 additions and 15 deletions
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -154,6 +154,7 @@ public enum AtlasErrorCode {
INVALID_TIMEBOUNDRY_DATERANGE(400, "ATLAS-400-00-87D", "Invalid dateRange: startTime {0} must be before endTime {1}"),
PROPAGATED_CLASSIFICATION_REMOVAL_NOT_SUPPORTED(400, "ATLAS-400-00-87E", "Removal of classification {0}, which is propagated from entity {1}, is not supported"),
IMPORT_ATTEMPTING_EMPTY_ZIP(400, "ATLAS-400-00-87F", "Attempting to import empty ZIP file."),
PATCH_MISSING_RELATIONSHIP_LABEL(400, "ATLAS-400-00-880", "{0} - must include relationship label for type {1}"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
......@@ -116,6 +117,7 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
};
private RelationshipCategory relationshipCategory;
private String relationshipLabel;
private PropagateTags propagateTags;
private AtlasRelationshipEndDef endDef1;
private AtlasRelationshipEndDef endDef2;
......@@ -193,11 +195,24 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
super(TypeCategory.RELATIONSHIP, name, description, typeVersion, attributeDefs, null);
setRelationshipCategory(relationshipCategory);
setRelationshipLabel(getDefaultRelationshipLabel());
setPropagateTags(propagatetags);
setEndDef1(endDef1);
setEndDef2(endDef2);
}
public AtlasRelationshipDef(AtlasRelationshipDef other) throws AtlasBaseException {
super(other);
if (other != null) {
setRelationshipCategory(other.getRelationshipCategory());
setRelationshipLabel(other.getRelationshipLabel());
setPropagateTags(other.getPropagateTags());
setEndDef1(other.getEndDef1());
setEndDef2(other.getEndDef2());
}
}
public void setRelationshipCategory(RelationshipCategory relationshipCategory) {
this.relationshipCategory = relationshipCategory;
}
......@@ -206,6 +221,14 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
return this.relationshipCategory;
}
public void setRelationshipLabel(String relationshipLabel) {
this.relationshipLabel = relationshipLabel;
}
public String getRelationshipLabel() {
return relationshipLabel != null ? relationshipLabel : ("r:" + super.getName());
}
public void setPropagateTags(PropagateTags propagateTags) {
this.propagateTags=propagateTags;
}
......@@ -230,18 +253,13 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
return this.endDef2;
}
public String getRelationshipLabel() { return "r:" + super.getName(); }
public AtlasRelationshipDef(AtlasRelationshipDef other) throws AtlasBaseException {
super(other);
@JsonIgnore
private String getDefaultRelationshipLabel() {
String name = super.getName();
if (other != null) {
setRelationshipCategory(other.getRelationshipCategory());
setPropagateTags(other.getPropagateTags());
setEndDef1(other.getEndDef1());
setEndDef2(other.getEndDef2());
}
return name != null ? ("r:" + name) : null;
}
@Override
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
......@@ -253,6 +271,8 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
sb.append(',');
sb.append(this.relationshipCategory);
sb.append(',');
sb.append(this.relationshipLabel);
sb.append(',');
sb.append(this.propagateTags);
sb.append(',');
if (this.endDef1 != null) {
......@@ -284,6 +304,8 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
AtlasRelationshipDef that = (AtlasRelationshipDef) o;
if (!Objects.equals(relationshipCategory, that.getRelationshipCategory()))
return false;
if (!Objects.equals(relationshipLabel, that.getRelationshipLabel()))
return false;
if (!Objects.equals(propagateTags, that.getPropagateTags()))
return false;
if (!Objects.equals(endDef1, that.getEndDef1()))
......@@ -293,7 +315,7 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), relationshipCategory, propagateTags, endDef1, endDef2);
return Objects.hash(super.hashCode(), relationshipCategory, relationshipLabel, propagateTags, endDef1, endDef2);
}
@Override
......
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
......@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.authorize.AtlasAuthorizerFactory;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
......@@ -34,10 +35,13 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
......@@ -59,6 +63,7 @@ import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -73,6 +78,7 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
public static final String PATCHES_FOLDER_NAME = "patches";
public static final String RELATIONSHIP_LABEL = "relationshipLabel";
private final AtlasTypeDefStore atlasTypeDefStore;
private final AtlasTypeRegistry atlasTypeRegistry;
......@@ -406,6 +412,7 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
PatchHandler[] patchHandlers = new PatchHandler[] {
new AddAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry),
new UpdateAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry),
new RemoveLegacyAttributesPatchHandler(atlasTypeDefStore, atlasTypeRegistry),
new UpdateTypeDefOptionsPatchHandler(atlasTypeDefStore, atlasTypeRegistry),
new SetServiceTypePatchHandler(atlasTypeDefStore, atlasTypeRegistry)
};
......@@ -701,6 +708,88 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
}
}
class RemoveLegacyAttributesPatchHandler extends PatchHandler {
public RemoveLegacyAttributesPatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry, new String[] { "REMOVE_LEGACY_ATTRIBUTES" });
}
@Override
public void applyPatch(TypeDefPatch patch) throws AtlasBaseException {
String typeName = patch.getTypeName();
AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName);
if (typeDef == null) {
throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
}
if (isPatchApplicable(patch, typeDef)) {
if (typeDef.getClass().equals(AtlasRelationshipDef.class)) {
AtlasRelationshipDef relationshipDef = (AtlasRelationshipDef) typeDef;
AtlasRelationshipEndDef end1Def = relationshipDef.getEndDef1();
AtlasRelationshipEndDef end2Def = relationshipDef.getEndDef2();
AtlasEntityType end1Type = typeRegistry.getEntityTypeByName(end1Def.getType());
AtlasEntityType end2Type = typeRegistry.getEntityTypeByName(end2Def.getType());
String newRelationshipLabel = null;
if (patch.getParams() != null) {
Object val = patch.getParams().get(RELATIONSHIP_LABEL);
if (val != null) {
newRelationshipLabel = val.toString();
}
}
if (StringUtils.isEmpty(newRelationshipLabel)) {
if (end1Def.getIsLegacyAttribute()) {
if (!end2Def.getIsLegacyAttribute()) {
AtlasAttribute legacyAttribute = end1Type.getAttribute(end1Def.getName());
newRelationshipLabel = "__" + legacyAttribute.getQualifiedName();
} else { // if both ends are legacy attributes, RELATIONSHIP_LABEL should be specified in the patch
throw new AtlasBaseException(AtlasErrorCode.PATCH_MISSING_RELATIONSHIP_LABEL, patch.getAction(), typeName);
}
} else if (end2Def.getIsLegacyAttribute()) {
AtlasAttribute legacyAttribute = end2Type.getAttribute(end2Def.getName());
newRelationshipLabel = "__" + legacyAttribute.getQualifiedName();
} else {
newRelationshipLabel = relationshipDef.getRelationshipLabel();
}
}
AtlasRelationshipDef updatedDef = new AtlasRelationshipDef(relationshipDef);
AtlasEntityDef updatedEntityDef1 = new AtlasEntityDef(end1Type.getEntityDef());
AtlasEntityDef updatedEntityDef2 = new AtlasEntityDef(end2Type.getEntityDef());
updatedDef.setRelationshipLabel(newRelationshipLabel);
updatedDef.getEndDef1().setIsLegacyAttribute(false);
updatedDef.getEndDef2().setIsLegacyAttribute(false);
updatedDef.setTypeVersion(patch.getUpdateToVersion());
updatedEntityDef1.removeAttribute(end1Def.getName());
updatedEntityDef2.removeAttribute(end2Def.getName());
AtlasTypesDef typesDef = new AtlasTypesDef();
typesDef.setEntityDefs(Arrays.asList(updatedEntityDef1, updatedEntityDef2));
typesDef.setRelationshipDefs(Collections.singletonList(updatedDef));
try {
RequestContext.get().setInTypePatching(true); // to allow removal of attributes
typeDefStore.updateTypesDef(typesDef);
} finally {
RequestContext.get().setInTypePatching(false);
}
}
} else {
LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
}
}
}
class UpdateTypeDefOptionsPatchHandler extends PatchHandler {
public UpdateTypeDefOptionsPatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry, new String[] { "UPDATE_TYPEDEF_OPTIONS" });
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
......@@ -43,6 +43,7 @@ import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
/**
* RelationshipDef store in v1 format.
......@@ -432,7 +433,7 @@ public class AtlasRelationshipDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasRe
AtlasRelationshipEndDef existingEnd1 = existingRelationshipDef.getEndDef1();
AtlasRelationshipEndDef newEnd1 = newRelationshipDef.getEndDef1();
if ( !newEnd1.equals(existingEnd1) ) {
if ( !isValidUpdate(existingEnd1, newEnd1) ) {
throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID_END1_UPDATE,
newRelationshipDef.getName(), newEnd1.toString(), existingEnd1.toString());
}
......@@ -440,7 +441,7 @@ public class AtlasRelationshipDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasRe
AtlasRelationshipEndDef existingEnd2 = existingRelationshipDef.getEndDef2();
AtlasRelationshipEndDef newEnd2 = newRelationshipDef.getEndDef2();
if ( !newEnd2.equals(existingEnd2) ) {
if ( !isValidUpdate(existingEnd2, newEnd2) ) {
throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID_END2_UPDATE,
newRelationshipDef.getName(), newEnd2.toString(), existingEnd2.toString());
}
......@@ -505,4 +506,12 @@ public class AtlasRelationshipDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasRe
return ret;
}
private static boolean isValidUpdate(AtlasRelationshipEndDef currentDef, AtlasRelationshipEndDef updatedDef) {
// permit updates to description and isLegacyAttribute (ref type-patch REMOVE_LEGACY_ATTRIBUTES)
return Objects.equals(currentDef.getType(), updatedDef.getType()) &&
Objects.equals(currentDef.getName(), updatedDef.getName()) &&
Objects.equals(currentDef.getIsContainer(), updatedDef.getIsContainer()) &&
Objects.equals(currentDef.getCardinality(), updatedDef.getCardinality());
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v2;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContext;
import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.authorize.AtlasTypeAccessRequest;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
......@@ -385,12 +386,34 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
// delete attributes that are not present in updated structDef
if (CollectionUtils.isNotEmpty(currAttrNames)) {
List<String> removedAttributes = null;
for (String currAttrName : currAttrNames) {
if (!attrNames.contains(currAttrName)) {
throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_DELETION_NOT_SUPPORTED,
structDef.getName(), currAttrName);
if (RequestContext.get().isInTypePatching()) {
String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, currAttrName);
AtlasGraphUtilsV2.setProperty(vertex, propertyKey, null);
if (removedAttributes == null) {
removedAttributes = new ArrayList<>();
}
removedAttributes.add(currAttrName);
LOG.warn("REMOVED ATTRIBUTE: {}.{}", structDef.getName(), currAttrName);
} else {
throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_DELETION_NOT_SUPPORTED,
structDef.getName(), currAttrName);
}
}
}
if (removedAttributes != null) {
currAttrNames.removeAll(removedAttributes);
vertex.setListProperty(encodedStructDefPropertyKey, currAttrNames);
}
}
typeDefStore.updateTypeVertex(structDef, vertex);
......
server-api/src/main/java/org/apache/atlas/RequestContext.java
......@@ -56,6 +56,7 @@ public class RequestContext {
private int attemptCount = 1;
private boolean isImportInProgress = false;
private boolean isInNotificationProcessing = false;
private boolean isInTypePatching = false;
private RequestContext() {
......@@ -173,6 +174,14 @@ public class RequestContext {
isInNotificationProcessing = inNotificationProcessing;
}
public boolean isInTypePatching() {
return isInTypePatching;
}
public void setInTypePatching(boolean inTypePatching) {
isInTypePatching = inTypePatching;
}
public void recordEntityUpdate(AtlasEntityHeader entity) {
if (entity != null && entity.getGuid() != null) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment