ATLAS-3065: added type-patch to remove legacy attributes

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -154,6 +154,7 @@ public enum AtlasErrorCode {
     INVALID_TIMEBOUNDRY_DATERANGE(400, "ATLAS-400-00-87D", "Invalid dateRange: startTime {0} must be before endTime {1}"),
     PROPAGATED_CLASSIFICATION_REMOVAL_NOT_SUPPORTED(400, "ATLAS-400-00-87E", "Removal of classification {0}, which is propagated from entity {1}, is not supported"),
     IMPORT_ATTEMPTING_EMPTY_ZIP(400, "ATLAS-400-00-87F", "Attempting to import empty ZIP file."),
+    PATCH_MISSING_RELATIONSHIP_LABEL(400, "ATLAS-400-00-880", "{0} - must include relationship label for type {1}"),
 
     UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
 

intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java

@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
 
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 
@@ -116,6 +117,7 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
     };
 
     private RelationshipCategory    relationshipCategory;
+    private String                  relationshipLabel;
     private PropagateTags           propagateTags;
     private AtlasRelationshipEndDef endDef1;
     private AtlasRelationshipEndDef endDef2;
@@ -193,11 +195,24 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
         super(TypeCategory.RELATIONSHIP, name, description, typeVersion, attributeDefs, null);
 
         setRelationshipCategory(relationshipCategory);
+        setRelationshipLabel(getDefaultRelationshipLabel());
         setPropagateTags(propagatetags);
         setEndDef1(endDef1);
         setEndDef2(endDef2);
     }
 
+    public AtlasRelationshipDef(AtlasRelationshipDef other) throws AtlasBaseException {
+        super(other);
+
+        if (other != null) {
+            setRelationshipCategory(other.getRelationshipCategory());
+            setRelationshipLabel(other.getRelationshipLabel());
+            setPropagateTags(other.getPropagateTags());
+            setEndDef1(other.getEndDef1());
+            setEndDef2(other.getEndDef2());
+        }
+    }
+
     public void setRelationshipCategory(RelationshipCategory relationshipCategory) {
         this.relationshipCategory = relationshipCategory;
     }
@@ -206,6 +221,14 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
         return this.relationshipCategory;
     }
 
+    public void setRelationshipLabel(String relationshipLabel) {
+        this.relationshipLabel = relationshipLabel;
+    }
+
+    public String getRelationshipLabel() {
+        return relationshipLabel != null ? relationshipLabel : ("r:" + super.getName());
+    }
+
     public void setPropagateTags(PropagateTags propagateTags) {
         this.propagateTags=propagateTags;
     }
@@ -230,18 +253,13 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
         return this.endDef2;
     }
 
-    public String getRelationshipLabel() { return "r:" + super.getName(); }
-
-    public AtlasRelationshipDef(AtlasRelationshipDef other) throws AtlasBaseException {
-        super(other);
+    @JsonIgnore
+    private String getDefaultRelationshipLabel() {
+        String name = super.getName();
 
-        if (other != null) {
-            setRelationshipCategory(other.getRelationshipCategory());
-            setPropagateTags(other.getPropagateTags());
-            setEndDef1(other.getEndDef1());
-            setEndDef2(other.getEndDef2());
-        }
+        return name != null ? ("r:" + name) : null;
     }
+
     @Override
     public StringBuilder toString(StringBuilder sb) {
         if (sb == null) {
@@ -253,6 +271,8 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
         sb.append(',');
         sb.append(this.relationshipCategory);
         sb.append(',');
+        sb.append(this.relationshipLabel);
+        sb.append(',');
         sb.append(this.propagateTags);
         sb.append(',');
         if (this.endDef1 != null) {
@@ -284,6 +304,8 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
         AtlasRelationshipDef that = (AtlasRelationshipDef) o;
         if (!Objects.equals(relationshipCategory, that.getRelationshipCategory()))
             return false;
+        if (!Objects.equals(relationshipLabel, that.getRelationshipLabel()))
+            return false;
         if (!Objects.equals(propagateTags, that.getPropagateTags()))
             return false;
         if (!Objects.equals(endDef1, that.getEndDef1()))
@@ -293,7 +315,7 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), relationshipCategory, propagateTags, endDef1, endDef2);
+        return Objects.hash(super.hashCode(), relationshipCategory, relationshipLabel, propagateTags, endDef1, endDef2);
     }
 
     @Override

repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java

@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
 import org.apache.atlas.authorize.AtlasAuthorizerFactory;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.ha.HAConfiguration;
@@ -34,10 +35,13 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
 import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
 import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
 import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.type.AtlasTypeUtil;
@@ -59,6 +63,7 @@ import java.io.File;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -73,6 +78,7 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
 public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
     private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
     public static final String PATCHES_FOLDER_NAME = "patches";
+    public static final String RELATIONSHIP_LABEL  = "relationshipLabel";
 
     private final AtlasTypeDefStore atlasTypeDefStore;
     private final AtlasTypeRegistry atlasTypeRegistry;
@@ -406,6 +412,7 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
             PatchHandler[] patchHandlers = new PatchHandler[] {
                     new AddAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry),
                     new UpdateAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry),
+                    new RemoveLegacyAttributesPatchHandler(atlasTypeDefStore, atlasTypeRegistry),
                     new UpdateTypeDefOptionsPatchHandler(atlasTypeDefStore, atlasTypeRegistry),
                     new SetServiceTypePatchHandler(atlasTypeDefStore, atlasTypeRegistry)
             };
@@ -701,6 +708,88 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
         }
     }
 
+    class RemoveLegacyAttributesPatchHandler extends PatchHandler {
+        public RemoveLegacyAttributesPatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
+            super(typeDefStore, typeRegistry, new String[] { "REMOVE_LEGACY_ATTRIBUTES" });
+        }
+
+        @Override
+        public void applyPatch(TypeDefPatch patch) throws AtlasBaseException {
+            String           typeName = patch.getTypeName();
+            AtlasBaseTypeDef typeDef  = typeRegistry.getTypeDefByName(typeName);
+
+            if (typeDef == null) {
+                throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
+            }
+
+            if (isPatchApplicable(patch, typeDef)) {
+                if (typeDef.getClass().equals(AtlasRelationshipDef.class)) {
+                    AtlasRelationshipDef    relationshipDef = (AtlasRelationshipDef) typeDef;
+                    AtlasRelationshipEndDef end1Def         = relationshipDef.getEndDef1();
+                    AtlasRelationshipEndDef end2Def         = relationshipDef.getEndDef2();
+                    AtlasEntityType         end1Type        = typeRegistry.getEntityTypeByName(end1Def.getType());
+                    AtlasEntityType         end2Type        = typeRegistry.getEntityTypeByName(end2Def.getType());
+
+                    String newRelationshipLabel = null;
+
+                    if (patch.getParams() != null) {
+                        Object val = patch.getParams().get(RELATIONSHIP_LABEL);
+
+                        if (val != null) {
+                            newRelationshipLabel = val.toString();
+                        }
+                    }
+
+                    if (StringUtils.isEmpty(newRelationshipLabel)) {
+                        if (end1Def.getIsLegacyAttribute()) {
+                            if (!end2Def.getIsLegacyAttribute()) {
+                                AtlasAttribute legacyAttribute = end1Type.getAttribute(end1Def.getName());
+
+                                newRelationshipLabel = "__" + legacyAttribute.getQualifiedName();
+                            } else { // if both ends are legacy attributes, RELATIONSHIP_LABEL should be specified in the patch
+                                throw new AtlasBaseException(AtlasErrorCode.PATCH_MISSING_RELATIONSHIP_LABEL, patch.getAction(), typeName);
+                            }
+                        } else if (end2Def.getIsLegacyAttribute()) {
+                            AtlasAttribute legacyAttribute = end2Type.getAttribute(end2Def.getName());
+
+                            newRelationshipLabel = "__" + legacyAttribute.getQualifiedName();
+                        } else {
+                            newRelationshipLabel = relationshipDef.getRelationshipLabel();
+                        }
+                    }
+
+                    AtlasRelationshipDef updatedDef        = new AtlasRelationshipDef(relationshipDef);
+                    AtlasEntityDef       updatedEntityDef1 = new AtlasEntityDef(end1Type.getEntityDef());
+                    AtlasEntityDef       updatedEntityDef2 = new AtlasEntityDef(end2Type.getEntityDef());
+
+                    updatedDef.setRelationshipLabel(newRelationshipLabel);
+                    updatedDef.getEndDef1().setIsLegacyAttribute(false);
+                    updatedDef.getEndDef2().setIsLegacyAttribute(false);
+                    updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+                    updatedEntityDef1.removeAttribute(end1Def.getName());
+                    updatedEntityDef2.removeAttribute(end2Def.getName());
+
+                    AtlasTypesDef typesDef = new AtlasTypesDef();
+
+                    typesDef.setEntityDefs(Arrays.asList(updatedEntityDef1, updatedEntityDef2));
+                    typesDef.setRelationshipDefs(Collections.singletonList(updatedDef));
+
+                    try {
+                        RequestContext.get().setInTypePatching(true); // to allow removal of attributes
+
+                        typeDefStore.updateTypesDef(typesDef);
+                    } finally {
+                        RequestContext.get().setInTypePatching(false);
+                    }
+                }
+            } else {
+                LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
+                         patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
+            }
+        }
+    }
+
     class UpdateTypeDefOptionsPatchHandler extends PatchHandler {
         public UpdateTypeDefOptionsPatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
             super(typeDefStore, typeRegistry, new String[] { "UPDATE_TYPEDEF_OPTIONS" });

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java

@@ -43,6 +43,7 @@ import javax.inject.Inject;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * RelationshipDef store in v1 format.
@@ -432,7 +433,7 @@ public class AtlasRelationshipDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasRe
         AtlasRelationshipEndDef existingEnd1 = existingRelationshipDef.getEndDef1();
         AtlasRelationshipEndDef newEnd1      = newRelationshipDef.getEndDef1();
 
-        if ( !newEnd1.equals(existingEnd1) ) {
+        if ( !isValidUpdate(existingEnd1, newEnd1) ) {
             throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID_END1_UPDATE,
                                          newRelationshipDef.getName(), newEnd1.toString(), existingEnd1.toString());
         }
@@ -440,7 +441,7 @@ public class AtlasRelationshipDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasRe
         AtlasRelationshipEndDef existingEnd2 = existingRelationshipDef.getEndDef2();
         AtlasRelationshipEndDef newEnd2      = newRelationshipDef.getEndDef2();
 
-        if ( !newEnd2.equals(existingEnd2) ) {
+        if ( !isValidUpdate(existingEnd2, newEnd2) ) {
                 throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID_END2_UPDATE,
                                          newRelationshipDef.getName(), newEnd2.toString(), existingEnd2.toString());
         }
@@ -505,4 +506,12 @@ public class AtlasRelationshipDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasRe
         return ret;
     }
 
+    private static boolean isValidUpdate(AtlasRelationshipEndDef currentDef, AtlasRelationshipEndDef updatedDef) {
+        // permit updates to description and isLegacyAttribute (ref type-patch REMOVE_LEGACY_ATTRIBUTES)
+        return Objects.equals(currentDef.getType(), updatedDef.getType()) &&
+                Objects.equals(currentDef.getName(), updatedDef.getName()) &&
+                Objects.equals(currentDef.getIsContainer(), updatedDef.getIsContainer()) &&
+                Objects.equals(currentDef.getCardinality(), updatedDef.getCardinality());
+    }
+
 }

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java

@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v2;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.RequestContext;
 import org.apache.atlas.authorize.AtlasPrivilege;
 import org.apache.atlas.authorize.AtlasTypeAccessRequest;
 import org.apache.atlas.authorize.AtlasAuthorizationUtils;
@@ -385,14 +386,36 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
 
         // delete attributes that are not present in updated structDef
         if (CollectionUtils.isNotEmpty(currAttrNames)) {
+            List<String> removedAttributes = null;
+
             for (String currAttrName : currAttrNames) {
                 if (!attrNames.contains(currAttrName)) {
+                    if (RequestContext.get().isInTypePatching()) {
+                        String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, currAttrName);
+
+                        AtlasGraphUtilsV2.setProperty(vertex, propertyKey, null);
+
+                        if (removedAttributes == null) {
+                            removedAttributes = new ArrayList<>();
+                        }
+
+                        removedAttributes.add(currAttrName);
+
+                        LOG.warn("REMOVED ATTRIBUTE: {}.{}", structDef.getName(), currAttrName);
+                    } else {
                         throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_DELETION_NOT_SUPPORTED,
                                 structDef.getName(), currAttrName);
                     }
                 }
             }
 
+            if (removedAttributes != null) {
+                currAttrNames.removeAll(removedAttributes);
+
+                vertex.setListProperty(encodedStructDefPropertyKey, currAttrNames);
+            }
+        }
+
         typeDefStore.updateTypeVertex(structDef, vertex);
 
         // Load up current struct definition for matching attributes

server-api/src/main/java/org/apache/atlas/RequestContext.java

@@ -56,6 +56,7 @@ public class RequestContext {
     private int         attemptCount = 1;
     private boolean     isImportInProgress = false;
     private boolean     isInNotificationProcessing = false;
+    private boolean     isInTypePatching           = false;
 
 
     private RequestContext() {
@@ -173,6 +174,14 @@ public class RequestContext {
         isInNotificationProcessing = inNotificationProcessing;
     }
 
+    public boolean isInTypePatching() {
+        return isInTypePatching;
+    }
+
+    public void setInTypePatching(boolean inTypePatching) {
+        isInTypePatching = inTypePatching;
+    }
+
 
     public void recordEntityUpdate(AtlasEntityHeader entity) {
         if (entity != null && entity.getGuid() != null) {