ATLAS-1314: update typedefs with addition of 'options' attribute

addons/models/patches/002-hive_column_table_add_options.json

+{
+  "patches": [
+    {
+      "action": "UPDATE_TYPEDEF_OPTIONS",
+      "typeName": "hive_column",
+      "applyToVersion": "1.1",
+      "updateToVersion": "1.2",
+      "typeDefOptions": {
+        "supportsProfile": "true"
+      }
+    },
+    {
+      "action": "UPDATE_TYPEDEF_OPTIONS",
+      "typeName": "hive_table",
+      "applyToVersion": "1.0",
+      "updateToVersion": "1.1",
+      "typeDefOptions": {
+        "supportsProfile": "true",
+        "supportsSchema": "true"
+      }
+    }
+  ]
+}

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -55,6 +55,7 @@ public final class Constants {
     public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
     public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description";
     public static final String TYPEVERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.version";
+    public static final String TYPEOPTIONS_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.options";
 
     /**
      * Trait names property key and index name.

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -62,6 +62,7 @@ public enum AtlasErrorCode {
 
     PATCH_NOT_APPLICABLE_FOR_TYPE(500, "ATLAS5004E", "{0} - invalid patch for type {1}"),
     PATCH_FOR_UNKNOWN_TYPE(500, "ATLAS5005E", "{0} - patch references unknown type {1}"),
+    PATCH_INVALID_DATA(500, "ATLAS5006E", "{0} - patch data is invalid for type {1}"),
     ATTRIBUTE_UNIQUE_INVALID(400, "ATLAS40015E", "Type {0} with unique attribute {1} does not exist"),
     TYPE_NAME_INVALID(400, "ATLAS40016E", "Type {0} with name {1} does not exist"),
     TYPE_CATEGORY_INVALID(400, "ATLAS40017E", "Type Category {0} does not match {1} or is invalid");

intg/src/main/java/org/apache/atlas/model/typedef/AtlasBaseTypeDef.java

@@ -21,6 +21,7 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Collection;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.Map;
 
 import javax.xml.bind.annotation.XmlAccessType;
@@ -72,6 +73,9 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
     public static final String ATLAS_TYPE_ASSET          = "Asset";
     public static final String ATLAS_TYPE_INFRASTRUCTURE = "Infrastructure";
 
+    public static final String TYPEDEF_OPTION_SUPPORTS_SCHEMA  = "supportsSchema";
+    public static final String TYPEDEF_OPTION_SUPPORTS_PROFILE = "supportsProfile";
+
     public static final String[] ATLAS_PRIMITIVE_TYPES = {
         ATLAS_TYPE_BOOLEAN,
         ATLAS_TYPE_BYTE,
@@ -114,8 +118,10 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
     private String  name;
     private String  description;
     private String  typeVersion;
+    private Map<String, String> options;
 
-    protected AtlasBaseTypeDef(TypeCategory category, String name, String description, String typeVersion) {
+    protected AtlasBaseTypeDef(TypeCategory category, String name, String description, String typeVersion,
+                               Map<String, String> options) {
         super();
 
         this.category = category;
@@ -129,6 +135,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
         setName(name);
         setDescription(description);
         setTypeVersion(typeVersion);
+        setOptions(options);
     }
 
     protected AtlasBaseTypeDef(AtlasBaseTypeDef other) {
@@ -144,6 +151,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
             setName(other.getName());
             setDescription(other.getDescription());
             setTypeVersion(other.getTypeVersion());
+            setOptions(other.getOptions());
         } else {
             this.category = TypeCategory.PRIMITIVE;
 
@@ -156,6 +164,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
             setName(null);
             setDescription(null);
             setTypeVersion(null);
+            setOptions(null);
         }
     }
 
@@ -234,6 +243,18 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
         this.typeVersion = typeVersion;
     }
 
+    public Map<String, String> getOptions() {
+        return options;
+    }
+
+    public void setOptions(Map<String, String> options) {
+        if (options != null) {
+            this.options = new HashMap<>(options);
+        } else {
+            this.options = null;
+        }
+    }
+
     public StringBuilder toString(StringBuilder sb) {
         if (sb == null) {
             sb = new StringBuilder();
@@ -250,6 +271,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
         sb.append(", name='").append(name).append('\'');
         sb.append(", description='").append(description).append('\'');
         sb.append(", typeVersion='").append(typeVersion).append('\'');
+        sb.append(", options='").append(options).append('\'');
         sb.append('}');
 
         return sb;
@@ -272,6 +294,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
         if (name != null ? !name.equals(that.name) : that.name != null) { return false; }
         if (description != null ? !description.equals(that.description) : that.description != null) { return false; }
         if (typeVersion != null ? !typeVersion.equals(that.typeVersion) : that.typeVersion != null) { return false; }
+        if (options != null ? !options.equals(that.options) : that.options != null) { return false; }
 
         return true;
 
@@ -289,6 +312,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
         result = 31 * result + (name != null ? name.hashCode() : 0);
         result = 31 * result + (description != null ? description.hashCode() : 0);
         result = 31 * result + (typeVersion != null ? typeVersion.hashCode() : 0);
+        result = 31 * result + (options != null ? options.hashCode() : 0);
         return result;
     }
 

intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java

@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import javax.xml.bind.annotation.XmlAccessType;
@@ -52,29 +53,35 @@ public class AtlasClassificationDef extends AtlasStructDef implements java.io.Se
 
 
     public AtlasClassificationDef() {
-        this(null, null, null, null, null);
+        this(null, null, null, null, null, null);
     }
 
     public AtlasClassificationDef(String name) {
-        this(name, null, null, null, null);
+        this(name, null, null, null, null, null);
     }
 
     public AtlasClassificationDef(String name, String description) {
-        this(name, description, null, null, null);
+        this(name, description, null, null, null, null);
     }
 
     public AtlasClassificationDef(String name, String description, String typeVersion) {
-        this(name, description, typeVersion, null, null);
+        this(name, description, typeVersion, null, null, null);
     }
 
     public AtlasClassificationDef(String name, String description, String typeVersion,
                                   List<AtlasAttributeDef> attributeDefs) {
-        this(name, description, typeVersion, attributeDefs, null);
+        this(name, description, typeVersion, attributeDefs, null, null);
     }
 
     public AtlasClassificationDef(String name, String description, String typeVersion,
                                   List<AtlasAttributeDef> attributeDefs, Set<String> superTypes) {
-        super(TypeCategory.CLASSIFICATION, name, description, typeVersion, attributeDefs);
+        this(name, description, typeVersion, attributeDefs, superTypes, null);
+    }
+
+    public AtlasClassificationDef(String name, String description, String typeVersion,
+                                  List<AtlasAttributeDef> attributeDefs, Set<String> superTypes,
+                                  Map<String, String> options) {
+        super(TypeCategory.CLASSIFICATION, name, description, typeVersion, attributeDefs, options);
 
         setSuperTypes(superTypes);
     }

intg/src/main/java/org/apache/atlas/model/typedef/AtlasEntityDef.java

@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import javax.xml.bind.annotation.XmlAccessType;
@@ -52,28 +53,33 @@ public class AtlasEntityDef extends AtlasStructDef implements java.io.Serializab
 
 
     public AtlasEntityDef() {
-        this(null, null, null, null, null);
+        this(null, null, null, null, null, null);
     }
 
     public AtlasEntityDef(String name) {
-        this(name, null, null, null, null);
+        this(name, null, null, null, null, null);
     }
 
     public AtlasEntityDef(String name, String description) {
-        this(name, description, null, null, null);
+        this(name, description, null, null, null, null);
     }
 
     public AtlasEntityDef(String name, String description, String typeVersion) {
-        this(name, description, typeVersion, null, null);
+        this(name, description, typeVersion, null, null, null);
     }
 
     public AtlasEntityDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs) {
-        this(name, description, typeVersion, attributeDefs, null);
+        this(name, description, typeVersion, attributeDefs, null, null);
     }
 
     public AtlasEntityDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs,
                           Set<String> superTypes) {
-        super(TypeCategory.ENTITY, name, description, typeVersion, attributeDefs);
+        this(name, description, typeVersion, attributeDefs, superTypes, null);
+    }
+
+    public AtlasEntityDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs,
+                          Set<String> superTypes, Map<String, String> options) {
+        super(TypeCategory.ENTITY, name, description, typeVersion, attributeDefs, options);
 
         setSuperTypes(superTypes);
     }

intg/src/main/java/org/apache/atlas/model/typedef/AtlasEnumDef.java

@@ -52,32 +52,37 @@ public class AtlasEnumDef extends AtlasBaseTypeDef implements Serializable {
     private String                    defaultValue;
 
     public AtlasEnumDef() {
-        this(null, null, null, null, null);
+        this(null, null, null, null, null, null);
     }
 
     public AtlasEnumDef(String name) {
-        this(name, null, null, null, null);
+        this(name, null, null, null, null, null);
     }
 
     public AtlasEnumDef(String name, String description) {
-        this(name, description, null, null, null);
+        this(name, description, null, null, null, null);
     }
 
     public AtlasEnumDef(String name, String description, String typeVersion) {
-        this(name, description, typeVersion, null, null);
+        this(name, description, typeVersion, null, null, null);
     }
 
     public AtlasEnumDef(String name, String description, List<AtlasEnumElementDef> elementDefs) {
-        this(name, description, null, elementDefs, null);
+        this(name, description, null, elementDefs, null, null);
     }
 
     public AtlasEnumDef(String name, String description, String typeVersion, List<AtlasEnumElementDef> elementDefs) {
-        this(name, description, typeVersion, elementDefs, null);
+        this(name, description, typeVersion, elementDefs, null, null);
     }
 
     public AtlasEnumDef(String name, String description, String typeVersion, List<AtlasEnumElementDef> elementDefs,
                         String defaultValue) {
-        super(TypeCategory.ENUM, name, description, typeVersion);
+        this(name, description, typeVersion, elementDefs, defaultValue, null);
+    }
+
+    public AtlasEnumDef(String name, String description, String typeVersion, List<AtlasEnumElementDef> elementDefs,
+                        String defaultValue, Map<String, String> options) {
+        super(TypeCategory.ENUM, name, description, typeVersion, options);
 
         setElementDefs(elementDefs);
         setDefaultValue(defaultValue);

intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java

@@ -60,27 +60,31 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
     private List<AtlasAttributeDef> attributeDefs;
 
     public AtlasStructDef() {
-        this(null, null, null, null);
+        this(null, null, null, null, null);
     }
 
     public AtlasStructDef(String name) {
-        this(name, null, null, null);
+        this(name, null, null, null, null);
     }
 
     public AtlasStructDef(String name, String description) {
-        this(name, description, null, null);
+        this(name, description, null, null, null);
     }
 
     public AtlasStructDef(String name, String description, String typeVersion) {
-        this(name, description, typeVersion, null);
+        this(name, description, typeVersion, null, null);
     }
 
     public AtlasStructDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs) {
-        this(TypeCategory.STRUCT, name, description, typeVersion, attributeDefs);
+        this(name, description, typeVersion, attributeDefs, null);
     }
 
-    protected AtlasStructDef(TypeCategory category, String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs) {
-        super(category, name, description, typeVersion);
+    public AtlasStructDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs, Map<String, String> options) {
+        this(TypeCategory.STRUCT, name, description, typeVersion, attributeDefs, options);
+    }
+
+    protected AtlasStructDef(TypeCategory category, String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs, Map<String, String> options) {
+        super(category, name, description, typeVersion, options);
 
         setAttributeDefs(attributeDefs);
     }

repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java

@@ -30,6 +30,7 @@ import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
 import org.codehaus.jackson.annotate.JsonAutoDetect;
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -150,7 +151,10 @@ public class AtlasTypeDefStoreInitializer {
         // sort the files by filename
         Arrays.sort(typePatchFiles);
 
-        PatchHandler[] patchHandlers = new PatchHandler[] { new AddAttributePatchHandler(typeDefStore, typeRegistry) };
+        PatchHandler[] patchHandlers = new PatchHandler[] {
+                new AddAttributePatchHandler(typeDefStore, typeRegistry),
+                new UpdateTypeDefOptionsPatchHandler(typeDefStore, typeRegistry),
+        };
 
         Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
 
@@ -165,6 +169,8 @@ public class AtlasTypeDefStoreInitializer {
                 continue;
             }
 
+            LOG.info("Applying patches in file {}", typePatchFile.getAbsolutePath());
+
             try {
                 String         jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
                 TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
@@ -213,6 +219,7 @@ public class AtlasTypeDefStoreInitializer {
         private String                  updateToVersion;
         private Map<String, Object>     params;
         private List<AtlasAttributeDef> attributeDefs;
+        private Map<String, String>     typeDefOptions;
 
         public String getAction() {
             return action;
@@ -261,6 +268,14 @@ public class AtlasTypeDefStoreInitializer {
         public void setAttributeDefs(List<AtlasAttributeDef> attributeDefs) {
             this.attributeDefs = attributeDefs;
         }
+
+        public Map<String, String> getTypeDefOptions() {
+            return typeDefOptions;
+        }
+
+        public void setTypeDefOptions(Map<String, String> typeDefOptions) {
+            this.typeDefOptions = typeDefOptions;
+        }
     }
 
     /**
@@ -352,6 +367,80 @@ public class AtlasTypeDefStoreInitializer {
                     typeDefStore.updateStructDefByName(typeName, updatedDef);
                 } else {
                     throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
+                            patch.getAction(), typeDef.getClass().getSimpleName());
+                }
+            } else {
+                LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
+                        patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
+            }
+        }
+    }
+
+    class UpdateTypeDefOptionsPatchHandler extends PatchHandler {
+        public UpdateTypeDefOptionsPatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
+            super(typeDefStore, typeRegistry, new String[] { "UPDATE_TYPEDEF_OPTIONS" });
+        }
+
+        @Override
+        public void applyPatch(TypeDefPatch patch) throws AtlasBaseException {
+            String           typeName = patch.getTypeName();
+            AtlasBaseTypeDef typeDef  = typeRegistry.getTypeDefByName(typeName);
+
+            if (typeDef == null) {
+                throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
+            }
+
+            if (MapUtils.isEmpty(patch.getTypeDefOptions())) {
+                throw new AtlasBaseException(AtlasErrorCode.PATCH_INVALID_DATA, patch.getAction(), typeName);
+            }
+
+            if (isPatchApplicable(patch, typeDef)) {
+                if (typeDef.getClass().equals(AtlasEntityDef.class)) {
+                    AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef)typeDef);
+
+                    if (updatedDef.getOptions() == null) {
+                        updatedDef.setOptions(patch.getTypeDefOptions());
+                    } else {
+                        updatedDef.getOptions().putAll(patch.getTypeDefOptions());
+                    }
+                    updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+                    typeDefStore.updateEntityDefByName(typeName, updatedDef);
+                } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
+                    AtlasClassificationDef updatedDef = new AtlasClassificationDef((AtlasClassificationDef)typeDef);
+
+                    if (updatedDef.getOptions() == null) {
+                        updatedDef.setOptions(patch.getTypeDefOptions());
+                    } else {
+                        updatedDef.getOptions().putAll(patch.getTypeDefOptions());
+                    }
+                    updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+                    typeDefStore.updateClassificationDefByName(typeName, updatedDef);
+                } else if (typeDef.getClass().equals(AtlasStructDef.class)) {
+                    AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef)typeDef);
+
+                    if (updatedDef.getOptions() == null) {
+                        updatedDef.setOptions(patch.getTypeDefOptions());
+                    } else {
+                        updatedDef.getOptions().putAll(patch.getTypeDefOptions());
+                    }
+                    updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+                    typeDefStore.updateStructDefByName(typeName, updatedDef);
+                } else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
+                    AtlasEnumDef updatedDef = new AtlasEnumDef((AtlasEnumDef)typeDef);
+
+                    if (updatedDef.getOptions() == null) {
+                        updatedDef.setOptions(patch.getTypeDefOptions());
+                    } else {
+                        updatedDef.getOptions().putAll(patch.getTypeDefOptions());
+                    }
+                    updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+                    typeDefStore.updateEnumDefByName(typeName, updatedDef);
+                } else {
+                    throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
                                                  patch.getAction(), typeDef.getClass().getSimpleName());
                 }
             } else {

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEnumDefStoreV1.java

@@ -166,6 +166,8 @@ public class AtlasEnumDefStoreV1 extends AtlasAbstractDefStoreV1 implements Atla
             throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name);
         }
 
+        typeDefStore.updateTypeVertex(enumDef, vertex);
+
         toVertex(enumDef, vertex);
 
         AtlasEnumDef ret = toEnumDef(vertex);
@@ -189,6 +191,8 @@ public class AtlasEnumDefStoreV1 extends AtlasAbstractDefStoreV1 implements Atla
             throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid);
         }
 
+        typeDefStore.updateTypeVertex(enumDef, vertex);
+
         toVertex(enumDef, vertex);
 
         AtlasEnumDef ret = toEnumDef(vertex);

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java

@@ -39,6 +39,7 @@ import org.apache.atlas.repository.store.graph.AtlasEntityDefStore;
 import org.apache.atlas.repository.store.graph.AtlasEnumDefStore;
 import org.apache.atlas.repository.store.graph.AtlasStructDefStore;
 import org.apache.atlas.repository.store.graph.AtlasTypeDefGraphStore;
+import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.commons.collections.CollectionUtils;
@@ -49,6 +50,7 @@ import org.slf4j.LoggerFactory;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 
@@ -199,6 +201,7 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
         ret.setProperty(Constants.TIMESTAMP_PROPERTY_KEY, typeDef.getCreateTime().getTime());
         ret.setProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, typeDef.getUpdateTime().getTime());
         ret.setProperty(Constants.VERSION_PROPERTY_KEY, typeDef.getVersion());
+        ret.setProperty(Constants.TYPEOPTIONS_PROPERTY_KEY, AtlasType.toJson(typeDef.getOptions()));
 
         return ret;
     }
@@ -219,6 +222,7 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
          */
         updateVertexProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDef.getDescription());
         updateVertexProperty(vertex, Constants.TYPEVERSION_PROPERTY_KEY, typeDef.getTypeVersion());
+        updateVertexProperty(vertex, Constants.TYPEOPTIONS_PROPERTY_KEY, AtlasType.toJson(typeDef.getOptions()));
 
         markVertexUpdated(vertex);
     }
@@ -255,6 +259,7 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
         Long   createTime  = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
         Long   updateTime  = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
         Long   version     = vertex.getProperty(Constants.VERSION_PROPERTY_KEY, Long.class);
+        String options     = vertex.getProperty(Constants.TYPEOPTIONS_PROPERTY_KEY, String.class);
 
         typeDef.setName(name);
         typeDef.setDescription(description);
@@ -272,6 +277,10 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
         if (version != null) {
             typeDef.setVersion(version);
         }
+
+        if (options != null) {
+            typeDef.setOptions(AtlasType.fromJson(options, Map.class));
+        }
     }
 
     boolean isTypeVertex(AtlasVertex vertex) {