Skip to content

A
atlas
Commit c1b57d56 by Madhan Neethiraj
Options

ATLAS-1314: update typedefs with addition of 'options' attribute

parent 699498d7
Showing with 201 additions and 28 deletions
addons/models/patches/002-hive_column_table_add_options.json 0 → 100644
{
"patches": [
{
"action": "UPDATE_TYPEDEF_OPTIONS",
"typeName": "hive_column",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"typeDefOptions": {
"supportsProfile": "true"
}
},
{
"action": "UPDATE_TYPEDEF_OPTIONS",
"typeName": "hive_table",
"applyToVersion": "1.0",
"updateToVersion": "1.1",
"typeDefOptions": {
"supportsProfile": "true",
"supportsSchema": "true"
}
}
]
}
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -55,6 +55,7 @@ public final class Constants {
public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description";
public static final String TYPEVERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.version";
public static final String TYPEOPTIONS_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.options";
/**
* Trait names property key and index name.
......
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -62,6 +62,7 @@ public enum AtlasErrorCode {
PATCH_NOT_APPLICABLE_FOR_TYPE(500, "ATLAS5004E", "{0} - invalid patch for type {1}"),
PATCH_FOR_UNKNOWN_TYPE(500, "ATLAS5005E", "{0} - patch references unknown type {1}"),
PATCH_INVALID_DATA(500, "ATLAS5006E", "{0} - patch data is invalid for type {1}"),
ATTRIBUTE_UNIQUE_INVALID(400, "ATLAS40015E", "Type {0} with unique attribute {1} does not exist"),
TYPE_NAME_INVALID(400, "ATLAS40016E", "Type {0} with name {1} does not exist"),
TYPE_CATEGORY_INVALID(400, "ATLAS40017E", "Type Category {0} does not match {1} or is invalid");
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasBaseTypeDef.java
......@@ -21,6 +21,7 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
......@@ -72,6 +73,9 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
public static final String ATLAS_TYPE_ASSET = "Asset";
public static final String ATLAS_TYPE_INFRASTRUCTURE = "Infrastructure";
public static final String TYPEDEF_OPTION_SUPPORTS_SCHEMA = "supportsSchema";
public static final String TYPEDEF_OPTION_SUPPORTS_PROFILE = "supportsProfile";
public static final String[] ATLAS_PRIMITIVE_TYPES = {
ATLAS_TYPE_BOOLEAN,
ATLAS_TYPE_BYTE,
......@@ -114,8 +118,10 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
private String name;
private String description;
private String typeVersion;
private Map<String, String> options;
protected AtlasBaseTypeDef(TypeCategory category, String name, String description, String typeVersion) {
protected AtlasBaseTypeDef(TypeCategory category, String name, String description, String typeVersion,
Map<String, String> options) {
super();
this.category = category;
......@@ -129,6 +135,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
setName(name);
setDescription(description);
setTypeVersion(typeVersion);
setOptions(options);
}
protected AtlasBaseTypeDef(AtlasBaseTypeDef other) {
......@@ -144,6 +151,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
setName(other.getName());
setDescription(other.getDescription());
setTypeVersion(other.getTypeVersion());
setOptions(other.getOptions());
} else {
this.category = TypeCategory.PRIMITIVE;
......@@ -156,6 +164,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
setName(null);
setDescription(null);
setTypeVersion(null);
setOptions(null);
}
}
......@@ -234,6 +243,18 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
this.typeVersion = typeVersion;
}
public Map<String, String> getOptions() {
return options;
}
public void setOptions(Map<String, String> options) {
if (options != null) {
this.options = new HashMap<>(options);
} else {
this.options = null;
}
}
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
......@@ -250,6 +271,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
sb.append(", name='").append(name).append('\'');
sb.append(", description='").append(description).append('\'');
sb.append(", typeVersion='").append(typeVersion).append('\'');
sb.append(", options='").append(options).append('\'');
sb.append('}');
return sb;
......@@ -272,6 +294,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
if (name != null ? !name.equals(that.name) : that.name != null) { return false; }
if (description != null ? !description.equals(that.description) : that.description != null) { return false; }
if (typeVersion != null ? !typeVersion.equals(that.typeVersion) : that.typeVersion != null) { return false; }
if (options != null ? !options.equals(that.options) : that.options != null) { return false; }
return true;
......@@ -289,6 +312,7 @@ public abstract class AtlasBaseTypeDef implements java.io.Serializable {
result = 31 * result + (name != null ? name.hashCode() : 0);
result = 31 * result + (description != null ? description.hashCode() : 0);
result = 31 * result + (typeVersion != null ? typeVersion.hashCode() : 0);
result = 31 * result + (options != null ? options.hashCode() : 0);
return result;
}
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.xml.bind.annotation.XmlAccessType;
......@@ -52,29 +53,35 @@ public class AtlasClassificationDef extends AtlasStructDef implements java.io.Se
public AtlasClassificationDef() {
this(null, null, null, null, null);
this(null, null, null, null, null, null);
}
public AtlasClassificationDef(String name) {
this(name, null, null, null, null);
this(name, null, null, null, null, null);
}
public AtlasClassificationDef(String name, String description) {
this(name, description, null, null, null);
this(name, description, null, null, null, null);
}
public AtlasClassificationDef(String name, String description, String typeVersion) {
this(name, description, typeVersion, null, null);
this(name, description, typeVersion, null, null, null);
}
public AtlasClassificationDef(String name, String description, String typeVersion,
List<AtlasAttributeDef> attributeDefs) {
this(name, description, typeVersion, attributeDefs, null);
this(name, description, typeVersion, attributeDefs, null, null);
}
public AtlasClassificationDef(String name, String description, String typeVersion,
List<AtlasAttributeDef> attributeDefs, Set<String> superTypes) {
super(TypeCategory.CLASSIFICATION, name, description, typeVersion, attributeDefs);
this(name, description, typeVersion, attributeDefs, superTypes, null);
}
public AtlasClassificationDef(String name, String description, String typeVersion,
List<AtlasAttributeDef> attributeDefs, Set<String> superTypes,
Map<String, String> options) {
super(TypeCategory.CLASSIFICATION, name, description, typeVersion, attributeDefs, options);
setSuperTypes(superTypes);
}
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasEntityDef.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.xml.bind.annotation.XmlAccessType;
......@@ -52,28 +53,33 @@ public class AtlasEntityDef extends AtlasStructDef implements java.io.Serializab
public AtlasEntityDef() {
this(null, null, null, null, null);
this(null, null, null, null, null, null);
}
public AtlasEntityDef(String name) {
this(name, null, null, null, null);
this(name, null, null, null, null, null);
}
public AtlasEntityDef(String name, String description) {
this(name, description, null, null, null);
this(name, description, null, null, null, null);
}
public AtlasEntityDef(String name, String description, String typeVersion) {
this(name, description, typeVersion, null, null);
this(name, description, typeVersion, null, null, null);
}
public AtlasEntityDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs) {
this(name, description, typeVersion, attributeDefs, null);
this(name, description, typeVersion, attributeDefs, null, null);
}
public AtlasEntityDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs,
Set<String> superTypes) {
super(TypeCategory.ENTITY, name, description, typeVersion, attributeDefs);
this(name, description, typeVersion, attributeDefs, superTypes, null);
}
public AtlasEntityDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs,
Set<String> superTypes, Map<String, String> options) {
super(TypeCategory.ENTITY, name, description, typeVersion, attributeDefs, options);
setSuperTypes(superTypes);
}
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasEnumDef.java
......@@ -52,32 +52,37 @@ public class AtlasEnumDef extends AtlasBaseTypeDef implements Serializable {
private String defaultValue;
public AtlasEnumDef() {
this(null, null, null, null, null);
this(null, null, null, null, null, null);
}
public AtlasEnumDef(String name) {
this(name, null, null, null, null);
this(name, null, null, null, null, null);
}
public AtlasEnumDef(String name, String description) {
this(name, description, null, null, null);
this(name, description, null, null, null, null);
}
public AtlasEnumDef(String name, String description, String typeVersion) {
this(name, description, typeVersion, null, null);
this(name, description, typeVersion, null, null, null);
}
public AtlasEnumDef(String name, String description, List<AtlasEnumElementDef> elementDefs) {
this(name, description, null, elementDefs, null);
this(name, description, null, elementDefs, null, null);
}
public AtlasEnumDef(String name, String description, String typeVersion, List<AtlasEnumElementDef> elementDefs) {
this(name, description, typeVersion, elementDefs, null);
this(name, description, typeVersion, elementDefs, null, null);
}
public AtlasEnumDef(String name, String description, String typeVersion, List<AtlasEnumElementDef> elementDefs,
String defaultValue) {
super(TypeCategory.ENUM, name, description, typeVersion);
this(name, description, typeVersion, elementDefs, defaultValue, null);
}
public AtlasEnumDef(String name, String description, String typeVersion, List<AtlasEnumElementDef> elementDefs,
String defaultValue, Map<String, String> options) {
super(TypeCategory.ENUM, name, description, typeVersion, options);
setElementDefs(elementDefs);
setDefaultValue(defaultValue);
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
......@@ -60,27 +60,31 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
private List<AtlasAttributeDef> attributeDefs;
public AtlasStructDef() {
this(null, null, null, null);
this(null, null, null, null, null);
}
public AtlasStructDef(String name) {
this(name, null, null, null);
this(name, null, null, null, null);
}
public AtlasStructDef(String name, String description) {
this(name, description, null, null);
this(name, description, null, null, null);
}
public AtlasStructDef(String name, String description, String typeVersion) {
this(name, description, typeVersion, null);
this(name, description, typeVersion, null, null);
}
public AtlasStructDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs) {
this(TypeCategory.STRUCT, name, description, typeVersion, attributeDefs);
this(name, description, typeVersion, attributeDefs, null);
}
protected AtlasStructDef(TypeCategory category, String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs) {
super(category, name, description, typeVersion);
public AtlasStructDef(String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs, Map<String, String> options) {
this(TypeCategory.STRUCT, name, description, typeVersion, attributeDefs, options);
}
protected AtlasStructDef(TypeCategory category, String name, String description, String typeVersion, List<AtlasAttributeDef> attributeDefs, Map<String, String> options) {
super(category, name, description, typeVersion, options);
setAttributeDefs(attributeDefs);
}
......
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
......@@ -30,6 +30,7 @@ import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
......@@ -150,7 +151,10 @@ public class AtlasTypeDefStoreInitializer {
// sort the files by filename
Arrays.sort(typePatchFiles);
PatchHandler[] patchHandlers = new PatchHandler[] { new AddAttributePatchHandler(typeDefStore, typeRegistry) };
PatchHandler[] patchHandlers = new PatchHandler[] {
new AddAttributePatchHandler(typeDefStore, typeRegistry),
new UpdateTypeDefOptionsPatchHandler(typeDefStore, typeRegistry),
};
Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
......@@ -165,6 +169,8 @@ public class AtlasTypeDefStoreInitializer {
continue;
}
LOG.info("Applying patches in file {}", typePatchFile.getAbsolutePath());
try {
String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
......@@ -213,6 +219,7 @@ public class AtlasTypeDefStoreInitializer {
private String updateToVersion;
private Map<String, Object> params;
private List<AtlasAttributeDef> attributeDefs;
private Map<String, String> typeDefOptions;
public String getAction() {
return action;
......@@ -261,6 +268,14 @@ public class AtlasTypeDefStoreInitializer {
public void setAttributeDefs(List<AtlasAttributeDef> attributeDefs) {
this.attributeDefs = attributeDefs;
}
public Map<String, String> getTypeDefOptions() {
return typeDefOptions;
}
public void setTypeDefOptions(Map<String, String> typeDefOptions) {
this.typeDefOptions = typeDefOptions;
}
}
/**
......@@ -352,6 +367,80 @@ public class AtlasTypeDefStoreInitializer {
typeDefStore.updateStructDefByName(typeName, updatedDef);
} else {
throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
patch.getAction(), typeDef.getClass().getSimpleName());
}
} else {
LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
}
}
}
class UpdateTypeDefOptionsPatchHandler extends PatchHandler {
public UpdateTypeDefOptionsPatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry, new String[] { "UPDATE_TYPEDEF_OPTIONS" });
}
@Override
public void applyPatch(TypeDefPatch patch) throws AtlasBaseException {
String typeName = patch.getTypeName();
AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName);
if (typeDef == null) {
throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
}
if (MapUtils.isEmpty(patch.getTypeDefOptions())) {
throw new AtlasBaseException(AtlasErrorCode.PATCH_INVALID_DATA, patch.getAction(), typeName);
}
if (isPatchApplicable(patch, typeDef)) {
if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef)typeDef);
if (updatedDef.getOptions() == null) {
updatedDef.setOptions(patch.getTypeDefOptions());
} else {
updatedDef.getOptions().putAll(patch.getTypeDefOptions());
}
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateEntityDefByName(typeName, updatedDef);
} else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
AtlasClassificationDef updatedDef = new AtlasClassificationDef((AtlasClassificationDef)typeDef);
if (updatedDef.getOptions() == null) {
updatedDef.setOptions(patch.getTypeDefOptions());
} else {
updatedDef.getOptions().putAll(patch.getTypeDefOptions());
}
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateClassificationDefByName(typeName, updatedDef);
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef)typeDef);
if (updatedDef.getOptions() == null) {
updatedDef.setOptions(patch.getTypeDefOptions());
} else {
updatedDef.getOptions().putAll(patch.getTypeDefOptions());
}
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateStructDefByName(typeName, updatedDef);
} else if (typeDef.getClass().equals(AtlasEnumDef.class)) {
AtlasEnumDef updatedDef = new AtlasEnumDef((AtlasEnumDef)typeDef);
if (updatedDef.getOptions() == null) {
updatedDef.setOptions(patch.getTypeDefOptions());
} else {
updatedDef.getOptions().putAll(patch.getTypeDefOptions());
}
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateEnumDefByName(typeName, updatedDef);
} else {
throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
patch.getAction(), typeDef.getClass().getSimpleName());
}
} else {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEnumDefStoreV1.java
......@@ -166,6 +166,8 @@ public class AtlasEnumDefStoreV1 extends AtlasAbstractDefStoreV1 implements Atla
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name);
}
typeDefStore.updateTypeVertex(enumDef, vertex);
toVertex(enumDef, vertex);
AtlasEnumDef ret = toEnumDef(vertex);
......@@ -189,6 +191,8 @@ public class AtlasEnumDefStoreV1 extends AtlasAbstractDefStoreV1 implements Atla
throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid);
}
typeDefStore.updateTypeVertex(enumDef, vertex);
toVertex(enumDef, vertex);
AtlasEnumDef ret = toEnumDef(vertex);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
......@@ -39,6 +39,7 @@ import org.apache.atlas.repository.store.graph.AtlasEntityDefStore;
import org.apache.atlas.repository.store.graph.AtlasEnumDefStore;
import org.apache.atlas.repository.store.graph.AtlasStructDefStore;
import org.apache.atlas.repository.store.graph.AtlasTypeDefGraphStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.collections.CollectionUtils;
......@@ -49,6 +50,7 @@ import org.slf4j.LoggerFactory;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
......@@ -199,6 +201,7 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
ret.setProperty(Constants.TIMESTAMP_PROPERTY_KEY, typeDef.getCreateTime().getTime());
ret.setProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, typeDef.getUpdateTime().getTime());
ret.setProperty(Constants.VERSION_PROPERTY_KEY, typeDef.getVersion());
ret.setProperty(Constants.TYPEOPTIONS_PROPERTY_KEY, AtlasType.toJson(typeDef.getOptions()));
return ret;
}
......@@ -219,6 +222,7 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
*/
updateVertexProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDef.getDescription());
updateVertexProperty(vertex, Constants.TYPEVERSION_PROPERTY_KEY, typeDef.getTypeVersion());
updateVertexProperty(vertex, Constants.TYPEOPTIONS_PROPERTY_KEY, AtlasType.toJson(typeDef.getOptions()));
markVertexUpdated(vertex);
}
......@@ -255,6 +259,7 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
Long createTime = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
Long updateTime = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Long version = vertex.getProperty(Constants.VERSION_PROPERTY_KEY, Long.class);
String options = vertex.getProperty(Constants.TYPEOPTIONS_PROPERTY_KEY, String.class);
typeDef.setName(name);
typeDef.setDescription(description);
......@@ -272,6 +277,10 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
if (version != null) {
typeDef.setVersion(version);
}
if (options != null) {
typeDef.setOptions(AtlasType.fromJson(options, Map.class));
}
}
boolean isTypeVertex(AtlasVertex vertex) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment