Skip to content

A
atlas
Commit 180c6e57 by Shwetha GS
Options

ATLAS-1174 Framework to apply updates to types in the type-system…

ATLAS-1174 Framework to apply updates to types in the type-system (sarath.kum4r@gmail.com via shwethags)
parent 71e0e09b
Showing with 806 additions and 56 deletions
common/src/main/java/org/apache/atlas/AtlasConstants.java
......@@ -33,5 +33,6 @@ public final class AtlasConstants {
public static final String ATLAS_REST_ADDRESS_KEY = "atlas.rest.address";
public static final String DEFAULT_ATLAS_REST_ADDRESS = "http://localhost:21000";
public static final int ATLAS_SHUTDOWN_HOOK_PRIORITY = 30;
public static final String DEFAULT_TYPE_VERSION = "1.0";
}
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -54,6 +54,7 @@ public final class Constants {
public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type";
public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description";
public static final String TYPEVERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.version";
/**
* Trait names property key and index name.
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1174 Framework to apply updates to types in the type-system (sarath.kum4r@gmail.com via shwethags)
ATLAS-1155 Errors in Eclipse when I bring in the latest code (davidrad via shwethags)
ATLAS-1098 Atlas allows creation of tag with name "isa" which causes exceptions during search (apoorvnaik via shwethags)
ATLAS-1142 Lineage UI Improvement (kevalbhatt via shwethags)
......
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
......@@ -26,6 +26,7 @@ import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.Constants;
......@@ -90,14 +91,14 @@ public class GraphBackedTypeStore implements ITypeStore {
case STRUCT:
StructType structType = (StructType) dataType;
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(),
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), dataType.getVersion(),
ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of());
break;
case TRAIT:
case CLASS:
HierarchicalType type = (HierarchicalType) dataType;
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.immediateAttrs,
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.getVersion(), type.immediateAttrs,
type.superTypes);
break;
......@@ -108,7 +109,7 @@ public class GraphBackedTypeStore implements ITypeStore {
}
private void storeInGraph(EnumType dataType) {
Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription());
Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), dataType.getVersion());
List<String> values = new ArrayList<>(dataType.values().size());
for (EnumValue enumValue : dataType.values()) {
String key = getPropertyKey(dataType.getName(), enumValue.value);
......@@ -130,9 +131,9 @@ public class GraphBackedTypeStore implements ITypeStore {
return PROPERTY_PREFIX + "edge." + parent + "." + child;
}
private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription,
private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription, String typeVersion,
ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes) throws AtlasException {
Vertex vertex = createVertex(category, typeName, typeDescription);
Vertex vertex = createVertex(category, typeName, typeDescription, typeVersion);
List<String> attrNames = new ArrayList<>();
if (attributes != null) {
for (AttributeInfo attribute : attributes) {
......@@ -152,7 +153,7 @@ public class GraphBackedTypeStore implements ITypeStore {
if (superTypes != null) {
for (String superTypeName : superTypes) {
HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription());
Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription(), AtlasConstants.DEFAULT_TYPE_VERSION);
graphHelper.getOrCreateEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL);
}
}
......@@ -200,7 +201,7 @@ public class GraphBackedTypeStore implements ITypeStore {
for (IDataType attrType : attrDataTypes) {
if (!coreTypes.contains(attrType.getName())) {
Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription());
Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription(), attrType.getVersion());
String label = getEdgeLabel(vertexTypeName, attribute.name);
graphHelper.getOrCreateEdge(vertex, attrVertex, label);
}
......@@ -238,6 +239,7 @@ public class GraphBackedTypeStore implements ITypeStore {
DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
String typeVersion = vertex.getProperty(Constants.TYPEVERSION_PROPERTY_KEY);
LOG.info("Restoring type {}.{}.{}", typeCategory, typeName, typeDescription);
switch (typeCategory) {
case ENUM:
......@@ -246,19 +248,19 @@ public class GraphBackedTypeStore implements ITypeStore {
case STRUCT:
AttributeDefinition[] attributes = getAttributes(vertex, typeName);
structs.add(new StructTypeDefinition(typeName, typeDescription, attributes));
structs.add(new StructTypeDefinition(typeName, typeDescription, typeVersion, attributes));
break;
case CLASS:
ImmutableSet<String> superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex, typeName);
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, superTypes, attributes));
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, typeVersion, superTypes, attributes));
break;
case TRAIT:
superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex, typeName);
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, superTypes, attributes));
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, typeVersion, superTypes, attributes));
break;
default:
......@@ -271,13 +273,14 @@ public class GraphBackedTypeStore implements ITypeStore {
private EnumTypeDefinition getEnumType(Vertex vertex) {
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
String typeVersion = vertex.getProperty(Constants.TYPEVERSION_PROPERTY_KEY);
List<EnumValue> enumValues = new ArrayList<>();
List<String> values = graphHelper.getProperty(vertex, getPropertyKey(typeName));
for (String value : values) {
String valueProperty = getPropertyKey(typeName, value);
enumValues.add(new EnumValue(value, (Integer) graphHelper.getProperty(vertex, valueProperty)));
}
return new EnumTypeDefinition(typeName, typeDescription, enumValues.toArray(new EnumValue[enumValues.size()]));
return new EnumTypeDefinition(typeName, typeDescription, typeVersion, enumValues.toArray(new EnumValue[enumValues.size()]));
}
private ImmutableSet<String> getSuperTypes(Vertex vertex) {
......@@ -324,7 +327,7 @@ public class GraphBackedTypeStore implements ITypeStore {
return vertex;
}
private Vertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription) {
private Vertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription, String typeVersion) {
Vertex vertex = findVertex(category, typeName);
if (vertex == null) {
LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
......@@ -341,6 +344,16 @@ public class GraphBackedTypeStore implements ITypeStore {
} else {
LOG.debug(" type description is null ");
}
if (typeVersion != null) {
String oldVersion = getPropertyKey(Constants.TYPEVERSION_PROPERTY_KEY);
if (!typeVersion.equals(oldVersion)) {
setProperty(vertex, Constants.TYPEVERSION_PROPERTY_KEY, typeVersion);
LOG.info(" updating type {} to version {}", typeName, typeVersion);
}
} else {
LOG.info(" type version is null ");
}
return vertex;
}
}
repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonSyntaxException;
import org.apache.atlas.services.AtlasTypePatch.PatchContent;
import org.apache.atlas.services.AtlasTypePatch.PatchData;
import org.apache.atlas.services.AtlasTypePatch.PatchResult;
import org.apache.atlas.services.AtlasTypePatch.PatchStatus;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUpdateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class AtlasPatchHandler {
private static final Logger LOG = LoggerFactory.getLogger(AtlasPatchHandler.class);
public static void handlePatches(DefaultMetadataService metadataService, TypeSystem typeSystem) throws TypeUpdateException {
Map<String, AtlasTypePatch> patchHandlerMap = initializePatchHandlerMap(metadataService, typeSystem);
if (patchHandlerMap == null || patchHandlerMap.isEmpty())
return;
String patchDirName = ReservedTypesRegistrar.getTypesDir() + File.separator + "patches";
LOG.info("Checking for any patches to be applied to the type system in " + patchDirName);
File patchDir = new File(patchDirName);
if (!patchDir.exists()) {
LOG.info("Patch directory {} doesn't exist, not applying any patches", patchDirName);
return;
}
File[] patchFiles = patchDir.listFiles();
if (patchFiles == null || patchFiles.length == 0) {
LOG.info("No patch files found in {}, not applying any patches", patchDirName);
return;
}
// Sort the patch files based on file name.
Arrays.sort(patchFiles, new Comparator<File>() {
public int compare(File f1, File f2) {
return String.valueOf(f1.getName()).compareTo(f2.getName());
}
});
LOG.info("Found " + patchFiles.length + " patch files to process.");
int patchNumber = 0;
Gson gson = initializeGson();
AtlasTypePatch typePatch;
for (File patchFile : patchFiles) {
try {
LOG.info("Processing patch file " + (++patchNumber) + " - " + patchFile.getAbsolutePath());
String content = new String(Files.readAllBytes(patchFile.toPath()), StandardCharsets.UTF_8);
PatchContent patchContent = gson.fromJson(content, PatchContent.class);
PatchData[] patchDatas = patchContent.getPatches();
PatchResult result;
int patchCounter = 0;
for (PatchData patch : patchDatas) {
typePatch = patchHandlerMap.get(patch.getAction());
if (typePatch != null) {
result = typePatch.applyPatch(patch);
if (result != null) {
LOG.info(result.getMessage() + " Patch " + (++patchCounter) + " out of " + patchDatas.length + " processed in : " + patchFile.toPath());
if (result.getStatus().equals(PatchStatus.FAILED)) {
throw new TypeUpdateException(result.getMessage() + " patch " + patchNumber + " failed in :" + patchFile.getAbsolutePath());
}
}
}
}
} catch (IOException e) {
throw new TypeUpdateException("Unable to read patch file from " + patchFile.getAbsolutePath());
} catch (JsonSyntaxException e) {
throw new TypeUpdateException("Invalid non-parseable JSON patch file in " + patchFile.getAbsolutePath());
}
}
LOG.info("Processed " + patchFiles.length + " patch files.");
}
private static Map<String, AtlasTypePatch> initializePatchHandlerMap(DefaultMetadataService metadataService, TypeSystem typeSystem) {
Map<String, AtlasTypePatch> patchHandlerMap = new HashMap<String, AtlasTypePatch>();
List<AtlasTypePatch> patchers = new ArrayList<AtlasTypePatch>();
// Register new patch classes here
patchers.add(new AtlasTypeAttributePatch(metadataService, typeSystem));
for (AtlasTypePatch patcher : patchers) {
for (String action : patcher.getSupportedActions()) {
patchHandlerMap.put(action, patcher);
}
}
return patchHandlerMap;
}
public static Gson initializeGson() {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.registerTypeAdapter(Multiplicity.class, new MultiplicityDeserializer());
gsonBuilder.setFieldNamingPolicy(FieldNamingPolicy.IDENTITY);
Gson gson = gsonBuilder.create();
return gson;
}
static class MultiplicityDeserializer implements JsonDeserializer<Multiplicity> {
@Override
public Multiplicity deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
String multiplicityString = json.getAsString().toLowerCase();
Multiplicity m = null;
switch (multiplicityString) {
case "optional":
m = Multiplicity.OPTIONAL;
break;
case "required":
m = Multiplicity.REQUIRED;
break;
case "collection":
m = Multiplicity.COLLECTION;
break;
case "set":
m = Multiplicity.SET;
break;
default:
break;
}
return m;
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
public class AtlasTypeAttributePatch extends AtlasTypePatch {
private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeAttributePatch.class);
private static final String STRUCT_TYPE = "STRUCT";
private static final String CLASS_TYPE = "CLASS";
private static final String TRAIT_TYPE = "TRAIT";
private static final String[] SUPPORTED_ACTIONS = new String[] { "ADD_ATTRIBUTE", "UPDATE_ATTRIBUTE", "DELETE_ATTRIBUTE" };
public AtlasTypeAttributePatch(DefaultMetadataService metadataService, TypeSystem typeSystem) {
super(metadataService, typeSystem, SUPPORTED_ACTIONS);
}
/********* SAMPLE PATCH DATA ***********
*
*
{
"patches": [
{
"action": "ADD_ATTRIBUTE",
"typeName": "hive_column",
"applyToVersion": "1.0",
"updateToVersion": "2.0",
"params": {
"optionalParam1" : "true",
"optionalParam2" : "false"
},
"attributeDefinitions": [
{
"name": "position",
"dataTypeName": "string",
"multiplicity": "optional",
"isComposite": false,
"isUnique": false,
"isIndexable": false,
"reverseAttributeName": null
}
]
}
]
}
*
*/
@Override
public PatchResult applyPatch(PatchData patchData) {
String typeName = patchData.getTypeName();
String applyVersion = patchData.getApplyToVersion();
TypesDef updatedTypeDef;
PatchResult result;
try {
// Check if type exists in type system
if (checkIfTypeExists(typeName, metadataService)) {
TypesDef typesDef = getTypeDef(typeName);
String currentVersion = getTypeVersion(typeName);
// Check version to apply patch
if (currentVersion == null || currentVersion.equalsIgnoreCase(applyVersion) || currentVersion.startsWith(applyVersion + ".")) {
updatedTypeDef = updateTypesDef(typesDef, patchData);
if (updatedTypeDef != null) {
metadataService.updateType(TypesSerialization.toJson(updatedTypeDef));
LOG.info("updated " + patchData.getTypeName() + " type from version " + patchData.getApplyToVersion() + " to " + patchData.getUpdateToVersion());
result = new PatchResult("patch applied successfully!", PatchStatus.SUCCESS);
} else {
LOG.error("failed to create updated typedef for type=" +typeName+ "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() );
result = new PatchResult("patch failed!", PatchStatus.FAILED);
}
} else {
LOG.info("patch skipped for " + patchData.getTypeName());
result = new PatchResult("patch skipped!", PatchStatus.SKIPPED);
}
} else {
LOG.error("failed to apply patch (typeName=" + typeName + "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() + "): type doesn't exist");
result = new PatchResult("patch failed!", PatchStatus.FAILED);
}
} catch (AtlasException e) {
LOG.error("error in updating type for " + patchData.getTypeName());
result = new PatchResult("unable to update type", PatchStatus.FAILED);
} catch (JSONException e) {
LOG.error("error in updating typedef for " + patchData.getTypeName());
result = new PatchResult("unable to update typedef", PatchStatus.FAILED);
}
return result;
}
public TypesDef updateTypesDef(TypesDef typesDef, PatchData patchData) throws AtlasException, JSONException {
AttributeDefinition[] typeAttributes = getAttributesFromTypesDef(typesDef, patchData.getTypeName());
AttributeDefinition[] patchAttributes = patchData.getAttributeDefinitions();
AttributeDefinition[] newAttributes = new AttributeDefinition[0];
String patchAction = patchData.getAction();
TypesDef newTypeDef = null;
if (patchAction != null && typeAttributes != null && patchAttributes != null) {
switch (patchAction) {
case "ADD_ATTRIBUTE":
LOG.info("adding new attribute to type {}", patchData.getTypeName());
newAttributes = addAttributes(typeAttributes, patchAttributes);
break;
case "DELETE_ATTRIBUTE":
LOG.info("deleting attribute from type {}", patchData.getTypeName());
newAttributes = deleteAttributes(typeAttributes, patchAttributes);
break;
case "UPDATE_ATTRIBUTE":
LOG.info("updating attribute in type {}", patchData.getTypeName());
newAttributes = updateAttributes(typeAttributes, patchAttributes);
break;
default:
LOG.info("invalid action " + patchAction + ", supported actions: " + Arrays.toString(SUPPORTED_ACTIONS));
break;
}
newTypeDef = createTypeDefWithNewAttributes(typesDef, patchData.getTypeName(), newAttributes, patchData.getUpdateToVersion());
}
return newTypeDef;
}
private AttributeDefinition[] addAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>(Arrays.asList(typeAttributes));
Map<String, AttributeDefinition> typeAttrs = getAttributeNamesFromDefinition(typeAttributes);
for (AttributeDefinition attr : patchAttributes) {
if (!typeAttrs.containsKey(attr.name))
newAttrList.add(attr);
}
return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
}
private AttributeDefinition[] deleteAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>();
Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes);
for (AttributeDefinition attr : typeAttributes) {
if (!patchAttrs.containsKey(attr.name))
newAttrList.add(attr);
}
return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
}
private AttributeDefinition[] updateAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>();
Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes);
AttributeDefinition newAttr;
for (AttributeDefinition attr : typeAttributes) {
newAttr = patchAttrs.get(attr.name);
if (patchAttrs.containsKey(attr.name) && checkIfAttributeUpdatable(attr, newAttr)) {
newAttrList.add(newAttr);
} else {
newAttrList.add(attr);
}
}
return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
}
private TypesDef createTypeDefWithNewAttributes(TypesDef typesDef, String typeName, AttributeDefinition[] newAttributes, String newVersion) throws AtlasException {
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
String dataType = getDataType(typeName).toUpperCase();
switch (dataType) {
case STRUCT_TYPE:
StructTypeDefinition structType = typesDef.structTypesAsJavaList().get(0);
structs.add(new StructTypeDefinition(structType.typeName, structType.typeDescription, newVersion, newAttributes));
break;
case CLASS_TYPE:
HierarchicalTypeDefinition<ClassType> classType = typesDef.classTypesAsJavaList().get(0);
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, classType.typeName, classType.typeDescription, newVersion, classType.superTypes, newAttributes));
break;
case TRAIT_TYPE:
HierarchicalTypeDefinition<TraitType> traitType = typesDef.traitTypesAsJavaList().get(0);
traits.add(new HierarchicalTypeDefinition(TraitType.class, traitType.typeName, traitType.typeDescription, newVersion, traitType.superTypes, newAttributes));
break;
default:
LOG.error("unhandled datatype {} when creating new typedef", dataType);
}
return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
}
private AttributeDefinition[] getAttributesFromTypesDef(TypesDef typesDef, String typeName) throws AtlasException {
AttributeDefinition[] typeAttributes = null;
String dataType = getDataType(typeName).toUpperCase();
switch (dataType) {
case STRUCT_TYPE:
typeAttributes = typesDef.structTypesAsJavaList().get(0).attributeDefinitions;
break;
case CLASS_TYPE:
typeAttributes = typesDef.classTypesAsJavaList().get(0).attributeDefinitions;
break;
case TRAIT_TYPE:
typeAttributes = typesDef.traitTypesAsJavaList().get(0).attributeDefinitions;
break;
default:
LOG.error("unhandled datatype {}", dataType);
}
return typeAttributes;
}
private Map<String, AttributeDefinition> getAttributeNamesFromDefinition(AttributeDefinition[] attribDef) {
Map<String, AttributeDefinition> attrsMap = new HashMap<String, AttributeDefinition>();
for (AttributeDefinition attr : attribDef) {
attrsMap.put(attr.name, attr);
}
return attrsMap;
}
private boolean checkIfAttributeUpdatable(AttributeDefinition attr, AttributeDefinition newAttr) {
boolean result = false;
if (!attr.equals(newAttr) && (attr.multiplicity == Multiplicity.REQUIRED
&& newAttr.multiplicity == Multiplicity.OPTIONAL)) {
result = true;
}
return result;
}
// Returns the datatype the typename belongs to - PRIMITIVE, ENUM, ARRAY, MAP, STRUCT, TRAIT, CLASS
private String getDataType(String typeName) throws AtlasException {
IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
return dataType.getTypeCategory().toString();
}
private String getTypeVersion(String typeName) throws AtlasException {
return typeSystem.getDataType(IDataType.class, typeName).getVersion();
}
private TypesDef getTypeDef(String typeName) throws AtlasException {
return TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
}
private static boolean checkIfTypeExists(String typeName, DefaultMetadataService metadataService) {
boolean result = true;
try {
metadataService.getTypeDefinition(typeName);
} catch (AtlasException e) {
result = false;
}
return result;
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.TypeSystem;
import java.util.Map;
public abstract class AtlasTypePatch {
protected final TypeSystem typeSystem;
protected final DefaultMetadataService metadataService;
protected final String[] supportedActions;
protected AtlasTypePatch(DefaultMetadataService metadataService, TypeSystem typeSystem, String[] supportedActions) {
this.metadataService = metadataService;
this.typeSystem = typeSystem;
this.supportedActions = supportedActions;
}
public final String[] getSupportedActions() { return supportedActions; }
public abstract PatchResult applyPatch(PatchData patch);
public enum PatchStatus { SUCCESS, FAILED, SKIPPED }
public class PatchResult {
private String message;
private PatchStatus status;
public PatchResult(String message, PatchStatus status) {
this.message = message;
this.status = status;
}
public String getMessage() { return message; }
public void setMessage(String message) { this.message = message; }
public PatchStatus getStatus() { return status; }
public void setStatus(PatchStatus status) { this.status = status; }
}
/**
* A class to capture patch content.
*/
public class PatchContent {
private PatchData[] patches;
public PatchData[] getPatches() {
return patches;
}
}
public static class PatchData {
private String action;
private String typeName;
private String applyToVersion;
private String updateToVersion;
private Map<String, String> params;
private AttributeDefinition[] attributeDefinitions;
public PatchData(String action, String typeName, String applyToVersion, String updateToVersion, Map<String, String> params, AttributeDefinition[] attributeDefinitions) {
this.action = action;
this.typeName = typeName;
this.applyToVersion = applyToVersion;
this.updateToVersion = updateToVersion;
this.params = params;
this.attributeDefinitions = attributeDefinitions;
}
public String getAction() { return action; }
public String getTypeName() { return typeName; }
public String getApplyToVersion() { return applyToVersion; }
public String getUpdateToVersion() { return updateToVersion; }
public Map<String, String> getParams() { return params; }
public AttributeDefinition[] getAttributeDefinitions() { return attributeDefinitions; }
}
}
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -153,6 +153,8 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
restoreTypeSystem();
}
AtlasPatchHandler.handlePatches(this, typeSystem);
maxAuditResults = configuration.getShort(CONFIG_MAX_AUDIT_RESULTS, DEFAULT_MAX_AUDIT_RESULTS);
}
......
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
......@@ -36,6 +36,10 @@ import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.audit.HBaseTestUtils;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.AtlasTypeAttributePatch;
import org.apache.atlas.services.AtlasTypePatch;
import org.apache.atlas.services.AtlasTypePatch.PatchData;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct;
......@@ -111,7 +115,6 @@ public class DefaultMetadataServiceTest {
private Referenceable table;
private Id tableId;
private final String NAME = "name";
......@@ -1092,6 +1095,43 @@ public class DefaultMetadataServiceTest {
}
@Test
public void testPatchFrameworkForTypeUpdate() throws AtlasException, JSONException {
String typeName = "test_type_" + RandomStringUtils.randomAlphanumeric(10);
HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef(typeName, ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("type_attr1", DataTypes.STRING_TYPE));
TypesDef typesDef = new TypesDef(typeDef, false);
metadataService.createType(TypesSerialization.toJson(typesDef));
AtlasTypeAttributePatch patch = new AtlasTypeAttributePatch((DefaultMetadataService) metadataService, TypeSystem.getInstance());
AttributeDefinition[] attrDefs = new AttributeDefinition[]{
new AttributeDefinition("type_attr2", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("type_attr3", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)};
// Testing add attribute patch
AtlasTypePatch.PatchData addAttributePatch = new PatchData("ADD_ATTRIBUTE", typeName, "1.0", "2.0", null, attrDefs);
TypesDef newAttrTypesDef = patch.updateTypesDef(typesDef, addAttributePatch);
metadataService.updateType(TypesSerialization.toJson(newAttrTypesDef));
TypesDef addedTypesDef = TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
// test added attributes and update version to 2.0
assertEquals(addedTypesDef.classTypes().head().attributeDefinitions.length, 3);
assertEquals(addedTypesDef.classTypes().head().typeVersion, "2.0");
// Testing update attribute patch
AttributeDefinition[] updateAttrDef = new AttributeDefinition[]{
new AttributeDefinition("type_attr1", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)};
AtlasTypePatch.PatchData updateAttributePatch = new PatchData("UPDATE_ATTRIBUTE", typeName, "2.0", "3.0", null, updateAttrDef);
TypesDef updateAttrTypesDef = patch.updateTypesDef(addedTypesDef, updateAttributePatch);
metadataService.updateType(TypesSerialization.toJson(updateAttrTypesDef));
TypesDef updatedTypesDef = TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
// test update attribute to optional and update version to 3.0
assertEquals(updatedTypesDef.classTypes().head().attributeDefinitions[0].multiplicity, Multiplicity.OPTIONAL);
assertEquals(updatedTypesDef.classTypes().head().typeVersion, "3.0");
}
@Test
public void testAuditEventsInvalidParams() throws Exception {
//entity id can't be null
try {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableSortedMap;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import java.io.IOException;
......@@ -30,12 +31,22 @@ abstract class AbstractDataType<T> implements IDataType<T> {
public final String name;
public final String description;
public final String version;
public AbstractDataType(String name, String description) {
super();
this.name = name;
this.description = description;
this.version = AtlasConstants.DEFAULT_TYPE_VERSION;
}
public AbstractDataType(String name, String description, String version) {
super();
this.name = name;
this.description = description;
this.version = version;
}
protected T convertNull(Multiplicity m) throws AtlasException {
......@@ -99,5 +110,10 @@ abstract class AbstractDataType<T> implements IDataType<T> {
public String getDescription() {
return description;
}
@Override
public String getVersion() {
return version;
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
......@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct;
......@@ -51,13 +52,22 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
public final Map<AttributeInfo, List<String>> infoToNameMap;
ClassType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTypes, int numFields) {
super(typeSystem, ClassType.class, name, description, superTypes, numFields);
this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, numFields);
}
ClassType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTypes, int numFields) {
super(typeSystem, ClassType.class, name, description, version, superTypes, numFields);
infoToNameMap = null;
}
ClassType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTypes, AttributeInfo... fields)
throws AtlasException {
super(typeSystem, ClassType.class, name, description, superTypes, fields);
this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, fields);
}
ClassType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTypes, AttributeInfo... fields)
throws AtlasException {
super(typeSystem, ClassType.class, name, description, version, superTypes, fields);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import scala.math.BigInt;
......@@ -37,7 +38,11 @@ public class EnumType extends AbstractDataType<EnumValue> {
}
protected EnumType(TypeSystem typeSystem, String name, String description, EnumValue... values) {
super(name, description);
this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, values);
}
protected EnumType(TypeSystem typeSystem, String name, String description, String version, EnumValue... values) {
super(name, description, version);
this.typeSystem = typeSystem;
ImmutableMap.Builder<String, EnumValue> b1 = new ImmutableMap.Builder();
ImmutableMap.Builder<Integer, EnumValue> b2 = new ImmutableMap.Builder();
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.AtlasConstants;
import java.util.Arrays;
......@@ -26,16 +27,22 @@ public final class EnumTypeDefinition {
public final String name;
public final String description;
public final String version;
public final EnumValue[] enumValues;
public EnumTypeDefinition(String name, EnumValue... enumValues) {
this(name, null, enumValues);
this(name, null, AtlasConstants.DEFAULT_TYPE_VERSION, enumValues);
}
public EnumTypeDefinition(String name, String description, EnumValue... enumValues) {
this(name, description, AtlasConstants.DEFAULT_TYPE_VERSION, enumValues);
}
public EnumTypeDefinition(String name, String description, String version, EnumValue... enumValues) {
this.name = ParamChecker.notEmpty(name, "Enum type name");
this.description = description;
this.enumValues = ParamChecker.notNullElements(enumValues, "Enum values");
this.version = version;
}
@Override
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
......@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.UnmodifiableIterator;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.persistence.DownCastStructInstance;
......@@ -70,7 +71,12 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
*/
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableSet<String> superTypes,
int numFields) {
super(name, description);
this( typeSystem, superTypeClass, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, numFields);
}
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, String version, ImmutableSet<String> superTypes,
int numFields) {
super(name, description, version);
this.typeSystem = typeSystem;
this.superTypeClass = superTypeClass;
this.fieldMapping = null;
......@@ -86,7 +92,12 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
}
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableSet<String> superTypes,
AttributeInfo... fields) throws AtlasException {
super(name, description);
this(typeSystem, superTypeClass, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, fields);
}
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, String version, ImmutableSet<String> superTypes,
AttributeInfo... fields) throws AtlasException {
super(name, description, version);
this.typeSystem = typeSystem;
this.superTypeClass = superTypeClass;
Pair<FieldMapping, ImmutableMap<String, String>> p = constructFieldMapping(superTypes, fields);
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java
......@@ -21,34 +21,41 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.utils.ParamChecker;
public class HierarchicalTypeDefinition<T extends HierarchicalType> extends StructTypeDefinition {
public final ImmutableSet<String> superTypes;
public final String hierarchicalMetaTypeName;
/**
* Used for json deserialization only.
* not intended public consumption
* @param hierarchicalMetaTypeName
* @param typeName
* @param typeDescription
* @param superTypes
* @param attributeDefinitions
* @throws ClassNotFoundException
*/
@InterfaceAudience.Private
public HierarchicalTypeDefinition(String hierarchicalMetaTypeName, String typeName, String typeDescription, String[] superTypes,
AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, ImmutableSet.copyOf(superTypes),
public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, ImmutableSet<String> superTypes,
AttributeDefinition[] attributeDefinitions) {
this(hierarchicalMetaType, typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes,
attributeDefinitions);
}
public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, ImmutableSet<String> superTypes,
AttributeDefinition[] attributeDefinitions) {
super(typeName, typeDescription, false, attributeDefinitions);
hierarchicalMetaTypeName = hierarchicalMetaType.getName();
// Used only for de-serializing JSON String to typedef.
public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String typeVersion, String[] superTypes, AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, typeVersion, ImmutableSet.copyOf(superTypes), attributeDefinitions);
}
// Used only for de-serializing JSON String to typedef (no typeVersion).
public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String[] superTypes, AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, ImmutableSet.copyOf(superTypes), attributeDefinitions);
}
// Used only for serializing typedef to JSON String.
public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String typeVersion, ImmutableSet<String> superTypes, AttributeDefinition[] attributeDefinitions, String typeDef) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, typeVersion, superTypes, attributeDefinitions);
}
// Used only for serializing typedef to JSON String (no typeVersion).
public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, ImmutableSet<String> superTypes, AttributeDefinition[] attributeDefinitions, String typeDef) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attributeDefinitions);
}
public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, String typeVersion, ImmutableSet<String> superTypes, AttributeDefinition[] attributeDefinitions) {
super(typeName, typeDescription, typeVersion, false, attributeDefinitions);
this.hierarchicalMetaTypeName = hierarchicalMetaType.getName();
this.superTypes = superTypes == null ? ImmutableSet.<String>of() : superTypes;
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
......@@ -55,5 +55,7 @@ public interface IDataType<T> {
void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException;
String getDescription();
String getVersion();
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
......@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
......@@ -42,7 +43,11 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
private final TypedStructHandler handler;
protected StructType(TypeSystem typeSystem, String name, String description, int numFields) {
super(name, description);
this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, numFields);
}
protected StructType(TypeSystem typeSystem, String name, String description, String version, int numFields) {
super(name, description, version);
this.typeSystem = typeSystem;
this.fieldMapping = null;
infoToNameMap = null;
......@@ -52,7 +57,12 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
protected StructType(TypeSystem typeSystem, String name, String description, AttributeInfo... fields)
throws AtlasException {
super(name, description);
this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, fields);
}
protected StructType(TypeSystem typeSystem, String name, String description, String version, AttributeInfo... fields)
throws AtlasException {
super(name, description, version);
this.typeSystem = typeSystem;
this.fieldMapping = constructFieldMapping(fields);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping);
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.utils.ParamChecker;
import java.util.Arrays;
......@@ -26,29 +27,45 @@ public class StructTypeDefinition {
public final String typeName;
public final String typeDescription;//optional field
public final String typeVersion;
public final AttributeDefinition[] attributeDefinitions;
protected StructTypeDefinition(String typeName, String typeDescription, boolean validate,
AttributeDefinition... attributeDefinitions) {
this(typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, validate, attributeDefinitions);
}
protected StructTypeDefinition(String typeName, String typeDescription, String typeVersion, boolean validate,
AttributeDefinition... attributeDefinitions) {
this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
this.typeDescription = typeDescription;
if (validate) {
ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
}
this.attributeDefinitions = attributeDefinitions;
this.typeVersion = typeVersion;
}
public StructTypeDefinition(String typeName, AttributeDefinition[] attributeDefinitions) {
this(typeName, null, attributeDefinitions);
this(typeName, null, AtlasConstants.DEFAULT_TYPE_VERSION, attributeDefinitions);
}
public StructTypeDefinition(String typeName, String typeDescription,
AttributeDefinition[] attributeDefinitions) {
this(typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, attributeDefinitions);
}
public StructTypeDefinition(String typeName, String typeDescription, String typeVersion,
AttributeDefinition[] attributeDefinitions) {
this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
this.typeDescription = typeDescription;
this.typeVersion = typeVersion;
this.attributeDefinitions = ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
}
@Override
public boolean equals(Object o) {
if (this == o) {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
......@@ -37,14 +38,23 @@ public class TraitType extends HierarchicalType<TraitType, IStruct>
private final TypedStructHandler handler;
TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, int numFields) {
super(typeSystem, TraitType.class, name, description, superTraits, numFields);
this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTraits, numFields);
}
TraitType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTraits, int numFields) {
super(typeSystem, TraitType.class, name, description, version, superTraits, numFields);
handler = null;
infoToNameMap = null;
}
TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, AttributeInfo... fields)
throws AtlasException {
super(typeSystem, TraitType.class, name, description, superTraits, fields);
this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTraits, fields);
}
TraitType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTraits, AttributeInfo... fields)
throws AtlasException {
super(typeSystem, TraitType.class, name, description, version, superTraits, fields);
handler = new TypedStructHandler(this);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -311,7 +311,7 @@ public class TypeSystem {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
}
EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.enumValues);
EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.version, eDef.enumValues);
typeCache.put(eT);
return eT;
}
......@@ -408,7 +408,7 @@ public class TypeSystem {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
}
EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.enumValues);
EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.version, eDef.enumValues);
transientTypes.put(eDef.name, eT);
}
......@@ -417,7 +417,7 @@ public class TypeSystem {
if (!update && isRegistered(sDef.typeName)) {
throw new TypeExistsException(String.format("Cannot redefine type %s", sDef.typeName));
}
StructType sT = new StructType(this, sDef.typeName, sDef.typeDescription, sDef.attributeDefinitions.length);
StructType sT = new StructType(this, sDef.typeName, sDef.typeDescription, sDef.typeVersion, sDef.attributeDefinitions.length);
structNameToDefMap.put(sDef.typeName, sDef);
transientTypes.put(sDef.typeName, sT);
}
......@@ -427,7 +427,7 @@ public class TypeSystem {
if (!update && isRegistered(traitDef.typeName)) {
throw new TypeExistsException(String.format("Cannot redefine type %s", traitDef.typeName));
}
TraitType tT = new TraitType(this, traitDef.typeName, traitDef.typeDescription, traitDef.superTypes,
TraitType tT = new TraitType(this, traitDef.typeName, traitDef.typeDescription, traitDef.typeVersion, traitDef.superTypes,
traitDef.attributeDefinitions.length);
traitNameToDefMap.put(traitDef.typeName, traitDef);
transientTypes.put(traitDef.typeName, tT);
......@@ -439,7 +439,7 @@ public class TypeSystem {
throw new TypeExistsException(String.format("Cannot redefine type %s", classDef.typeName));
}
ClassType cT = new ClassType(this, classDef.typeName, classDef.typeDescription, classDef.superTypes,
ClassType cT = new ClassType(this, classDef.typeName, classDef.typeDescription, classDef.typeVersion, classDef.superTypes,
classDef.attributeDefinitions.length);
classNameToDefMap.put(classDef.typeName, classDef);
transientTypes.put(classDef.typeName, cT);
......@@ -526,7 +526,7 @@ public class TypeSystem {
infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
}
StructType type = new StructType(this, def.typeName, def.typeDescription, infos);
StructType type = new StructType(this, def.typeName, def.typeDescription, def.typeVersion, infos);
transientTypes.put(def.typeName, type);
return type;
}
......@@ -539,9 +539,9 @@ public class TypeSystem {
}
try {
Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, ImmutableSet.class,
Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, String.class, ImmutableSet.class,
AttributeInfo[].class);
U type = cons.newInstance(this, def.typeName, def.typeDescription, def.superTypes, infos);
U type = cons.newInstance(this, def.typeName, def.typeDescription, def.typeVersion, def.superTypes, infos);
transientTypes.put(def.typeName, type);
return type;
} catch (Exception e) {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
......@@ -33,6 +33,7 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.typesystem.types.TypeSystem;
import scala.collection.JavaConversions;
......@@ -76,6 +77,11 @@ public class TypesUtil {
public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
return createTraitTypeDef(name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attrDefs);
}
public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description, String version,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(TraitType.class, name, description, superTypes, attrDefs);
}
......@@ -94,6 +100,11 @@ public class TypesUtil {
public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, String description,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
return createClassTypeDef(name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attrDefs);
}
public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, String description, String version,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(ClassType.class, name, description, superTypes, attrDefs);
}
......
typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala
......@@ -21,6 +21,7 @@ package org.apache.atlas.typesystem.json
import java.text.SimpleDateFormat
import com.google.common.collect.ImmutableList
import org.apache.atlas.AtlasException
import org.apache.atlas.AtlasConstants
import org.apache.atlas.typesystem.TypesDef
import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, MapType, TypeCategory}
import org.apache.atlas.typesystem.types._
......@@ -131,28 +132,28 @@ object TypesSerialization {
private def convertEnumTypeToEnumTypeDef(et: EnumType) = {
val eVals: Seq[EnumValue] = et.valueMap.values().toSeq
new EnumTypeDefinition(et.name, et.description, eVals: _*)
new EnumTypeDefinition(et.name, et.description, et.version, eVals: _*)
}
private def convertStructTypeToStructDef(st: StructType): StructTypeDefinition = {
val aDefs: Iterable[AttributeDefinition] =
st.fieldMapping.fields.values().map(convertAttributeInfoToAttributeDef(_))
new StructTypeDefinition(st.name, st.description, aDefs.toArray)
new StructTypeDefinition(st.name, st.description, st.version, aDefs.toArray)
}
private def convertTraitTypeToHierarchicalTypeDefinition(tt: TraitType): HierarchicalTypeDefinition[TraitType] = {
val aDefs: Iterable[AttributeDefinition] =
tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.description, tt.superTypes, aDefs.toArray)
new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.description, tt.version, tt.superTypes, aDefs.toArray)
}
private def convertClassTypeToHierarchicalTypeDefinition(tt: ClassType): HierarchicalTypeDefinition[ClassType] = {
val aDefs: Iterable[AttributeDefinition] =
tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.description, tt.superTypes, aDefs.toArray)
new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.description, tt.version, tt.superTypes, aDefs.toArray)
}
def convertToTypesDef(ts: TypeSystem, export: IDataType[_] => Boolean): TypesDef = {
......@@ -232,6 +233,11 @@ trait TypeHelpers {
def createTraitTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[TraitType] = {
createTraitTypeDef(name, None, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attrDefs:_*)
}
def createTraitTypeDef(name: String, description: Option[String], version: String,superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[TraitType] = {
val sts = ImmutableSet.copyOf(superTypes.toArray)
return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name, description.getOrElse(null),
sts, attrDefs.toArray)
......@@ -244,11 +250,16 @@ trait TypeHelpers {
def createClassTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[ClassType] = {
createClassTypeDef( name, None, None, superTypes, attrDefs:_*)
}
def createClassTypeDef(name: String, description: Option[String], version: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[ClassType] = {
val sts = ImmutableSet.copyOf(superTypes.toArray)
return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name, description.getOrElse(null),
sts, attrDefs.toArray)
return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name, description.getOrElse(null), AtlasConstants.DEFAULT_TYPE_VERSION, sts, attrDefs.toArray)
}
@throws(classOf[AtlasException])
def defineClassType(ts: TypeSystem, classDef: HierarchicalTypeDefinition[ClassType]): ClassType = {
ts.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.of[StructTypeDefinition],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment