Commit 180c6e57 by Shwetha GS

ATLAS-1174 Framework to apply updates to types in the type-system…

ATLAS-1174 Framework to apply updates to types in the type-system (sarath.kum4r@gmail.com via shwethags)
parent 71e0e09b
...@@ -33,5 +33,6 @@ public final class AtlasConstants { ...@@ -33,5 +33,6 @@ public final class AtlasConstants {
public static final String ATLAS_REST_ADDRESS_KEY = "atlas.rest.address"; public static final String ATLAS_REST_ADDRESS_KEY = "atlas.rest.address";
public static final String DEFAULT_ATLAS_REST_ADDRESS = "http://localhost:21000"; public static final String DEFAULT_ATLAS_REST_ADDRESS = "http://localhost:21000";
public static final int ATLAS_SHUTDOWN_HOOK_PRIORITY = 30; public static final int ATLAS_SHUTDOWN_HOOK_PRIORITY = 30;
public static final String DEFAULT_TYPE_VERSION = "1.0";
} }
...@@ -54,6 +54,7 @@ public final class Constants { ...@@ -54,6 +54,7 @@ public final class Constants {
public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type"; public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type";
public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name"; public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description"; public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description";
public static final String TYPEVERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.version";
/** /**
* Trait names property key and index name. * Trait names property key and index name.
......
...@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ...@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES: ALL CHANGES:
ATLAS-1174 Framework to apply updates to types in the type-system (sarath.kum4r@gmail.com via shwethags)
ATLAS-1155 Errors in Eclipse when I bring in the latest code (davidrad via shwethags) ATLAS-1155 Errors in Eclipse when I bring in the latest code (davidrad via shwethags)
ATLAS-1098 Atlas allows creation of tag with name "isa" which causes exceptions during search (apoorvnaik via shwethags) ATLAS-1098 Atlas allows creation of tag with name "isa" which causes exceptions during search (apoorvnaik via shwethags)
ATLAS-1142 Lineage UI Improvement (kevalbhatt via shwethags) ATLAS-1142 Lineage UI Improvement (kevalbhatt via shwethags)
......
...@@ -26,6 +26,7 @@ import com.thinkaurelius.titan.core.TitanGraph; ...@@ -26,6 +26,7 @@ import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction; import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
...@@ -90,14 +91,14 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -90,14 +91,14 @@ public class GraphBackedTypeStore implements ITypeStore {
case STRUCT: case STRUCT:
StructType structType = (StructType) dataType; StructType structType = (StructType) dataType;
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), dataType.getVersion(),
ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of()); ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of());
break; break;
case TRAIT: case TRAIT:
case CLASS: case CLASS:
HierarchicalType type = (HierarchicalType) dataType; HierarchicalType type = (HierarchicalType) dataType;
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.immediateAttrs, storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.getVersion(), type.immediateAttrs,
type.superTypes); type.superTypes);
break; break;
...@@ -108,7 +109,7 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -108,7 +109,7 @@ public class GraphBackedTypeStore implements ITypeStore {
} }
private void storeInGraph(EnumType dataType) { private void storeInGraph(EnumType dataType) {
Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription()); Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), dataType.getVersion());
List<String> values = new ArrayList<>(dataType.values().size()); List<String> values = new ArrayList<>(dataType.values().size());
for (EnumValue enumValue : dataType.values()) { for (EnumValue enumValue : dataType.values()) {
String key = getPropertyKey(dataType.getName(), enumValue.value); String key = getPropertyKey(dataType.getName(), enumValue.value);
...@@ -130,9 +131,9 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -130,9 +131,9 @@ public class GraphBackedTypeStore implements ITypeStore {
return PROPERTY_PREFIX + "edge." + parent + "." + child; return PROPERTY_PREFIX + "edge." + parent + "." + child;
} }
private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription, private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription, String typeVersion,
ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes) throws AtlasException { ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes) throws AtlasException {
Vertex vertex = createVertex(category, typeName, typeDescription); Vertex vertex = createVertex(category, typeName, typeDescription, typeVersion);
List<String> attrNames = new ArrayList<>(); List<String> attrNames = new ArrayList<>();
if (attributes != null) { if (attributes != null) {
for (AttributeInfo attribute : attributes) { for (AttributeInfo attribute : attributes) {
...@@ -152,7 +153,7 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -152,7 +153,7 @@ public class GraphBackedTypeStore implements ITypeStore {
if (superTypes != null) { if (superTypes != null) {
for (String superTypeName : superTypes) { for (String superTypeName : superTypes) {
HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName); HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription()); Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription(), AtlasConstants.DEFAULT_TYPE_VERSION);
graphHelper.getOrCreateEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL); graphHelper.getOrCreateEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL);
} }
} }
...@@ -200,7 +201,7 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -200,7 +201,7 @@ public class GraphBackedTypeStore implements ITypeStore {
for (IDataType attrType : attrDataTypes) { for (IDataType attrType : attrDataTypes) {
if (!coreTypes.contains(attrType.getName())) { if (!coreTypes.contains(attrType.getName())) {
Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription()); Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription(), attrType.getVersion());
String label = getEdgeLabel(vertexTypeName, attribute.name); String label = getEdgeLabel(vertexTypeName, attribute.name);
graphHelper.getOrCreateEdge(vertex, attrVertex, label); graphHelper.getOrCreateEdge(vertex, attrVertex, label);
} }
...@@ -238,6 +239,7 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -238,6 +239,7 @@ public class GraphBackedTypeStore implements ITypeStore {
DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY); DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY); String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY); String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
String typeVersion = vertex.getProperty(Constants.TYPEVERSION_PROPERTY_KEY);
LOG.info("Restoring type {}.{}.{}", typeCategory, typeName, typeDescription); LOG.info("Restoring type {}.{}.{}", typeCategory, typeName, typeDescription);
switch (typeCategory) { switch (typeCategory) {
case ENUM: case ENUM:
...@@ -246,19 +248,19 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -246,19 +248,19 @@ public class GraphBackedTypeStore implements ITypeStore {
case STRUCT: case STRUCT:
AttributeDefinition[] attributes = getAttributes(vertex, typeName); AttributeDefinition[] attributes = getAttributes(vertex, typeName);
structs.add(new StructTypeDefinition(typeName, typeDescription, attributes)); structs.add(new StructTypeDefinition(typeName, typeDescription, typeVersion, attributes));
break; break;
case CLASS: case CLASS:
ImmutableSet<String> superTypes = getSuperTypes(vertex); ImmutableSet<String> superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex, typeName); attributes = getAttributes(vertex, typeName);
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, superTypes, attributes)); classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, typeVersion, superTypes, attributes));
break; break;
case TRAIT: case TRAIT:
superTypes = getSuperTypes(vertex); superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex, typeName); attributes = getAttributes(vertex, typeName);
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, superTypes, attributes)); traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, typeVersion, superTypes, attributes));
break; break;
default: default:
...@@ -271,13 +273,14 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -271,13 +273,14 @@ public class GraphBackedTypeStore implements ITypeStore {
private EnumTypeDefinition getEnumType(Vertex vertex) { private EnumTypeDefinition getEnumType(Vertex vertex) {
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY); String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY); String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
String typeVersion = vertex.getProperty(Constants.TYPEVERSION_PROPERTY_KEY);
List<EnumValue> enumValues = new ArrayList<>(); List<EnumValue> enumValues = new ArrayList<>();
List<String> values = graphHelper.getProperty(vertex, getPropertyKey(typeName)); List<String> values = graphHelper.getProperty(vertex, getPropertyKey(typeName));
for (String value : values) { for (String value : values) {
String valueProperty = getPropertyKey(typeName, value); String valueProperty = getPropertyKey(typeName, value);
enumValues.add(new EnumValue(value, (Integer) graphHelper.getProperty(vertex, valueProperty))); enumValues.add(new EnumValue(value, (Integer) graphHelper.getProperty(vertex, valueProperty)));
} }
return new EnumTypeDefinition(typeName, typeDescription, enumValues.toArray(new EnumValue[enumValues.size()])); return new EnumTypeDefinition(typeName, typeDescription, typeVersion, enumValues.toArray(new EnumValue[enumValues.size()]));
} }
private ImmutableSet<String> getSuperTypes(Vertex vertex) { private ImmutableSet<String> getSuperTypes(Vertex vertex) {
...@@ -324,7 +327,7 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -324,7 +327,7 @@ public class GraphBackedTypeStore implements ITypeStore {
return vertex; return vertex;
} }
private Vertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription) { private Vertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription, String typeVersion) {
Vertex vertex = findVertex(category, typeName); Vertex vertex = findVertex(category, typeName);
if (vertex == null) { if (vertex == null) {
LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName); LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
...@@ -341,6 +344,16 @@ public class GraphBackedTypeStore implements ITypeStore { ...@@ -341,6 +344,16 @@ public class GraphBackedTypeStore implements ITypeStore {
} else { } else {
LOG.debug(" type description is null "); LOG.debug(" type description is null ");
} }
if (typeVersion != null) {
String oldVersion = getPropertyKey(Constants.TYPEVERSION_PROPERTY_KEY);
if (!typeVersion.equals(oldVersion)) {
setProperty(vertex, Constants.TYPEVERSION_PROPERTY_KEY, typeVersion);
LOG.info(" updating type {} to version {}", typeName, typeVersion);
}
} else {
LOG.info(" type version is null ");
}
return vertex; return vertex;
} }
} }
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonSyntaxException;
import org.apache.atlas.services.AtlasTypePatch.PatchContent;
import org.apache.atlas.services.AtlasTypePatch.PatchData;
import org.apache.atlas.services.AtlasTypePatch.PatchResult;
import org.apache.atlas.services.AtlasTypePatch.PatchStatus;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUpdateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class AtlasPatchHandler {
private static final Logger LOG = LoggerFactory.getLogger(AtlasPatchHandler.class);
public static void handlePatches(DefaultMetadataService metadataService, TypeSystem typeSystem) throws TypeUpdateException {
Map<String, AtlasTypePatch> patchHandlerMap = initializePatchHandlerMap(metadataService, typeSystem);
if (patchHandlerMap == null || patchHandlerMap.isEmpty())
return;
String patchDirName = ReservedTypesRegistrar.getTypesDir() + File.separator + "patches";
LOG.info("Checking for any patches to be applied to the type system in " + patchDirName);
File patchDir = new File(patchDirName);
if (!patchDir.exists()) {
LOG.info("Patch directory {} doesn't exist, not applying any patches", patchDirName);
return;
}
File[] patchFiles = patchDir.listFiles();
if (patchFiles == null || patchFiles.length == 0) {
LOG.info("No patch files found in {}, not applying any patches", patchDirName);
return;
}
// Sort the patch files based on file name.
Arrays.sort(patchFiles, new Comparator<File>() {
public int compare(File f1, File f2) {
return String.valueOf(f1.getName()).compareTo(f2.getName());
}
});
LOG.info("Found " + patchFiles.length + " patch files to process.");
int patchNumber = 0;
Gson gson = initializeGson();
AtlasTypePatch typePatch;
for (File patchFile : patchFiles) {
try {
LOG.info("Processing patch file " + (++patchNumber) + " - " + patchFile.getAbsolutePath());
String content = new String(Files.readAllBytes(patchFile.toPath()), StandardCharsets.UTF_8);
PatchContent patchContent = gson.fromJson(content, PatchContent.class);
PatchData[] patchDatas = patchContent.getPatches();
PatchResult result;
int patchCounter = 0;
for (PatchData patch : patchDatas) {
typePatch = patchHandlerMap.get(patch.getAction());
if (typePatch != null) {
result = typePatch.applyPatch(patch);
if (result != null) {
LOG.info(result.getMessage() + " Patch " + (++patchCounter) + " out of " + patchDatas.length + " processed in : " + patchFile.toPath());
if (result.getStatus().equals(PatchStatus.FAILED)) {
throw new TypeUpdateException(result.getMessage() + " patch " + patchNumber + " failed in :" + patchFile.getAbsolutePath());
}
}
}
}
} catch (IOException e) {
throw new TypeUpdateException("Unable to read patch file from " + patchFile.getAbsolutePath());
} catch (JsonSyntaxException e) {
throw new TypeUpdateException("Invalid non-parseable JSON patch file in " + patchFile.getAbsolutePath());
}
}
LOG.info("Processed " + patchFiles.length + " patch files.");
}
private static Map<String, AtlasTypePatch> initializePatchHandlerMap(DefaultMetadataService metadataService, TypeSystem typeSystem) {
Map<String, AtlasTypePatch> patchHandlerMap = new HashMap<String, AtlasTypePatch>();
List<AtlasTypePatch> patchers = new ArrayList<AtlasTypePatch>();
// Register new patch classes here
patchers.add(new AtlasTypeAttributePatch(metadataService, typeSystem));
for (AtlasTypePatch patcher : patchers) {
for (String action : patcher.getSupportedActions()) {
patchHandlerMap.put(action, patcher);
}
}
return patchHandlerMap;
}
public static Gson initializeGson() {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.registerTypeAdapter(Multiplicity.class, new MultiplicityDeserializer());
gsonBuilder.setFieldNamingPolicy(FieldNamingPolicy.IDENTITY);
Gson gson = gsonBuilder.create();
return gson;
}
static class MultiplicityDeserializer implements JsonDeserializer<Multiplicity> {
@Override
public Multiplicity deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
String multiplicityString = json.getAsString().toLowerCase();
Multiplicity m = null;
switch (multiplicityString) {
case "optional":
m = Multiplicity.OPTIONAL;
break;
case "required":
m = Multiplicity.REQUIRED;
break;
case "collection":
m = Multiplicity.COLLECTION;
break;
case "set":
m = Multiplicity.SET;
break;
default:
break;
}
return m;
}
}
}
\ No newline at end of file
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.TypeSystem;
import java.util.Map;
public abstract class AtlasTypePatch {
protected final TypeSystem typeSystem;
protected final DefaultMetadataService metadataService;
protected final String[] supportedActions;
protected AtlasTypePatch(DefaultMetadataService metadataService, TypeSystem typeSystem, String[] supportedActions) {
this.metadataService = metadataService;
this.typeSystem = typeSystem;
this.supportedActions = supportedActions;
}
public final String[] getSupportedActions() { return supportedActions; }
public abstract PatchResult applyPatch(PatchData patch);
public enum PatchStatus { SUCCESS, FAILED, SKIPPED }
public class PatchResult {
private String message;
private PatchStatus status;
public PatchResult(String message, PatchStatus status) {
this.message = message;
this.status = status;
}
public String getMessage() { return message; }
public void setMessage(String message) { this.message = message; }
public PatchStatus getStatus() { return status; }
public void setStatus(PatchStatus status) { this.status = status; }
}
/**
* A class to capture patch content.
*/
public class PatchContent {
private PatchData[] patches;
public PatchData[] getPatches() {
return patches;
}
}
public static class PatchData {
private String action;
private String typeName;
private String applyToVersion;
private String updateToVersion;
private Map<String, String> params;
private AttributeDefinition[] attributeDefinitions;
public PatchData(String action, String typeName, String applyToVersion, String updateToVersion, Map<String, String> params, AttributeDefinition[] attributeDefinitions) {
this.action = action;
this.typeName = typeName;
this.applyToVersion = applyToVersion;
this.updateToVersion = updateToVersion;
this.params = params;
this.attributeDefinitions = attributeDefinitions;
}
public String getAction() { return action; }
public String getTypeName() { return typeName; }
public String getApplyToVersion() { return applyToVersion; }
public String getUpdateToVersion() { return updateToVersion; }
public Map<String, String> getParams() { return params; }
public AttributeDefinition[] getAttributeDefinitions() { return attributeDefinitions; }
}
}
...@@ -153,6 +153,8 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang ...@@ -153,6 +153,8 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
restoreTypeSystem(); restoreTypeSystem();
} }
AtlasPatchHandler.handlePatches(this, typeSystem);
maxAuditResults = configuration.getShort(CONFIG_MAX_AUDIT_RESULTS, DEFAULT_MAX_AUDIT_RESULTS); maxAuditResults = configuration.getShort(CONFIG_MAX_AUDIT_RESULTS, DEFAULT_MAX_AUDIT_RESULTS);
} }
......
...@@ -36,6 +36,10 @@ import org.apache.atlas.repository.audit.EntityAuditRepository; ...@@ -36,6 +36,10 @@ import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.audit.HBaseTestUtils; import org.apache.atlas.repository.audit.HBaseTestUtils;
import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.AtlasTypeAttributePatch;
import org.apache.atlas.services.AtlasTypePatch;
import org.apache.atlas.services.AtlasTypePatch.PatchData;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.IStruct;
...@@ -111,7 +115,6 @@ public class DefaultMetadataServiceTest { ...@@ -111,7 +115,6 @@ public class DefaultMetadataServiceTest {
private Referenceable table; private Referenceable table;
private Id tableId; private Id tableId;
private final String NAME = "name"; private final String NAME = "name";
...@@ -1092,6 +1095,43 @@ public class DefaultMetadataServiceTest { ...@@ -1092,6 +1095,43 @@ public class DefaultMetadataServiceTest {
} }
@Test @Test
public void testPatchFrameworkForTypeUpdate() throws AtlasException, JSONException {
String typeName = "test_type_" + RandomStringUtils.randomAlphanumeric(10);
HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef(typeName, ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("type_attr1", DataTypes.STRING_TYPE));
TypesDef typesDef = new TypesDef(typeDef, false);
metadataService.createType(TypesSerialization.toJson(typesDef));
AtlasTypeAttributePatch patch = new AtlasTypeAttributePatch((DefaultMetadataService) metadataService, TypeSystem.getInstance());
AttributeDefinition[] attrDefs = new AttributeDefinition[]{
new AttributeDefinition("type_attr2", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("type_attr3", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)};
// Testing add attribute patch
AtlasTypePatch.PatchData addAttributePatch = new PatchData("ADD_ATTRIBUTE", typeName, "1.0", "2.0", null, attrDefs);
TypesDef newAttrTypesDef = patch.updateTypesDef(typesDef, addAttributePatch);
metadataService.updateType(TypesSerialization.toJson(newAttrTypesDef));
TypesDef addedTypesDef = TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
// test added attributes and update version to 2.0
assertEquals(addedTypesDef.classTypes().head().attributeDefinitions.length, 3);
assertEquals(addedTypesDef.classTypes().head().typeVersion, "2.0");
// Testing update attribute patch
AttributeDefinition[] updateAttrDef = new AttributeDefinition[]{
new AttributeDefinition("type_attr1", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)};
AtlasTypePatch.PatchData updateAttributePatch = new PatchData("UPDATE_ATTRIBUTE", typeName, "2.0", "3.0", null, updateAttrDef);
TypesDef updateAttrTypesDef = patch.updateTypesDef(addedTypesDef, updateAttributePatch);
metadataService.updateType(TypesSerialization.toJson(updateAttrTypesDef));
TypesDef updatedTypesDef = TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
// test update attribute to optional and update version to 3.0
assertEquals(updatedTypesDef.classTypes().head().attributeDefinitions[0].multiplicity, Multiplicity.OPTIONAL);
assertEquals(updatedTypesDef.classTypes().head().typeVersion, "3.0");
}
@Test
public void testAuditEventsInvalidParams() throws Exception { public void testAuditEventsInvalidParams() throws Exception {
//entity id can't be null //entity id can't be null
try { try {
...@@ -1178,7 +1218,7 @@ public class DefaultMetadataServiceTest { ...@@ -1178,7 +1218,7 @@ public class DefaultMetadataServiceTest {
deletedEntities.add(entity.getId()._getId()); deletedEntities.add(entity.getId()._getId());
} }
} }
public List<String> getDeletedEntities() { public List<String> getDeletedEntities() {
return deletedEntities; return deletedEntities;
} }
......
...@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types; ...@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedMap;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import java.io.IOException; import java.io.IOException;
...@@ -30,12 +31,22 @@ abstract class AbstractDataType<T> implements IDataType<T> { ...@@ -30,12 +31,22 @@ abstract class AbstractDataType<T> implements IDataType<T> {
public final String name; public final String name;
public final String description; public final String description;
public final String version;
public AbstractDataType(String name, String description) { public AbstractDataType(String name, String description) {
super(); super();
this.name = name; this.name = name;
this.description = description; this.description = description;
this.version = AtlasConstants.DEFAULT_TYPE_VERSION;
}
public AbstractDataType(String name, String description, String version) {
super();
this.name = name;
this.description = description;
this.version = version;
} }
protected T convertNull(Multiplicity m) throws AtlasException { protected T convertNull(Multiplicity m) throws AtlasException {
...@@ -99,5 +110,10 @@ abstract class AbstractDataType<T> implements IDataType<T> { ...@@ -99,5 +110,10 @@ abstract class AbstractDataType<T> implements IDataType<T> {
public String getDescription() { public String getDescription() {
return description; return description;
} }
@Override
public String getVersion() {
return version;
}
} }
...@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; ...@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.IStruct;
...@@ -51,13 +52,22 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc ...@@ -51,13 +52,22 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
public final Map<AttributeInfo, List<String>> infoToNameMap; public final Map<AttributeInfo, List<String>> infoToNameMap;
ClassType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTypes, int numFields) { ClassType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTypes, int numFields) {
super(typeSystem, ClassType.class, name, description, superTypes, numFields); this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, numFields);
}
ClassType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTypes, int numFields) {
super(typeSystem, ClassType.class, name, description, version, superTypes, numFields);
infoToNameMap = null; infoToNameMap = null;
} }
ClassType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTypes, AttributeInfo... fields) ClassType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTypes, AttributeInfo... fields)
throws AtlasException { throws AtlasException {
super(typeSystem, ClassType.class, name, description, superTypes, fields); this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, fields);
}
ClassType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTypes, AttributeInfo... fields)
throws AtlasException {
super(typeSystem, ClassType.class, name, description, version, superTypes, fields);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping); infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
} }
......
...@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types; ...@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import scala.math.BigInt; import scala.math.BigInt;
...@@ -33,11 +34,15 @@ public class EnumType extends AbstractDataType<EnumValue> { ...@@ -33,11 +34,15 @@ public class EnumType extends AbstractDataType<EnumValue> {
public final ImmutableMap<Integer, EnumValue> ordinalMap; public final ImmutableMap<Integer, EnumValue> ordinalMap;
protected EnumType(TypeSystem typeSystem, String name, EnumValue... values) { protected EnumType(TypeSystem typeSystem, String name, EnumValue... values) {
this(typeSystem, name, null, values); this(typeSystem, name, null, values);
} }
protected EnumType(TypeSystem typeSystem, String name, String description, EnumValue... values) { protected EnumType(TypeSystem typeSystem, String name, String description, EnumValue... values) {
super(name, description); this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, values);
}
protected EnumType(TypeSystem typeSystem, String name, String description, String version, EnumValue... values) {
super(name, description, version);
this.typeSystem = typeSystem; this.typeSystem = typeSystem;
ImmutableMap.Builder<String, EnumValue> b1 = new ImmutableMap.Builder(); ImmutableMap.Builder<String, EnumValue> b1 = new ImmutableMap.Builder();
ImmutableMap.Builder<Integer, EnumValue> b2 = new ImmutableMap.Builder(); ImmutableMap.Builder<Integer, EnumValue> b2 = new ImmutableMap.Builder();
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
package org.apache.atlas.typesystem.types; package org.apache.atlas.typesystem.types;
import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.AtlasConstants;
import java.util.Arrays; import java.util.Arrays;
...@@ -26,16 +27,22 @@ public final class EnumTypeDefinition { ...@@ -26,16 +27,22 @@ public final class EnumTypeDefinition {
public final String name; public final String name;
public final String description; public final String description;
public final String version;
public final EnumValue[] enumValues; public final EnumValue[] enumValues;
public EnumTypeDefinition(String name, EnumValue... enumValues) { public EnumTypeDefinition(String name, EnumValue... enumValues) {
this(name, null, enumValues); this(name, null, AtlasConstants.DEFAULT_TYPE_VERSION, enumValues);
} }
public EnumTypeDefinition(String name, String description, EnumValue... enumValues) { public EnumTypeDefinition(String name, String description, EnumValue... enumValues) {
this(name, description, AtlasConstants.DEFAULT_TYPE_VERSION, enumValues);
}
public EnumTypeDefinition(String name, String description, String version, EnumValue... enumValues) {
this.name = ParamChecker.notEmpty(name, "Enum type name"); this.name = ParamChecker.notEmpty(name, "Enum type name");
this.description = description; this.description = description;
this.enumValues = ParamChecker.notNullElements(enumValues, "Enum values"); this.enumValues = ParamChecker.notNullElements(enumValues, "Enum values");
this.version = version;
} }
@Override @Override
......
...@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; ...@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.google.common.collect.UnmodifiableIterator; import com.google.common.collect.UnmodifiableIterator;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.persistence.DownCastStructInstance; import org.apache.atlas.typesystem.persistence.DownCastStructInstance;
...@@ -70,7 +71,12 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A ...@@ -70,7 +71,12 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
*/ */
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableSet<String> superTypes, HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableSet<String> superTypes,
int numFields) { int numFields) {
super(name, description); this( typeSystem, superTypeClass, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, numFields);
}
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, String version, ImmutableSet<String> superTypes,
int numFields) {
super(name, description, version);
this.typeSystem = typeSystem; this.typeSystem = typeSystem;
this.superTypeClass = superTypeClass; this.superTypeClass = superTypeClass;
this.fieldMapping = null; this.fieldMapping = null;
...@@ -86,7 +92,12 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A ...@@ -86,7 +92,12 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
} }
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableSet<String> superTypes, HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableSet<String> superTypes,
AttributeInfo... fields) throws AtlasException { AttributeInfo... fields) throws AtlasException {
super(name, description); this(typeSystem, superTypeClass, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, fields);
}
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, String version, ImmutableSet<String> superTypes,
AttributeInfo... fields) throws AtlasException {
super(name, description, version);
this.typeSystem = typeSystem; this.typeSystem = typeSystem;
this.superTypeClass = superTypeClass; this.superTypeClass = superTypeClass;
Pair<FieldMapping, ImmutableMap<String, String>> p = constructFieldMapping(superTypes, fields); Pair<FieldMapping, ImmutableMap<String, String>> p = constructFieldMapping(superTypes, fields);
......
...@@ -21,34 +21,41 @@ package org.apache.atlas.typesystem.types; ...@@ -21,34 +21,41 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.utils.ParamChecker;
public class HierarchicalTypeDefinition<T extends HierarchicalType> extends StructTypeDefinition { public class HierarchicalTypeDefinition<T extends HierarchicalType> extends StructTypeDefinition {
public final ImmutableSet<String> superTypes; public final ImmutableSet<String> superTypes;
public final String hierarchicalMetaTypeName; public final String hierarchicalMetaTypeName;
/** public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, ImmutableSet<String> superTypes,
* Used for json deserialization only. AttributeDefinition[] attributeDefinitions) {
* not intended public consumption this(hierarchicalMetaType, typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes,
* @param hierarchicalMetaTypeName
* @param typeName
* @param typeDescription
* @param superTypes
* @param attributeDefinitions
* @throws ClassNotFoundException
*/
@InterfaceAudience.Private
public HierarchicalTypeDefinition(String hierarchicalMetaTypeName, String typeName, String typeDescription, String[] superTypes,
AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, ImmutableSet.copyOf(superTypes),
attributeDefinitions); attributeDefinitions);
} }
public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, ImmutableSet<String> superTypes, // Used only for de-serializing JSON String to typedef.
AttributeDefinition[] attributeDefinitions) { public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String typeVersion, String[] superTypes, AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException {
super(typeName, typeDescription, false, attributeDefinitions); this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, typeVersion, ImmutableSet.copyOf(superTypes), attributeDefinitions);
hierarchicalMetaTypeName = hierarchicalMetaType.getName(); }
// Used only for de-serializing JSON String to typedef (no typeVersion).
public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String[] superTypes, AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, ImmutableSet.copyOf(superTypes), attributeDefinitions);
}
// Used only for serializing typedef to JSON String.
public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, String typeVersion, ImmutableSet<String> superTypes, AttributeDefinition[] attributeDefinitions, String typeDef) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, typeVersion, superTypes, attributeDefinitions);
}
// Used only for serializing typedef to JSON String (no typeVersion).
public HierarchicalTypeDefinition( String hierarchicalMetaTypeName, String typeName, String typeDescription, ImmutableSet<String> superTypes, AttributeDefinition[] attributeDefinitions, String typeDef) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attributeDefinitions);
}
public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, String typeVersion, ImmutableSet<String> superTypes, AttributeDefinition[] attributeDefinitions) {
super(typeName, typeDescription, typeVersion, false, attributeDefinitions);
this.hierarchicalMetaTypeName = hierarchicalMetaType.getName();
this.superTypes = superTypes == null ? ImmutableSet.<String>of() : superTypes; this.superTypes = superTypes == null ? ImmutableSet.<String>of() : superTypes;
} }
......
...@@ -55,5 +55,7 @@ public interface IDataType<T> { ...@@ -55,5 +55,7 @@ public interface IDataType<T> {
void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException; void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException;
String getDescription(); String getDescription();
String getVersion();
} }
...@@ -29,6 +29,7 @@ import java.util.List; ...@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.ITypedStruct;
...@@ -42,7 +43,11 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa ...@@ -42,7 +43,11 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
private final TypedStructHandler handler; private final TypedStructHandler handler;
protected StructType(TypeSystem typeSystem, String name, String description, int numFields) { protected StructType(TypeSystem typeSystem, String name, String description, int numFields) {
super(name, description); this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, numFields);
}
protected StructType(TypeSystem typeSystem, String name, String description, String version, int numFields) {
super(name, description, version);
this.typeSystem = typeSystem; this.typeSystem = typeSystem;
this.fieldMapping = null; this.fieldMapping = null;
infoToNameMap = null; infoToNameMap = null;
...@@ -52,7 +57,12 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa ...@@ -52,7 +57,12 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
protected StructType(TypeSystem typeSystem, String name, String description, AttributeInfo... fields) protected StructType(TypeSystem typeSystem, String name, String description, AttributeInfo... fields)
throws AtlasException { throws AtlasException {
super(name, description); this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, fields);
}
protected StructType(TypeSystem typeSystem, String name, String description, String version, AttributeInfo... fields)
throws AtlasException {
super(name, description, version);
this.typeSystem = typeSystem; this.typeSystem = typeSystem;
this.fieldMapping = constructFieldMapping(fields); this.fieldMapping = constructFieldMapping(fields);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping); infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping);
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.atlas.typesystem.types; package org.apache.atlas.typesystem.types;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.utils.ParamChecker;
import java.util.Arrays; import java.util.Arrays;
...@@ -26,29 +27,45 @@ public class StructTypeDefinition { ...@@ -26,29 +27,45 @@ public class StructTypeDefinition {
public final String typeName; public final String typeName;
public final String typeDescription;//optional field public final String typeDescription;//optional field
public final String typeVersion;
public final AttributeDefinition[] attributeDefinitions; public final AttributeDefinition[] attributeDefinitions;
protected StructTypeDefinition(String typeName, String typeDescription, boolean validate, protected StructTypeDefinition(String typeName, String typeDescription, boolean validate,
AttributeDefinition... attributeDefinitions) { AttributeDefinition... attributeDefinitions) {
this(typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, validate, attributeDefinitions);
}
protected StructTypeDefinition(String typeName, String typeDescription, String typeVersion, boolean validate,
AttributeDefinition... attributeDefinitions) {
this.typeName = ParamChecker.notEmpty(typeName, "Struct type name"); this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
this.typeDescription = typeDescription; this.typeDescription = typeDescription;
if (validate) { if (validate) {
ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions"); ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
} }
this.attributeDefinitions = attributeDefinitions; this.attributeDefinitions = attributeDefinitions;
this.typeVersion = typeVersion;
} }
public StructTypeDefinition(String typeName, AttributeDefinition[] attributeDefinitions) { public StructTypeDefinition(String typeName, AttributeDefinition[] attributeDefinitions) {
this(typeName, null, attributeDefinitions); this(typeName, null, AtlasConstants.DEFAULT_TYPE_VERSION, attributeDefinitions);
} }
public StructTypeDefinition(String typeName, String typeDescription, public StructTypeDefinition(String typeName, String typeDescription,
AttributeDefinition[] attributeDefinitions) { AttributeDefinition[] attributeDefinitions) {
this(typeName, typeDescription, AtlasConstants.DEFAULT_TYPE_VERSION, attributeDefinitions);
}
public StructTypeDefinition(String typeName, String typeDescription, String typeVersion,
AttributeDefinition[] attributeDefinitions) {
this.typeName = ParamChecker.notEmpty(typeName, "Struct type name"); this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
this.typeDescription = typeDescription; this.typeDescription = typeDescription;
this.typeVersion = typeVersion;
this.attributeDefinitions = ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions"); this.attributeDefinitions = ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
} }
@Override @Override
public boolean equals(Object o) { public boolean equals(Object o) {
if (this == o) { if (this == o) {
......
...@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types; ...@@ -20,6 +20,7 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.ITypedStruct;
...@@ -37,14 +38,23 @@ public class TraitType extends HierarchicalType<TraitType, IStruct> ...@@ -37,14 +38,23 @@ public class TraitType extends HierarchicalType<TraitType, IStruct>
private final TypedStructHandler handler; private final TypedStructHandler handler;
TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, int numFields) { TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, int numFields) {
super(typeSystem, TraitType.class, name, description, superTraits, numFields); this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTraits, numFields);
}
TraitType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTraits, int numFields) {
super(typeSystem, TraitType.class, name, description, version, superTraits, numFields);
handler = null; handler = null;
infoToNameMap = null; infoToNameMap = null;
} }
TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, AttributeInfo... fields) TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, AttributeInfo... fields)
throws AtlasException { throws AtlasException {
super(typeSystem, TraitType.class, name, description, superTraits, fields); this(typeSystem, name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTraits, fields);
}
TraitType(TypeSystem typeSystem, String name, String description, String version, ImmutableSet<String> superTraits, AttributeInfo... fields)
throws AtlasException {
super(typeSystem, TraitType.class, name, description, version, superTraits, fields);
handler = new TypedStructHandler(this); handler = new TypedStructHandler(this);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping); infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
} }
......
...@@ -311,7 +311,7 @@ public class TypeSystem { ...@@ -311,7 +311,7 @@ public class TypeSystem {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name)); throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
} }
EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.enumValues); EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.version, eDef.enumValues);
typeCache.put(eT); typeCache.put(eT);
return eT; return eT;
} }
...@@ -408,7 +408,7 @@ public class TypeSystem { ...@@ -408,7 +408,7 @@ public class TypeSystem {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name)); throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
} }
EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.enumValues); EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.version, eDef.enumValues);
transientTypes.put(eDef.name, eT); transientTypes.put(eDef.name, eT);
} }
...@@ -417,7 +417,7 @@ public class TypeSystem { ...@@ -417,7 +417,7 @@ public class TypeSystem {
if (!update && isRegistered(sDef.typeName)) { if (!update && isRegistered(sDef.typeName)) {
throw new TypeExistsException(String.format("Cannot redefine type %s", sDef.typeName)); throw new TypeExistsException(String.format("Cannot redefine type %s", sDef.typeName));
} }
StructType sT = new StructType(this, sDef.typeName, sDef.typeDescription, sDef.attributeDefinitions.length); StructType sT = new StructType(this, sDef.typeName, sDef.typeDescription, sDef.typeVersion, sDef.attributeDefinitions.length);
structNameToDefMap.put(sDef.typeName, sDef); structNameToDefMap.put(sDef.typeName, sDef);
transientTypes.put(sDef.typeName, sT); transientTypes.put(sDef.typeName, sT);
} }
...@@ -427,7 +427,7 @@ public class TypeSystem { ...@@ -427,7 +427,7 @@ public class TypeSystem {
if (!update && isRegistered(traitDef.typeName)) { if (!update && isRegistered(traitDef.typeName)) {
throw new TypeExistsException(String.format("Cannot redefine type %s", traitDef.typeName)); throw new TypeExistsException(String.format("Cannot redefine type %s", traitDef.typeName));
} }
TraitType tT = new TraitType(this, traitDef.typeName, traitDef.typeDescription, traitDef.superTypes, TraitType tT = new TraitType(this, traitDef.typeName, traitDef.typeDescription, traitDef.typeVersion, traitDef.superTypes,
traitDef.attributeDefinitions.length); traitDef.attributeDefinitions.length);
traitNameToDefMap.put(traitDef.typeName, traitDef); traitNameToDefMap.put(traitDef.typeName, traitDef);
transientTypes.put(traitDef.typeName, tT); transientTypes.put(traitDef.typeName, tT);
...@@ -439,7 +439,7 @@ public class TypeSystem { ...@@ -439,7 +439,7 @@ public class TypeSystem {
throw new TypeExistsException(String.format("Cannot redefine type %s", classDef.typeName)); throw new TypeExistsException(String.format("Cannot redefine type %s", classDef.typeName));
} }
ClassType cT = new ClassType(this, classDef.typeName, classDef.typeDescription, classDef.superTypes, ClassType cT = new ClassType(this, classDef.typeName, classDef.typeDescription, classDef.typeVersion, classDef.superTypes,
classDef.attributeDefinitions.length); classDef.attributeDefinitions.length);
classNameToDefMap.put(classDef.typeName, classDef); classNameToDefMap.put(classDef.typeName, classDef);
transientTypes.put(classDef.typeName, cT); transientTypes.put(classDef.typeName, cT);
...@@ -526,7 +526,7 @@ public class TypeSystem { ...@@ -526,7 +526,7 @@ public class TypeSystem {
infos[i] = constructAttributeInfo(def.attributeDefinitions[i]); infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
} }
StructType type = new StructType(this, def.typeName, def.typeDescription, infos); StructType type = new StructType(this, def.typeName, def.typeDescription, def.typeVersion, infos);
transientTypes.put(def.typeName, type); transientTypes.put(def.typeName, type);
return type; return type;
} }
...@@ -539,9 +539,9 @@ public class TypeSystem { ...@@ -539,9 +539,9 @@ public class TypeSystem {
} }
try { try {
Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, ImmutableSet.class, Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, String.class, ImmutableSet.class,
AttributeInfo[].class); AttributeInfo[].class);
U type = cons.newInstance(this, def.typeName, def.typeDescription, def.superTypes, infos); U type = cons.newInstance(this, def.typeName, def.typeDescription, def.typeVersion, def.superTypes, infos);
transientTypes.put(def.typeName, type); transientTypes.put(def.typeName, type);
return type; return type;
} catch (Exception e) { } catch (Exception e) {
......
...@@ -33,6 +33,7 @@ import org.apache.atlas.typesystem.types.IDataType; ...@@ -33,6 +33,7 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import scala.collection.JavaConversions; import scala.collection.JavaConversions;
...@@ -76,6 +77,11 @@ public class TypesUtil { ...@@ -76,6 +77,11 @@ public class TypesUtil {
public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description, public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) { ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
return createTraitTypeDef(name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attrDefs);
}
public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description, String version,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(TraitType.class, name, description, superTypes, attrDefs); return new HierarchicalTypeDefinition<>(TraitType.class, name, description, superTypes, attrDefs);
} }
...@@ -94,6 +100,11 @@ public class TypesUtil { ...@@ -94,6 +100,11 @@ public class TypesUtil {
public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, String description, public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, String description,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) { ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
return createClassTypeDef(name, description, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attrDefs);
}
public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, String description, String version,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(ClassType.class, name, description, superTypes, attrDefs); return new HierarchicalTypeDefinition<>(ClassType.class, name, description, superTypes, attrDefs);
} }
......
...@@ -21,6 +21,7 @@ package org.apache.atlas.typesystem.json ...@@ -21,6 +21,7 @@ package org.apache.atlas.typesystem.json
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import com.google.common.collect.ImmutableList import com.google.common.collect.ImmutableList
import org.apache.atlas.AtlasException import org.apache.atlas.AtlasException
import org.apache.atlas.AtlasConstants
import org.apache.atlas.typesystem.TypesDef import org.apache.atlas.typesystem.TypesDef
import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, MapType, TypeCategory} import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, MapType, TypeCategory}
import org.apache.atlas.typesystem.types._ import org.apache.atlas.typesystem.types._
...@@ -131,28 +132,28 @@ object TypesSerialization { ...@@ -131,28 +132,28 @@ object TypesSerialization {
private def convertEnumTypeToEnumTypeDef(et: EnumType) = { private def convertEnumTypeToEnumTypeDef(et: EnumType) = {
val eVals: Seq[EnumValue] = et.valueMap.values().toSeq val eVals: Seq[EnumValue] = et.valueMap.values().toSeq
new EnumTypeDefinition(et.name, et.description, eVals: _*) new EnumTypeDefinition(et.name, et.description, et.version, eVals: _*)
} }
private def convertStructTypeToStructDef(st: StructType): StructTypeDefinition = { private def convertStructTypeToStructDef(st: StructType): StructTypeDefinition = {
val aDefs: Iterable[AttributeDefinition] = val aDefs: Iterable[AttributeDefinition] =
st.fieldMapping.fields.values().map(convertAttributeInfoToAttributeDef(_)) st.fieldMapping.fields.values().map(convertAttributeInfoToAttributeDef(_))
new StructTypeDefinition(st.name, st.description, aDefs.toArray) new StructTypeDefinition(st.name, st.description, st.version, aDefs.toArray)
} }
private def convertTraitTypeToHierarchicalTypeDefinition(tt: TraitType): HierarchicalTypeDefinition[TraitType] = { private def convertTraitTypeToHierarchicalTypeDefinition(tt: TraitType): HierarchicalTypeDefinition[TraitType] = {
val aDefs: Iterable[AttributeDefinition] = val aDefs: Iterable[AttributeDefinition] =
tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_)) tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.description, tt.superTypes, aDefs.toArray) new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.description, tt.version, tt.superTypes, aDefs.toArray)
} }
private def convertClassTypeToHierarchicalTypeDefinition(tt: ClassType): HierarchicalTypeDefinition[ClassType] = { private def convertClassTypeToHierarchicalTypeDefinition(tt: ClassType): HierarchicalTypeDefinition[ClassType] = {
val aDefs: Iterable[AttributeDefinition] = val aDefs: Iterable[AttributeDefinition] =
tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_)) tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.description, tt.superTypes, aDefs.toArray) new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.description, tt.version, tt.superTypes, aDefs.toArray)
} }
def convertToTypesDef(ts: TypeSystem, export: IDataType[_] => Boolean): TypesDef = { def convertToTypesDef(ts: TypeSystem, export: IDataType[_] => Boolean): TypesDef = {
...@@ -229,26 +230,36 @@ trait TypeHelpers { ...@@ -229,26 +230,36 @@ trait TypeHelpers {
HierarchicalTypeDefinition[TraitType] = { HierarchicalTypeDefinition[TraitType] = {
createTraitTypeDef(name, None, superTypes, attrDefs:_*) createTraitTypeDef(name, None, superTypes, attrDefs:_*)
} }
def createTraitTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*): def createTraitTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[TraitType] = { HierarchicalTypeDefinition[TraitType] = {
val sts = ImmutableSet.copyOf(superTypes.toArray) createTraitTypeDef(name, None, AtlasConstants.DEFAULT_TYPE_VERSION, superTypes, attrDefs:_*)
return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name, description.getOrElse(null), }
sts, attrDefs.toArray)
def createTraitTypeDef(name: String, description: Option[String], version: String,superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[TraitType] = {
val sts = ImmutableSet.copyOf(superTypes.toArray)
return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name, description.getOrElse(null),
sts, attrDefs.toArray)
} }
def createClassTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*): def createClassTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[ClassType] = { HierarchicalTypeDefinition[ClassType] = {
createClassTypeDef( name, None, superTypes, attrDefs:_*) createClassTypeDef( name, None, superTypes, attrDefs:_*)
} }
def createClassTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*): def createClassTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[ClassType] = { HierarchicalTypeDefinition[ClassType] = {
val sts = ImmutableSet.copyOf(superTypes.toArray) createClassTypeDef( name, None, None, superTypes, attrDefs:_*)
return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name, description.getOrElse(null),
sts, attrDefs.toArray)
} }
def createClassTypeDef(name: String, description: Option[String], version: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[ClassType] = {
val sts = ImmutableSet.copyOf(superTypes.toArray)
return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name, description.getOrElse(null), AtlasConstants.DEFAULT_TYPE_VERSION, sts, attrDefs.toArray)
}
@throws(classOf[AtlasException]) @throws(classOf[AtlasException])
def defineClassType(ts: TypeSystem, classDef: HierarchicalTypeDefinition[ClassType]): ClassType = { def defineClassType(ts: TypeSystem, classDef: HierarchicalTypeDefinition[ClassType]): ClassType = {
ts.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.of[StructTypeDefinition], ts.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.of[StructTypeDefinition],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment