Skip to content

A
atlas
Commit bf5672c5 by Suma Shivaprasad
Options

ATLAS-171 Ability to update type definition(shwethags via sumasai)

parent 919120f6
Showing with 937 additions and 136 deletions
addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
......@@ -34,7 +34,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -102,7 +102,7 @@ public class HiveDataModelGenerator {
}
public TypesDef getTypesDef() {
return TypeUtils.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
getClassTypeDefinitions());
}
......
client/src/main/java/org/apache/atlas/AtlasClient.java
......@@ -25,7 +25,9 @@ import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.atlas.security.SecureClientUtils;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
......@@ -132,6 +134,7 @@ public class AtlasClient {
//Type operations
CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST),
UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT),
GET_TYPE(BASE_URI + TYPES, HttpMethod.GET),
LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET),
LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET),
......@@ -181,13 +184,45 @@ public class AtlasClient {
* @return result json object
* @throws AtlasServiceException
*/
public JSONObject createType(String typeAsJson) throws AtlasServiceException {
return callAPI(API.CREATE_TYPE, typeAsJson);
public List<String> createType(String typeAsJson) throws AtlasServiceException {
JSONObject response = callAPI(API.CREATE_TYPE, typeAsJson);
return extractResults(response, AtlasClient.TYPES);
}
/**
* Register the given type(meta model)
* @param typeDef type definition
* @return result json object
* @throws AtlasServiceException
*/
public List<String> createType(TypesDef typeDef) throws AtlasServiceException {
return createType(TypesSerialization.toJson(typeDef));
}
/**
* Register the given type(meta model)
* @param typeAsJson type definition a jaon
* @return result json object
* @throws AtlasServiceException
*/
public List<String> updateType(String typeAsJson) throws AtlasServiceException {
JSONObject response = callAPI(API.UPDATE_TYPE, typeAsJson);
return extractResults(response, AtlasClient.TYPES);
}
/**
* Register the given type(meta model)
* @param typeDef type definition
* @return result json object
* @throws AtlasServiceException
*/
public List<String> updateType(TypesDef typeDef) throws AtlasServiceException {
return updateType(TypesSerialization.toJson(typeDef));
}
public List<String> listTypes() throws AtlasServiceException {
final JSONObject jsonObject = callAPI(API.LIST_TYPES, null);
return extractResults(jsonObject);
return extractResults(jsonObject, AtlasClient.RESULTS);
}
public String getType(String typeName) throws AtlasServiceException {
......@@ -230,6 +265,14 @@ public class AtlasClient {
return createEntity(new JSONArray(Arrays.asList(entitiesAsJson)));
}
public JSONArray createEntity(Referenceable... entities) throws AtlasServiceException {
JSONArray entityArray = new JSONArray(entities.length);
for (Referenceable entity : entities) {
entityArray.put(InstanceSerialization.toJson(entity, true));
}
return createEntity(entityArray);
}
/**
* Get an entity given the entity id
* @param guid entity id
......@@ -286,15 +329,20 @@ public class AtlasClient {
WebResource resource = getResource(API.LIST_ENTITIES);
resource = resource.queryParam(TYPE, entityType);
JSONObject jsonResponse = callAPIWithResource(API.LIST_ENTITIES, resource);
return extractResults(jsonResponse);
return extractResults(jsonResponse, AtlasClient.RESULTS);
}
private List<String> extractResults(JSONObject jsonResponse) throws AtlasServiceException {
private List<String> extractResults(JSONObject jsonResponse, String key) throws AtlasServiceException {
try {
JSONArray results = jsonResponse.getJSONArray(AtlasClient.RESULTS);
JSONArray results = jsonResponse.getJSONArray(key);
ArrayList<String> resultsList = new ArrayList<>();
for (int index = 0; index < results.length(); index++) {
resultsList.add(results.getString(index));
Object element = results.get(index);
if (element instanceof String) {
resultsList.add((String) element);
} else if (element instanceof JSONObject) {
resultsList.add(((JSONObject) element).getString(AtlasClient.NAME));
}
}
return resultsList;
} catch (JSONException e) {
......
common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
......@@ -44,6 +44,6 @@ public interface TypesChangeListener {
*/
// void onRemove(String typeName) throws MetadataException;
// This is upon updating an existing type to the store
// void onChange() throws MetadataException;
//This is upon updating an existing type to the store
void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException;
}
pom.xml
......@@ -1426,6 +1426,7 @@
<reuseForks>false</reuseForks>
<forkCount>1</forkCount>
<threadCount>5</threadCount>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration>
<executions>
<execution>
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
ATLAS-171 Ability to update type definition(shwethags via sumasai)
ATLAS-352 Improve write performance on type and entity creation with Hbase (sumasai)
ATLAS-350 Document jaas config details for atlas (tbeerbower via shwethags)
ATLAS-344 Document HBase permissions for secure cluster (tbeerbower via shwethags)
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -161,6 +161,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
commit();
}
@Override
public void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException {
onAdd(dataTypes);
}
private void addIndexForType(IDataType dataType) {
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
......
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
......@@ -45,6 +45,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -253,7 +254,7 @@ public class GraphBackedTypeStore implements ITypeStore {
throw new IllegalArgumentException("Unhandled type category " + typeCategory);
}
}
return TypeUtils.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
}
private EnumTypeDefinition getEnumType(Vertex vertex) {
......
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -50,7 +50,6 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONArray;
......@@ -148,8 +147,7 @@ public class DefaultMetadataService implements MetadataService {
private void createType(HierarchicalTypeDefinition<ClassType> type) throws AtlasException {
if (!typeSystem.isRegistered(type.typeName)) {
TypesDef typesDef = TypeUtils
.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(type));
createType(TypesSerialization.toJson(typesDef));
......@@ -191,6 +189,34 @@ public class DefaultMetadataService implements MetadataService {
}
}
@Override
public JSONObject updateType(String typeDefinition) throws AtlasException {
ParamChecker.notEmpty(typeDefinition, "type definition cannot be empty");
TypesDef typesDef = validateTypeDefinition(typeDefinition);
try {
final Map<String, IDataType> typesAdded = typeSystem.updateTypes(typesDef);
try {
/* Create indexes first so that if index creation fails then we rollback
the typesystem and also do not persist the graph
*/
onTypesUpdated(typesAdded);
typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
} catch (Throwable t) {
typeSystem.removeTypes(typesAdded.keySet());
throw new AtlasException("Unable to persist types ", t);
}
return new JSONObject() {{
put(AtlasClient.TYPES, typesAdded.keySet());
}};
} catch (JSONException e) {
LOG.error("Unable to create response for types={}", typeDefinition, e);
throw new AtlasException("Unable to create response ", e);
}
}
private TypesDef validateTypeDefinition(String typeDefinition) {
try {
TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
......@@ -343,7 +369,7 @@ public class DefaultMetadataService implements MetadataService {
repository.updateEntity(guid, property, value);
onEntityUpdated(repository.getEntityDefinition(guid), property, value);
onEntityUpdated(repository.getEntityDefinition(guid));
}
private void validateTypeExists(String entityType) throws AtlasException {
......@@ -466,7 +492,24 @@ public class DefaultMetadataService implements MetadataService {
}
}
private void onEntityUpdated(ITypedReferenceableInstance entity, String property, String value)
private void onTypesUpdated(Map<String, IDataType> typesUpdated) throws AtlasException {
Map<TypesChangeListener, Throwable> caughtExceptions = new HashMap<>();
for (Provider<TypesChangeListener> indexerProvider : typeChangeListeners) {
final TypesChangeListener listener = indexerProvider.get();
try {
listener.onChange(typesUpdated.values());
} catch (IndexCreationException ice) {
LOG.error("Index creation for listener {} failed ", indexerProvider, ice);
caughtExceptions.put(listener, ice);
}
}
if (caughtExceptions.size() > 0) {
throw new IndexCreationException("Index creation failed for types " + typesUpdated.keySet() + ". Aborting");
}
}
private void onEntityUpdated(ITypedReferenceableInstance entity)
throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntityUpdated(entity);
......
repository/src/main/java/org/apache/atlas/services/MetadataService.java
......@@ -40,6 +40,14 @@ public interface MetadataService {
JSONObject createType(String typeDefinition) throws AtlasException;
/**
* Updates the given types in the type definition
* @param typeDefinition
* @return
* @throws AtlasException
*/
JSONObject updateType(String typeDefinition) throws AtlasException;
/**
* Return the definition for the given type.
*
* @param typeName name for this type, must be unique
......
repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
......@@ -28,7 +28,6 @@ import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
......@@ -41,10 +40,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import javax.inject.Inject;
......@@ -170,7 +166,7 @@ public class BaseHiveRepositoryTest {
HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null);
return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef, partClsDef));
}
......
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -38,7 +38,6 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.RandomStringUtils;
import org.testng.Assert;
......@@ -272,7 +271,7 @@ public final class TestUtils {
createTraitTypeDef("fetl" + CLASSIFICATION, ImmutableList.of(CLASSIFICATION),
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
return TypeUtils.getTypesDef(ImmutableList.of(enumTypeDefinition),
return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition),
ImmutableList.of(structTypeDefinition, partitionDefinition),
ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition),
ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
......
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -18,13 +18,13 @@
package org.apache.atlas.repository.typestore;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphHelper;
......@@ -33,21 +33,30 @@ import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import org.testng.Assert;
import javax.inject.Inject;
import java.util.List;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedTypeStoreTest {
@Inject
......@@ -77,7 +86,6 @@ public class GraphBackedTypeStoreTest {
}
@Test
@GraphTransaction
public void testStore() throws AtlasException {
typeStore.store(ts);
dumpGraph();
......@@ -137,4 +145,54 @@ public class GraphBackedTypeStoreTest {
ts.reset();
ts.defineTypes(types);
}
@Test(dependsOnMethods = "testStore")
public void testTypeUpdate() throws Exception {
//Add enum value
EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1),
new EnumValue("L2", 2), new EnumValue("L3", 3));
//Add attribute
StructTypeDefinition addressDetails =
createStructTypeDef("Address", createRequiredAttrDef("street", DataTypes.STRING_TYPE),
createRequiredAttrDef("city", DataTypes.STRING_TYPE),
createOptionalAttrDef("state", DataTypes.STRING_TYPE));
//Add supertype
HierarchicalTypeDefinition<ClassType> superTypeDef = createClassTypeDef("Division", ImmutableList.<String>of(),
createOptionalAttrDef("dname", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department",
ImmutableList.of(superTypeDef.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
true, "department"));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(deptTypeDef, superTypeDef));
ts.updateTypes(typesDef);
typeStore.store(ts, ImmutableList.of(orgLevelEnum.name, addressDetails.typeName, superTypeDef.typeName,
deptTypeDef.typeName));
//Validate the updated types
TypesDef types = typeStore.restore();
ts.reset();
ts.defineTypes(types);
//Assert new enum value
EnumType orgLevel = ts.getDataType(EnumType.class, orgLevelEnum.name);
Assert.assertEquals(orgLevel.name, orgLevelEnum.name);
Assert.assertEquals(orgLevel.values().size(), orgLevelEnum.enumValues.length);
Assert.assertEquals(orgLevel.fromValue("L3").ordinal, 3);
//Assert new attribute
StructType addressType = ts.getDataType(StructType.class, addressDetails.typeName);
Assert.assertEquals(addressType.numFields, 3);
Assert.assertEquals(addressType.fieldMapping.fields.get("state").dataType(), DataTypes.STRING_TYPE);
//Assert new super type
ClassType deptType = ts.getDataType(ClassType.class, deptTypeDef.typeName);
Assert.assertTrue(deptType.superTypes.contains(superTypeDef.typeName));
Assert.assertNotNull(ts.getDataType(ClassType.class, superTypeDef.typeName));
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
......@@ -40,7 +40,17 @@ abstract class AbstractDataType<T> implements IDataType<T> {
} else {
TypeUtils.outputVal(val == null ? "<null>" : val.toString(), buf, prefix);
}
}
/**
* Validate that current definition can be updated with the new definition
* @param newType
*/
@Override
public void validateUpdate(IDataType newType) throws TypeUpdateException {
if (!getName().equals(newType.getName()) || !getClass().getName().equals(newType.getClass().getName())) {
throw new TypeUpdateException(newType);
}
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
......@@ -71,6 +71,43 @@ public class AttributeInfo {
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
AttributeInfo that = (AttributeInfo) o;
if (isComposite != that.isComposite) {
return false;
}
if (isUnique != that.isUnique) {
return false;
}
if (isIndexable != that.isIndexable) {
return false;
}
if (!dataType.getName().equals(that.dataType.getName())) {
return false;
}
if (!multiplicity.equals(that.multiplicity)) {
return false;
}
if (!name.equals(that.name)) {
return false;
}
if (reverseAttributeName != null ? !reverseAttributeName.equals(that.reverseAttributeName) :
that.reverseAttributeName != null) {
return false;
}
return true;
}
public String toJson() throws JSONException {
JSONObject json = new JSONObject();
json.put("name", name);
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
......@@ -76,6 +76,26 @@ public class EnumType extends AbstractDataType<EnumValue> {
return DataTypes.TypeCategory.ENUM;
}
@Override
public void validateUpdate(IDataType newType) throws TypeUpdateException {
super.validateUpdate(newType);
EnumType enumType = (EnumType)newType;
for (EnumValue enumValue : values()) {
//The old enum value should be part of new enum definition as well
if (!enumType.valueMap.containsKey(enumValue.value)) {
throw new TypeUpdateException("Value " + enumValue.value + " is missing in new type");
}
//The ordinal for old enum value can't change
EnumValue newEnumValue = enumType.valueMap.get(enumValue.value);
if (enumValue.ordinal != newEnumValue.ordinal) {
throw new TypeUpdateException(String.format("Ordinal mismatch %s(%s) != %s(%s)", enumValue.value,
enumValue.ordinal, newEnumValue.value, newEnumValue.ordinal));
}
}
}
public EnumValue fromOrdinal(int o) {
return ordinalMap.get(o);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
......@@ -105,6 +105,30 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
return (cType == this || cType.superTypePaths.containsKey(getName()));
}
/**
* Validate that current definition can be updated with the new definition
* @param newType
* @return true if the current definition can be updated with the new definition, else false
*/
@Override
public void validateUpdate(IDataType newType) throws TypeUpdateException {
super.validateUpdate(newType);
HierarchicalType newHierarchicalType = (HierarchicalType) newType;
//validate on supertypes
if (!newHierarchicalType.superTypes.containsAll(superTypes)) {
throw new TypeUpdateException(newType, "New type doesn't contain all super types of old type");
}
//validate on fields
try {
TypeUtils.validateUpdate(fieldMapping, newHierarchicalType.fieldMapping);
} catch (TypeUpdateException e) {
throw new TypeUpdateException(newType, e);
}
}
protected void setupSuperTypesGraph() throws AtlasException {
setupSuperTypesGraph(superTypes);
}
......@@ -147,9 +171,9 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
protected Pair<FieldMapping, ImmutableMap<String, String>> constructFieldMapping(ImmutableList<String> superTypes,
AttributeInfo... fields) throws AtlasException {
Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<String, AttributeInfo>();
Map<String, Integer> fieldPos = new HashMap<String, Integer>();
Map<String, Integer> fieldNullPos = new HashMap<String, Integer>();
Map<String, AttributeInfo> fieldsMap = new LinkedHashMap();
Map<String, Integer> fieldPos = new HashMap();
Map<String, Integer> fieldNullPos = new HashMap();
Map<String, String> attributeNameToType = new HashMap<>();
int numBools = 0;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
......@@ -28,4 +28,7 @@ public interface IDataType<T> {
DataTypes.TypeCategory getTypeCategory();
void output(T val, Appendable buf, String prefix) throws AtlasException;
void validateUpdate(IDataType newType) throws TypeUpdateException;
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
......@@ -18,7 +18,6 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
......@@ -49,11 +48,11 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
this.handler = null;
}
protected StructType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes, AttributeInfo... fields)
protected StructType(TypeSystem typeSystem, String name, AttributeInfo... fields)
throws AtlasException {
this.typeSystem = typeSystem;
this.name = name;
this.fieldMapping = constructFieldMapping(superTypes, fields);
this.fieldMapping = constructFieldMapping(fields);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping);
this.numFields = this.fieldMapping.fields.size();
this.handler = new TypedStructHandler(this);
......@@ -68,7 +67,24 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
return name;
}
protected FieldMapping constructFieldMapping(ImmutableList<String> superTypes, AttributeInfo... fields)
/**
* Validate that current definition can be updated with the new definition
* @param newType
* @return true if the current definition can be updated with the new definition, else false
*/
@Override
public void validateUpdate(IDataType newType) throws TypeUpdateException {
super.validateUpdate(newType);
StructType newStructType = (StructType) newType;
try {
TypeUtils.validateUpdate(fieldMapping, newStructType.fieldMapping);
} catch (TypeUpdateException e) {
throw new TypeUpdateException(newType, e);
}
}
protected FieldMapping constructFieldMapping(AttributeInfo... fields)
throws AtlasException {
Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<String, AttributeInfo>();
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -35,7 +35,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -193,7 +192,7 @@ public class TypeSystem {
infos[i] = new AttributeInfo(this, attrDefs[i], tempTypes);
}
return new StructType(this, name, null, infos);
return new StructType(this, name, infos);
}
public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef) throws AtlasException {
......@@ -225,6 +224,18 @@ public class TypeSystem {
return transientTypes.defineTypes();
}
public Map<String, IDataType> updateTypes(TypesDef typesDef) throws AtlasException {
ImmutableList<EnumTypeDefinition> enumDefs = ImmutableList.copyOf(typesDef.enumTypesAsJavaList());
ImmutableList<StructTypeDefinition> structDefs = ImmutableList.copyOf(typesDef.structTypesAsJavaList());
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs =
ImmutableList.copyOf(typesDef.traitTypesAsJavaList());
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs =
ImmutableList.copyOf(typesDef.classTypesAsJavaList());
TransientTypeSystem transientTypes = new TransientTypeSystem(enumDefs, structDefs, traitDefs, classDefs);
return transientTypes.defineTypes(true);
}
public Map<String, IDataType> defineTypes(TypesDef typesDef) throws AtlasException {
ImmutableList<EnumTypeDefinition> enumDefs = ImmutableList.copyOf(typesDef.enumTypesAsJavaList());
ImmutableList<StructTypeDefinition> structDefs = ImmutableList.copyOf(typesDef.structTypesAsJavaList());
......@@ -241,17 +252,12 @@ public class TypeSystem {
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) throws AtlasException {
TransientTypeSystem transientTypes = new TransientTypeSystem(enumDefs, structDefs, traitDefs, classDefs);
Map<String, IDataType> definedTypes = transientTypes.defineTypes();
// LOG.debug("Defined new types " + Arrays.toString(definedTypes.keySet().toArray(new
// String[definedTypes.size()])));
return definedTypes;
return transientTypes.defineTypes();
}
public DataTypes.ArrayType defineArrayType(IDataType elemType) throws AtlasException {
assert elemType != null;
DataTypes.ArrayType dT = new DataTypes.ArrayType(elemType);
// types.put(dT.getName(), dT);
// typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ARRAY, dT.getName());
return dT;
}
......@@ -259,8 +265,6 @@ public class TypeSystem {
assert keyType != null;
assert valueType != null;
DataTypes.MapType dT = new DataTypes.MapType(keyType, valueType);
// types.put(dT.getName(), dT);
// typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.MAP, dT.getName());
return dT;
}
......@@ -303,15 +307,16 @@ public class TypeSystem {
final ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs;
final ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs;
private final ImmutableList<EnumTypeDefinition> enumDefs;
Map<String, StructTypeDefinition> structNameToDefMap = new HashMap<>();
Map<String, HierarchicalTypeDefinition<TraitType>> traitNameToDefMap = new HashMap<>();
Map<String, HierarchicalTypeDefinition<ClassType>> classNameToDefMap = new HashMap<>();
Set<String> transientTypes;
Map<String, IDataType> transientTypes = null;
List<AttributeInfo> recursiveRefs;
List<DataTypes.ArrayType> recursiveArrayTypes;
List<DataTypes.MapType> recursiveMapTypes;
List<AttributeInfo> recursiveRefs = new ArrayList<>();
List<DataTypes.ArrayType> recursiveArrayTypes = new ArrayList<>();
List<DataTypes.MapType> recursiveMapTypes = new ArrayList<>();
TransientTypeSystem(ImmutableList<EnumTypeDefinition> enumDefs, ImmutableList<StructTypeDefinition> structDefs,
......@@ -321,17 +326,13 @@ public class TypeSystem {
this.structDefs = structDefs;
this.traitDefs = traitDefs;
this.classDefs = classDefs;
structNameToDefMap = new HashMap<>();
traitNameToDefMap = new HashMap<>();
classNameToDefMap = new HashMap<>();
recursiveRefs = new ArrayList<>();
recursiveArrayTypes = new ArrayList<>();
recursiveMapTypes = new ArrayList<>();
transientTypes = new LinkedHashSet<>();
transientTypes = new HashMap<>();
}
private IDataType dataType(String name) {
private IDataType dataType(String name) throws AtlasException {
if (transientTypes.containsKey(name)) {
return transientTypes.get(name);
}
return TypeSystem.this.types.get(name);
}
......@@ -340,52 +341,50 @@ public class TypeSystem {
* - validate cannot redefine types
* - setup shallow Type instances to facilitate recursive type graphs
*/
private void step1() throws AtlasException {
private void step1(boolean update) throws AtlasException {
for (EnumTypeDefinition eDef : enumDefs) {
assert eDef.name != null;
if (types.containsKey(eDef.name)) {
if (!update && (transientTypes.containsKey(eDef.name) || types.containsKey(eDef.name))) {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
}
EnumType eT = new EnumType(this, eDef.name, eDef.enumValues);
TypeSystem.this.types.put(eDef.name, eT);
typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ENUM, eDef.name);
transientTypes.add(eDef.name);
transientTypes.put(eDef.name, eT);
}
for (StructTypeDefinition sDef : structDefs) {
assert sDef.typeName != null;
if (dataType(sDef.typeName) != null) {
if (!update && (transientTypes.containsKey(sDef.typeName) || types.containsKey(sDef.typeName))) {
throw new TypeExistsException(String.format("Cannot redefine type %s", sDef.typeName));
}
TypeSystem.this.types
.put(sDef.typeName, new StructType(this, sDef.typeName, sDef.attributeDefinitions.length));
StructType sT = new StructType(this, sDef.typeName, sDef.attributeDefinitions.length);
structNameToDefMap.put(sDef.typeName, sDef);
transientTypes.add(sDef.typeName);
transientTypes.put(sDef.typeName, sT);
}
for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) {
assert traitDef.typeName != null;
if (types.containsKey(traitDef.typeName)) {
if (!update &&
(transientTypes.containsKey(traitDef.typeName) || types.containsKey(traitDef.typeName))) {
throw new TypeExistsException(String.format("Cannot redefine type %s", traitDef.typeName));
}
TypeSystem.this.types.put(traitDef.typeName, new TraitType(this, traitDef.typeName, traitDef.superTypes,
traitDef.attributeDefinitions.length));
TraitType tT = new TraitType(this, traitDef.typeName, traitDef.superTypes,
traitDef.attributeDefinitions.length);
traitNameToDefMap.put(traitDef.typeName, traitDef);
transientTypes.add(traitDef.typeName);
transientTypes.put(traitDef.typeName, tT);
}
for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) {
assert classDef.typeName != null;
if (types.containsKey(classDef.typeName)) {
if (!update &&
(transientTypes.containsKey(classDef.typeName) || types.containsKey(classDef.typeName))) {
throw new TypeExistsException(String.format("Cannot redefine type %s", classDef.typeName));
}
TypeSystem.this.types.put(classDef.typeName, new ClassType(this, classDef.typeName, classDef.superTypes,
classDef.attributeDefinitions.length));
ClassType cT = new ClassType(this, classDef.typeName, classDef.superTypes,
classDef.attributeDefinitions.length);
classNameToDefMap.put(classDef.typeName, classDef);
transientTypes.add(classDef.typeName);
transientTypes.put(classDef.typeName, cT);
}
}
......@@ -438,20 +437,20 @@ public class TypeSystem {
private AttributeInfo constructAttributeInfo(AttributeDefinition attrDef) throws AtlasException {
AttributeInfo info = new AttributeInfo(this, attrDef, null);
if (transientTypes.contains(attrDef.dataTypeName)) {
if (transientTypes.keySet().contains(attrDef.dataTypeName)) {
recursiveRefs.add(info);
}
if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
DataTypes.ArrayType arrType = (DataTypes.ArrayType) info.dataType();
if (transientTypes.contains(arrType.getElemType().getName())) {
if (transientTypes.keySet().contains(arrType.getElemType().getName())) {
recursiveArrayTypes.add(arrType);
}
}
if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
DataTypes.MapType mapType = (DataTypes.MapType) info.dataType();
if (transientTypes.contains(mapType.getKeyType().getName())) {
if (transientTypes.keySet().contains(mapType.getKeyType().getName())) {
recursiveMapTypes.add(mapType);
} else if (transientTypes.contains(mapType.getValueType().getName())) {
} else if (transientTypes.keySet().contains(mapType.getValueType().getName())) {
recursiveMapTypes.add(mapType);
}
}
......@@ -472,8 +471,8 @@ public class TypeSystem {
infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
}
StructType type = new StructType(TypeSystem.this, def.typeName, null, infos);
TypeSystem.this.types.put(def.typeName, type);
StructType type = new StructType(this, def.typeName, infos);
transientTypes.put(def.typeName, type);
return type;
}
......@@ -487,11 +486,12 @@ public class TypeSystem {
try {
Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, ImmutableList.class,
AttributeInfo[].class);
U type = cons.newInstance(TypeSystem.this, def.typeName, def.superTypes, infos);
TypeSystem.this.types.put(def.typeName, type);
U type = cons.newInstance(this, def.typeName, def.superTypes, infos);
transientTypes.put(def.typeName, type);
return type;
} catch (Exception e) {
throw new AtlasException(String.format("Cannot construct Type of MetaType %s", cls.getName()), e);
e.printStackTrace();
throw new AtlasException(String.format("Cannot construct Type of MetaType %s - %s", cls.getName(), def.typeName), e);
}
}
......@@ -516,17 +516,14 @@ public class TypeSystem {
for (StructTypeDefinition structDef : structDefs) {
constructStructureType(structDef);
typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.STRUCT, structDef.typeName);
}
for (TraitType traitType : traitTypes) {
constructHierarchicalType(TraitType.class, traitNameToDefMap.get(traitType.getName()));
typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.TRAIT, traitType.getName());
}
for (ClassType classType : classTypes) {
constructHierarchicalType(ClassType.class, classNameToDefMap.get(classType.getName()));
typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.CLASS, classType.getName());
}
}
......@@ -547,35 +544,89 @@ public class TypeSystem {
}
}
/**
* Step 5:
* - Validate that the update can be done
*/
private void step5() throws TypeUpdateException {
//If the type is modified, validate that update can be done
for (IDataType newType : transientTypes.values()) {
if (TypeSystem.this.types.containsKey(newType.getName())) {
IDataType oldType = TypeSystem.this.types.get(newType.getName());
oldType.validateUpdate(newType);
}
}
}
Map<String, IDataType> defineTypes() throws AtlasException {
try {
step1();
step2();
return defineTypes(false);
}
step3();
step4();
} catch (AtlasException me) {
for (String sT : transientTypes) {
types.remove(sT);
}
throw me;
Map<String, IDataType> defineTypes(boolean update) throws AtlasException {
step1(update);
step2();
step3();
step4();
if (update) {
step5();
}
Map<String, IDataType> newTypes = new HashMap<>();
for (String tName : transientTypes) {
newTypes.put(tName, dataType(tName));
for (Map.Entry<String, IDataType> typeEntry : transientTypes.entrySet()) {
String typeName = typeEntry.getKey();
IDataType type = typeEntry.getValue();
//Add/replace the new type in the typesystem
TypeSystem.this.types.put(typeName, type);
typeCategoriesToTypeNamesMap.put(type.getTypeCategory(), typeName);
newTypes.put(typeName, type);
}
return newTypes;
}
@Override
public ImmutableList<String> getTypeNames() {
return TypeSystem.this.getTypeNames();
Set<String> typeNames = transientTypes.keySet();
typeNames.addAll(TypeSystem.this.getTypeNames());
return ImmutableList.copyOf(typeNames);
}
//get from transient types. Else, from main type system
@Override
public <T> T getDataType(Class<T> cls, String name) throws AtlasException {
if (transientTypes != null) {
if (transientTypes.containsKey(name)) {
try {
return cls.cast(transientTypes.get(name));
} catch (ClassCastException cce) {
throw new AtlasException(cce);
}
}
/*
* is this an Array Type?
*/
String arrElemType = TypeUtils.parseAsArrayType(name);
if (arrElemType != null) {
IDataType dT = defineArrayType(getDataType(IDataType.class, arrElemType));
return cls.cast(dT);
}
/*
* is this a Map Type?
*/
String[] mapType = TypeUtils.parseAsMapType(name);
if (mapType != null) {
IDataType dT =
defineMapType(getDataType(IDataType.class, mapType[0]), getDataType(IDataType.class, mapType[1]));
return cls.cast(dT);
}
}
return TypeSystem.this.getDataType(cls, name);
}
......@@ -605,12 +656,12 @@ public class TypeSystem {
@Override
public DataTypes.ArrayType defineArrayType(IDataType elemType) throws AtlasException {
throw new AtlasException("Internal Error: define type called on TrasientTypeSystem");
return super.defineArrayType(elemType);
}
@Override
public DataTypes.MapType defineMapType(IDataType keyType, IDataType valueType) throws AtlasException {
throw new AtlasException("Internal Error: define type called on TrasientTypeSystem");
return super.defineMapType(keyType, valueType);
}
}
......@@ -631,7 +682,7 @@ public class TypeSystem {
infos[0] = new AttributeInfo(TypeSystem.this, idAttr, null);
infos[1] = new AttributeInfo(TypeSystem.this, typNmAttr, null);
StructType type = new StructType(TypeSystem.this, TYP_NAME, null, infos);
StructType type = new StructType(TypeSystem.this, TYP_NAME, infos);
TypeSystem.this.types.put(TYP_NAME, type);
} catch (AtlasException me) {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.typesystem.types;
import org.apache.atlas.AtlasException;
public class TypeUpdateException extends AtlasException {
public TypeUpdateException(IDataType newType) {
super(newType.getName() + " can't be updated");
}
public TypeUpdateException(IDataType newType, Exception e) {
super(newType.getName() + " can't be updated - " + e.getMessage(), e);
}
public TypeUpdateException(String message) {
super(message);
}
public TypeUpdateException(IDataType newType, String message) {
super(newType.getName() + " can't be updated - " + message);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java
......@@ -19,16 +19,15 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import scala.collection.JavaConversions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
......@@ -59,23 +58,16 @@ public class TypeUtils {
}
public static Map<AttributeInfo, List<String>> buildAttrInfoToNameMap(FieldMapping f) {
Map<AttributeInfo, List<String>> b = new HashMap<AttributeInfo, List<String>>();
Map<AttributeInfo, List<String>> b = new HashMap();
for (Map.Entry<String, AttributeInfo> e : f.fields.entrySet()) {
List<String> names = b.get(e.getValue());
if (names == null) {
names = new ArrayList<String>();
names = new ArrayList<>();
b.put(e.getValue(), names);
}
names.add(e.getKey());
}
return ImmutableMap.copyOf(b);
}
public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums,
ImmutableList<StructTypeDefinition> structs, ImmutableList<HierarchicalTypeDefinition<TraitType>> traits,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classes) {
return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
return b;
}
protected static class Pair<L, R> {
......@@ -87,4 +79,43 @@ public class TypeUtils {
this.right = right;
}
}
/**
* Validates that the old field mapping can be replaced with new field mapping
* @param oldFieldMapping
* @param newFieldMapping
*/
public static void validateUpdate(FieldMapping oldFieldMapping, FieldMapping newFieldMapping)
throws TypeUpdateException {
Map<String, AttributeInfo> newFields = newFieldMapping.fields;
for (AttributeInfo attribute : oldFieldMapping.fields.values()) {
if (newFields.containsKey(attribute.name)) {
AttributeInfo newAttribute = newFields.get(attribute.name);
//If old attribute is also in new definition, only allowed change is multiplicity change from REQUIRED to OPTIONAL
if (!newAttribute.equals(attribute)) {
if (attribute.multiplicity == Multiplicity.REQUIRED
&& newAttribute.multiplicity == Multiplicity.OPTIONAL) {
continue;
} else {
throw new TypeUpdateException("Attribute " + attribute.name + " can't be updated");
}
}
} else {
//If old attribute is missing in new definition, return false as attributes can't be deleted
throw new TypeUpdateException("Old Attribute " + attribute.name + " is missing");
}
}
//Only new attributes
Set<String> newAttributes = new HashSet<>(ImmutableList.copyOf(newFields.keySet()));
newAttributes.removeAll(oldFieldMapping.fields.keySet());
for (String attributeName : newAttributes) {
AttributeInfo newAttribute = newFields.get(attributeName);
//New required attribute can't be added
if (newAttribute.multiplicity == Multiplicity.REQUIRED) {
throw new TypeUpdateException("Can't add required attribute " + attributeName);
}
}
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.typesystem.types.utils;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
......@@ -28,6 +29,7 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import scala.collection.JavaConversions;
/**
* Types utilities class.
......@@ -74,4 +76,11 @@ public class TypesUtil {
ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(ClassType.class, name, superTypes, attrDefs);
}
public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums,
ImmutableList<StructTypeDefinition> structs, ImmutableList<HierarchicalTypeDefinition<TraitType>> traits,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classes) {
return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
}
}
typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.RandomStringUtils;
import org.testng.annotations.BeforeMethod;
import java.math.BigDecimal;
......@@ -96,8 +97,8 @@ public abstract class BaseTest {
System.out.println("defined recursiveStructType = " + recursiveStructType);
}
protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition... tDefs) throws AtlasException {
protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition<TraitType>... tDefs)
throws AtlasException {
return getTypeSystem().defineTraitTypes(tDefs);
}
......@@ -168,4 +169,8 @@ public abstract class BaseTest {
return hrDept;
}
protected String newName() {
return RandomStringUtils.randomAlphanumeric(10);
}
}
typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
......@@ -18,14 +18,17 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.Test;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class ClassTest extends BaseTest {
public class ClassTest extends HierarchicalTypeTest<ClassType> {
@BeforeMethod
public void setup() throws Exception {
......@@ -67,6 +70,29 @@ public class ClassTest extends BaseTest {
"\t\tlevel : \t\t1\n" +
"\t}}]\n" +
"}");
}
@Override
protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, AttributeDefinition... attributes) {
return new HierarchicalTypeDefinition(ClassType.class, name, null, attributes);
}
@Override
protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, ImmutableList<String> superTypes,
AttributeDefinition... attributes) {
return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attributes);
}
@Override
protected TypesDef getTypesDef(StructTypeDefinition typeDefinition) {
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of((HierarchicalTypeDefinition<ClassType>) typeDefinition));
}
@Override
protected TypesDef getTypesDef(HierarchicalTypeDefinition<ClassType>... typeDefinitions) {
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.copyOf(typeDefinitions));
}
}
typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java
......@@ -27,9 +27,11 @@ import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.Test;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.math.BigDecimal;
import java.math.BigInteger;
......@@ -59,7 +61,47 @@ public class EnumTest extends BaseTest {
ts.defineEnumType("LockLevel", new EnumValue("DB", 1), new EnumValue("TABLE", 2),
new EnumValue("PARTITION", 3));
}
@Test
public void testTypeUpdate() throws Exception {
TypeSystem ts = getTypeSystem();
EnumTypeDefinition etd = new EnumTypeDefinition(newName(), new EnumValue("A", 1));
TypesDef typesDef = getTypesDef(etd);
ts.defineTypes(typesDef);
//Allow adding new enum
etd = new EnumTypeDefinition(etd.name, new EnumValue("A", 1), new EnumValue("B", 2));
typesDef = getTypesDef(etd);
ts.updateTypes(typesDef);
//Don't allow deleting enum
etd = new EnumTypeDefinition(etd.name, new EnumValue("A", 1));
typesDef = getTypesDef(etd);
try {
ts.updateTypes(typesDef);
Assert.fail("Expected TypeUpdateException");
} catch (TypeUpdateException e) {
//assert that type is not updated when validation fails
EnumType enumType = ts.getDataType(EnumType.class, etd.name);
Assert.assertEquals(enumType.values().size(), 2);
}
//Don't allow changing ordinal of existing enum value
etd = new EnumTypeDefinition(etd.name, new EnumValue("A", 2));
typesDef = getTypesDef(etd);
try {
ts.updateTypes(typesDef);
Assert.fail("Expected TypeUpdateException");
} catch (TypeUpdateException e) {
//expected
}
}
private TypesDef getTypesDef(EnumTypeDefinition enumTypeDefinition) {
return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
}
protected void fillStruct(Struct s) throws AtlasException {
......
typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.Test;
public abstract class HierarchicalTypeTest<T extends HierarchicalType> extends TypeUpdateBaseTest {
@Test(enabled = false)
public void testTypeUpdate() throws Exception {
testTypeUpdateForAttributes();
//Test super types
HierarchicalTypeDefinition classType =
getTypeDefinition(newName(), TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
TypeSystem ts = getTypeSystem();
ts.defineTypes(getTypesDef(classType));
//Add super type with optional attribute
HierarchicalTypeDefinition superType =
getTypeDefinition(newName(), TypesUtil.createOptionalAttrDef("s", DataTypes.INT_TYPE));
classType = getTypeDefinition(classType.typeName, ImmutableList.of(superType.typeName),
TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
ts.updateTypes(getTypesDef(superType, classType));
//Add super type with required attribute should fail
HierarchicalTypeDefinition superTypeRequired =
getTypeDefinition(newName(), TypesUtil.createRequiredAttrDef("s", DataTypes.INT_TYPE));
classType = getTypeDefinition(classType.typeName,
ImmutableList.of(superTypeRequired.typeName, superType.typeName),
TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
try {
ts.updateTypes(getTypesDef(superTypeRequired, classType));
Assert.fail("Expected TypeUpdateException");
} catch (TypeUpdateException e) {
//expected
}
//Deleting super type should fail
classType = getTypeDefinition(classType.typeName, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
try {
ts.updateTypes(getTypesDef(superType, classType));
Assert.fail("Expected TypeUpdateException");
} catch (TypeUpdateException e) {
//expected
}
}
@Override
protected abstract HierarchicalTypeDefinition<T> getTypeDefinition(String name, AttributeDefinition... attributes);
protected abstract HierarchicalTypeDefinition<T> getTypeDefinition(String name, ImmutableList<String> superTypes,
AttributeDefinition... attributes);
@Override
protected abstract TypesDef getTypesDef(StructTypeDefinition typeDefinition);
protected abstract TypesDef getTypesDef(HierarchicalTypeDefinition<T>... typeDefinitions);
@Override
protected int getNumberOfFields(TypeSystem ts, String typeName) throws Exception {
return ts.getDataType(HierarchicalType.class, typeName).numFields;
}
}
\ No newline at end of file
typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
......@@ -18,14 +18,17 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.Test;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class StructTest extends BaseTest {
public class StructTest extends TypeUpdateBaseTest {
StructType structType;
StructType recursiveStructType;
......@@ -78,4 +81,25 @@ public class StructTest extends BaseTest {
"}");
}
@Test
public void testTypeUpdate() throws Exception {
testTypeUpdateForAttributes();
}
@Override
protected int getNumberOfFields(TypeSystem ts, String typeName) throws Exception {
return ts.getDataType(StructType.class, typeName).numFields;
}
@Override
protected StructTypeDefinition getTypeDefinition(String name, AttributeDefinition... attributes) {
return new StructTypeDefinition(name, attributes);
}
@Override
protected TypesDef getTypesDef(StructTypeDefinition typeDefinition) {
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(typeDefinition),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
}
}
typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
......@@ -23,9 +23,12 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.Map;
......@@ -33,7 +36,7 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAt
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
public class TraitTest extends BaseTest {
public class TraitTest extends HierarchicalTypeTest<TraitType> {
@BeforeMethod
......@@ -213,8 +216,30 @@ public class TraitTest extends BaseTest {
"\tA.C.D.c : \t3\n" +
"\tA.C.D.d : \t3\n" +
"}");
}
@Override
protected HierarchicalTypeDefinition<TraitType> getTypeDefinition(String name, AttributeDefinition... attributes) {
return new HierarchicalTypeDefinition(TraitType.class, name, null, attributes);
}
@Override
protected HierarchicalTypeDefinition<TraitType> getTypeDefinition(String name, ImmutableList<String> superTypes,
AttributeDefinition... attributes) {
return new HierarchicalTypeDefinition(TraitType.class, name, superTypes, attributes);
}
@Override
protected TypesDef getTypesDef(StructTypeDefinition typeDefinition) {
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of((HierarchicalTypeDefinition<TraitType>) typeDefinition),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
}
@Override
protected TypesDef getTypesDef(HierarchicalTypeDefinition<TraitType>... typeDefinitions) {
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.copyOf(typeDefinitions), ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
}
}
typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.typesystem.types;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
public abstract class TypeUpdateBaseTest extends BaseTest {
protected void testTypeUpdateForAttributes() throws Exception {
StructTypeDefinition typeDefinition =
getTypeDefinition(newName(), TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
TypeSystem ts = getTypeSystem();
TypesDef typesDef = getTypesDef(typeDefinition);
ts.defineTypes(typesDef);
String typeName = typeDefinition.typeName;
//Allow modifying required to optional attribute
typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE));
ts.updateTypes(getTypesDef(typeDefinition));
//Allow adding new optional attribute
typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("b", DataTypes.INT_TYPE));
ts.updateTypes(getTypesDef(typeDefinition));
//Don't allow adding required attribute
typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("b", DataTypes.INT_TYPE),
TypesUtil.createRequiredAttrDef("c", DataTypes.INT_TYPE));
try {
ts.updateTypes(getTypesDef(typeDefinition));
Assert.fail("Expected TypeUpdateException");
} catch (TypeUpdateException e) {
//assert that type is not updated when validation fails
Assert.assertEquals(getNumberOfFields(ts, typeDefinition.typeName), 2);
}
//Don't allow removing attribute
typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE));
try {
ts.updateTypes(getTypesDef(typeDefinition));
} catch (TypeUpdateException e) {
//expected
}
//Don't allow modifying other fields of attribute definition - optional to required
typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE),
TypesUtil.createRequiredAttrDef("b", DataTypes.INT_TYPE));
try {
ts.updateTypes(getTypesDef(typeDefinition));
} catch (TypeUpdateException e) {
//expected
}
//Don't allow modifying other fields of attribute definition - attribute type change
typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("b", DataTypes.STRING_TYPE));
try {
ts.updateTypes(getTypesDef(typeDefinition));
} catch (TypeUpdateException e) {
//expected
}
//Don't allow modifying other fields of attribute definition - attribute type change
typeDefinition = getTypeDefinition(typeName, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
new AttributeDefinition("b", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
Multiplicity.COLLECTION, false, null));
try {
ts.updateTypes(getTypesDef(typeDefinition));
} catch (TypeUpdateException e) {
//expected
}
}
protected abstract int getNumberOfFields(TypeSystem ts, String typeName) throws Exception;
protected abstract TypesDef getTypesDef(StructTypeDefinition typeDefinition);
protected abstract StructTypeDefinition getTypeDefinition(String typeName,
AttributeDefinition... attributeDefinitions);
}
webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
......@@ -35,7 +35,6 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONArray;
......@@ -153,7 +152,7 @@ public class QuickStart {
HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null);
return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef));
}
......
webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
......@@ -38,6 +38,7 @@ import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
......@@ -110,6 +111,50 @@ public class TypesResource {
}
/**
* Update of existing types - if the given type doesn't exist, creates new type
* Allowed updates are:
* 1. Add optional attribute
* 2. Change required to optional attribute
* 3. Add super types - super types shouldn't contain any required attributes
* @param request
* @return
*/
@PUT
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response update(@Context HttpServletRequest request) {
try {
final String typeDefinition = Servlets.getRequestPayload(request);
LOG.debug("Updating type with definition {} ", typeDefinition);
JSONObject typesJson = metadataService.updateType(typeDefinition);
final JSONArray typesJsonArray = typesJson.getJSONArray(AtlasClient.TYPES);
JSONArray typesResponse = new JSONArray();
for (int i = 0; i < typesJsonArray.length(); i++) {
final String name = typesJsonArray.getString(i);
typesResponse.put(new JSONObject() {{
put(AtlasClient.NAME, name);
}});
}
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
response.put(AtlasClient.TYPES, typesResponse);
return Response.ok().entity(response).build();
} catch (TypeExistsException e) {
LOG.error("Type already exists", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist types", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to persist types", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Fetch the complete definition of a given type name which is unique.
*
* @param typeName name of a type which is unique.
......
webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
......@@ -41,7 +41,6 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.configuration.Configuration;
......@@ -188,7 +187,7 @@ public abstract class BaseResourceIT {
HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL", null);
TypesDef typesDef = TypeUtils.getTypesDef(ImmutableList.of(enumTypeDefinition),
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition),
ImmutableList.of(structTypeDefinition),
ImmutableList.of(classificationTrait, piiTrait, phiTrait, pciTrait, soxTrait, secTrait, financeTrait,
dimTraitDef, factTraitDef, metricTraitDef, etlTraitDef),
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
......@@ -38,7 +38,6 @@ import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.lang.RandomStringUtils;
......@@ -112,6 +111,34 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertNotNull(response.get(AtlasClient.GUID));
}
@Test
public void testEntityDefinitionAcrossTypeUpdate() throws Exception {
//create type
HierarchicalTypeDefinition<ClassType> typeDefinition = TypesUtil
.createClassTypeDef(randomString(), ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE));
serviceClient.createType(TypesSerialization.toJson(typeDefinition, false));
//create entity for the type
Referenceable instance = new Referenceable(typeDefinition.typeName);
instance.set("name", randomString());
String guid = serviceClient.createEntity(instance).getString(0);
//update type - add attribute
typeDefinition = TypesUtil.createClassTypeDef(typeDefinition.typeName, ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE));
TypesDef typeDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(typeDefinition));
serviceClient.updateType(typeDef);
//Get definition after type update - new attributes should be null
Referenceable entity = serviceClient.getEntity(guid);
Assert.assertNull(entity.get("description"));
Assert.assertEquals(entity.get("name"), instance.get("name"));
}
@DataProvider
public Object[][] invalidAttrValues() {
return new Object[][]{{null}, {""}};
......@@ -506,10 +533,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
HierarchicalTypeDefinition<ClassType> classTypeDefinition = TypesUtil
.createClassTypeDef(classType, ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE));
TypesDef typesDef = TypeUtils
.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(classTypeDefinition));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(classTypeDefinition));
createType(typesDef);
Referenceable instance = new Referenceable(classType);
......
webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
......@@ -32,7 +32,6 @@ import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
......@@ -176,8 +175,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
HierarchicalTypeDefinition<TraitType> financeTrait =
TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
TypesDef typesDef = TypeUtils
.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList
.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait, soxTrait, secTrait,
financeTrait), ImmutableList.of(dslTestTypeDefinition));
......
webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
......@@ -28,8 +28,10 @@ import org.apache.atlas.typesystem.json.TypesSerialization$;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.web.util.Servlets;
......@@ -87,6 +89,33 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
}
}
@Test
public void testUpdate() throws Exception {
HierarchicalTypeDefinition<ClassType> typeDefinition = TypesUtil
.createClassTypeDef(randomString(), ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE));
List<String> typesCreated = serviceClient.createType(TypesSerialization.toJson(typeDefinition, false));
Assert.assertEquals(typesCreated.size(), 1);
Assert.assertEquals(typesCreated.get(0), typeDefinition.typeName);
//Add super type
HierarchicalTypeDefinition<ClassType> superTypeDefinition = TypesUtil
.createClassTypeDef(randomString(), ImmutableList.<String>of(),
TypesUtil.createOptionalAttrDef("sname", DataTypes.STRING_TYPE));
typeDefinition = TypesUtil.createClassTypeDef(typeDefinition.typeName,
ImmutableList.of(superTypeDefinition.typeName),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE));
TypesDef typeDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(superTypeDefinition, typeDefinition));
List<String> typesUpdated = serviceClient.updateType(typeDef);
Assert.assertEquals(typesUpdated.size(), 2);
Assert.assertTrue(typesUpdated.contains(superTypeDefinition.typeName));
Assert.assertTrue(typesUpdated.contains(typeDefinition.typeName));
}
@Test(dependsOnMethods = "testSubmit")
public void testGetDefinition() throws Exception {
for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment