Skip to content

A
atlas
Commit b6fb1f1a by Shwetha GS
Options

ATLAS-479 Add description for different types during create time (guptaneeru via shwethags)

parent b9575f29
Showing with 382 additions and 236 deletions
addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
......@@ -115,7 +115,7 @@ public class FalconDataModelGenerator {
Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS_ENTITY.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS_ENTITY.getName(), null,
ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(FalconDataTypes.FALCON_PROCESS_ENTITY.getName(), definition);
LOG.debug("Created definition for {}", FalconDataTypes.FALCON_PROCESS_ENTITY.getName());
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
......@@ -201,7 +201,7 @@ public class HiveDataModelGenerator {
Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null,
ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName());
......@@ -236,7 +236,7 @@ public class HiveDataModelGenerator {
false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), null,
ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName());
......@@ -250,7 +250,7 @@ public class HiveDataModelGenerator {
new AttributeDefinition("fields", String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.getName()),
Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.getName(), null,
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.getName(), null, null,
attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_TYPE.getName(), definition);
......@@ -263,7 +263,7 @@ public class HiveDataModelGenerator {
new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), null,
ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName());
......@@ -284,7 +284,7 @@ public class HiveDataModelGenerator {
Multiplicity.OPTIONAL, true, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PARTITION.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PARTITION.getName(), null,
ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_PARTITION.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PARTITION.getName());
......@@ -318,7 +318,7 @@ public class HiveDataModelGenerator {
new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null,
ImmutableList.of("DataSet"), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName());
......@@ -333,7 +333,7 @@ public class HiveDataModelGenerator {
new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_ROLE.getName(), null,
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_ROLE.getName(), null, null,
attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_ROLE.getName(), definition);
......@@ -357,7 +357,7 @@ public class HiveDataModelGenerator {
null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null,
ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName());
......
addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
......@@ -129,7 +129,7 @@ public class SqoopDataModelGenerator {
DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(), null,
ImmutableList.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(SqoopDataTypes.SQOOP_DBDATASTORE.getName(), definition);
LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_DBDATASTORE.getName());
......@@ -148,7 +148,7 @@ public class SqoopDataModelGenerator {
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(),
new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(), null,
ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
classTypeDefinitions.put(SqoopDataTypes.SQOOP_PROCESS.getName(), definition);
LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_PROCESS.getName());
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-479 Add description for different types during create time (guptaneeru via shwethags)
ATLAS-508 Apache nightly build failure - UnsupportedOperationException: Not a single key: __traitNames (shwethags)
ATLAS-422 JavaDoc NotificationConsumer and NotificationInterface.(tbeerbower via sumasai)
ATLAS-536 Falcon hook loads incorrect configuration when -Datlas.conf is not given when falcon server startup (ayubkhan via shwethags)
......
repository/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -49,6 +49,7 @@ public final class Constants {
public static final String TYPE_CATEGORY_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.category";
public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type";
public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description";
/**
* Trait names property key and index name.
......
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
......@@ -79,7 +79,7 @@ public class GraphBackedTypeStore implements ITypeStore {
public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws AtlasException {
for (String typeName : typeNames) {
IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
LOG.debug("Processing {}.{} in type store", dataType.getTypeCategory(), dataType.getName());
LOG.debug("Processing {}.{}.{} in type store", dataType.getTypeCategory(), dataType.getName(), dataType.getDescription());
switch (dataType.getTypeCategory()) {
case ENUM:
storeInGraph((EnumType) dataType);
......@@ -87,14 +87,14 @@ public class GraphBackedTypeStore implements ITypeStore {
case STRUCT:
StructType structType = (StructType) dataType;
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(),
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(),
ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableList.<String>of());
break;
case TRAIT:
case CLASS:
HierarchicalType type = (HierarchicalType) dataType;
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.immediateAttrs,
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.immediateAttrs,
type.superTypes);
break;
......@@ -110,7 +110,7 @@ public class GraphBackedTypeStore implements ITypeStore {
}
private void storeInGraph(EnumType dataType) {
Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName());
Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription());
List<String> values = new ArrayList<>(dataType.values().size());
for (EnumValue enumValue : dataType.values()) {
String key = getPropertyKey(dataType.getName(), enumValue.value);
......@@ -133,8 +133,13 @@ public class GraphBackedTypeStore implements ITypeStore {
}
private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName,
ImmutableList<AttributeInfo> attributes, ImmutableList<String> superTypes) throws AtlasException {
storeInGraph(typeSystem, category, typeName, null, attributes, superTypes);
}
private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription,
ImmutableList<AttributeInfo> attributes, ImmutableList<String> superTypes) throws AtlasException {
Vertex vertex = createVertex(category, typeName);
Vertex vertex = createVertex(category, typeName, typeDescription);
List<String> attrNames = new ArrayList<>();
if (attributes != null) {
for (AttributeInfo attribute : attributes) {
......@@ -154,7 +159,7 @@ public class GraphBackedTypeStore implements ITypeStore {
if (superTypes != null) {
for (String superTypeName : superTypes) {
HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName);
Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription());
addEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL);
}
}
......@@ -199,7 +204,7 @@ public class GraphBackedTypeStore implements ITypeStore {
for (IDataType attrType : attrDataTypes) {
if (!coreTypes.contains(attrType.getName())) {
Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName());
Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription());
String label = getEdgeLabel(vertexTypeName, attribute.name);
addEdge(vertex, attrVertex, label);
}
......@@ -227,7 +232,8 @@ public class GraphBackedTypeStore implements ITypeStore {
Vertex vertex = (Vertex) vertices.next();
DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
LOG.info("Restoring type {}.{}", typeCategory, typeName);
String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
LOG.info("Restoring type {}.{}.{}", typeCategory, typeName, typeDescription);
switch (typeCategory) {
case ENUM:
enums.add(getEnumType(vertex));
......@@ -235,19 +241,19 @@ public class GraphBackedTypeStore implements ITypeStore {
case STRUCT:
AttributeDefinition[] attributes = getAttributes(vertex, typeName);
structs.add(new StructTypeDefinition(typeName, attributes));
structs.add(new StructTypeDefinition(typeName, typeDescription, attributes));
break;
case CLASS:
ImmutableList<String> superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex, typeName);
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, superTypes, attributes));
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, superTypes, attributes));
break;
case TRAIT:
superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex, typeName);
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, superTypes, attributes));
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, superTypes, attributes));
break;
default:
......@@ -259,13 +265,14 @@ public class GraphBackedTypeStore implements ITypeStore {
private EnumTypeDefinition getEnumType(Vertex vertex) {
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
List<EnumValue> enumValues = new ArrayList<>();
List<String> values = vertex.getProperty(getPropertyKey(typeName));
for (String value : values) {
String valueProperty = getPropertyKey(typeName, value);
enumValues.add(new EnumValue(value, vertex.<Integer>getProperty(valueProperty)));
}
return new EnumTypeDefinition(typeName, enumValues.toArray(new EnumValue[enumValues.size()]));
return new EnumTypeDefinition(typeName, typeDescription, enumValues.toArray(new EnumValue[enumValues.size()]));
}
private ImmutableList<String> getSuperTypes(Vertex vertex) {
......@@ -316,15 +323,23 @@ public class GraphBackedTypeStore implements ITypeStore {
return vertex;
}
private Vertex createVertex(DataTypes.TypeCategory category, String typeName) {
private Vertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription) {
Vertex vertex = findVertex(category, typeName);
if (vertex == null) {
LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
vertex = titanGraph.addVertex(null);
addProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); //Mark as type vertex
addProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark as type vertex
addProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category);
addProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName);
}
if (typeDescription != null) {
String oldDescription = getPropertyKey(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
if (!typeDescription.equals(oldDescription)) {
addProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDescription);
}
} else {
LOG.debug(" type description is null ");
}
return vertex;
}
}
repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
......@@ -151,7 +151,7 @@ public class BaseHiveRepositoryTest {
new AttributeDefinition("table", HIVE_TABLE_TYPE, Multiplicity.REQUIRED, false, null),
};
HierarchicalTypeDefinition<ClassType> partClsDef =
new HierarchicalTypeDefinition<>(ClassType.class, PARTITION_TYPE, null,
new HierarchicalTypeDefinition<>(ClassType.class, PARTITION_TYPE, null, null,
attributeDefinitions);
HierarchicalTypeDefinition<TraitType> dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null);
......
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -23,7 +23,6 @@ import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
......@@ -97,19 +96,20 @@ public final class TestUtils {
*/
public static void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException {
String _description = "_description";
EnumTypeDefinition orgLevelEnum =
new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1), new EnumValue("L2", 2));
new EnumTypeDefinition("OrgLevel", "OrgLevel"+_description, new EnumValue("L1", 1), new EnumValue("L2", 2));
StructTypeDefinition addressDetails =
createStructTypeDef("Address", createRequiredAttrDef("street", DataTypes.STRING_TYPE),
createStructTypeDef("Address", "Address"+_description, createRequiredAttrDef("street", DataTypes.STRING_TYPE),
createRequiredAttrDef("city", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", ImmutableList.<String>of(),
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description, ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
true, "department"));
HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableList.<String>of(),
HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createOptionalAttrDef("orgLevel", "OrgLevel"),
createOptionalAttrDef("address", "Address"),
......@@ -117,12 +117,12 @@ public final class TestUtils {
new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"),
new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null));
HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", ImmutableList.of("Person"),
HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableList.of("Person"),
new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
false, "manager"));
HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef =
createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(),
createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableList.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE));
ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
......@@ -199,6 +199,7 @@ public final class TestUtils {
public static final String SERDE_TYPE = "serdeType";
public static TypesDef defineHiveTypes() {
String _description = "_description";
HierarchicalTypeDefinition<ClassType> superTypeDefinition =
createClassTypeDef(SUPER_TYPE_NAME, ImmutableList.<String>of(),
createOptionalAttrDef("namespace", DataTypes.STRING_TYPE),
......@@ -206,27 +207,27 @@ public final class TestUtils {
createOptionalAttrDef("colo", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef(DATABASE_TYPE, ImmutableList.of(SUPER_TYPE_NAME),
createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableList.of(SUPER_TYPE_NAME),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
createOptionalAttrDef("created", DataTypes.DATE_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType",
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", "serdeType" + _description,
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE),
createOptionalAttrDef("description", DataTypes.STRING_TYPE)});
EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),};
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", "tableType" + _description, values);
HierarchicalTypeDefinition<ClassType> columnsDefinition =
createClassTypeDef(COLUMN_TYPE, ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE));
StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type",
StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description,
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),});
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
......@@ -245,7 +246,7 @@ public final class TestUtils {
};
HierarchicalTypeDefinition<ClassType> storageDescClsDef =
new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE,
new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description,
ImmutableList.of(SUPER_TYPE_NAME), attributeDefinitions);
AttributeDefinition[] partClsAttributes = new AttributeDefinition[]{
......@@ -263,11 +264,11 @@ public final class TestUtils {
new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> partClsDef =
new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type",
new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", "partition_class_type" + _description,
ImmutableList.of(SUPER_TYPE_NAME), partClsAttributes);
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE, ImmutableList.of(SUPER_TYPE_NAME),
createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableList.of(SUPER_TYPE_NAME),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE),
......@@ -307,14 +308,14 @@ public final class TestUtils {
new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null));
HierarchicalTypeDefinition<TraitType> piiTypeDefinition =
createTraitTypeDef(PII, ImmutableList.<String>of());
createTraitTypeDef(PII, PII + _description, ImmutableList.<String>of());
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
createTraitTypeDef(CLASSIFICATION, ImmutableList.<String>of(),
createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableList.<String>of(),
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition =
createTraitTypeDef("fetl" + CLASSIFICATION, ImmutableList.of(CLASSIFICATION),
createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableList.of(CLASSIFICATION),
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition),
......
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -24,6 +24,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
......@@ -50,6 +51,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.List;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
......@@ -103,6 +105,7 @@ public class GraphBackedTypeStoreTest {
@Test(dependsOnMethods = "testStore")
public void testRestore() throws Exception {
String description = "_description";
TypesDef types = typeStore.restore();
//validate enum
......@@ -110,6 +113,7 @@ public class GraphBackedTypeStoreTest {
Assert.assertEquals(1, enumTypes.size());
EnumTypeDefinition orgLevel = enumTypes.get(0);
Assert.assertEquals(orgLevel.name, "OrgLevel");
Assert.assertEquals(orgLevel.description, "OrgLevel"+description);
Assert.assertEquals(orgLevel.enumValues.length, 2);
EnumValue enumValue = orgLevel.enumValues[0];
Assert.assertEquals(enumValue.value, "L1");
......@@ -126,6 +130,7 @@ public class GraphBackedTypeStoreTest {
ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
Assert.assertEquals(classType.typeDescription, classType.typeName+description);
clsTypeFound = true;
}
}
......@@ -136,6 +141,7 @@ public class GraphBackedTypeStoreTest {
Assert.assertEquals(1, traitTypes.size());
HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0);
Assert.assertEquals("SecurityClearance", trait.typeName);
Assert.assertEquals(trait.typeName+description, trait.typeDescription);
Assert.assertEquals(1, trait.attributeDefinitions.length);
AttributeDefinition attribute = trait.attributeDefinitions[0];
Assert.assertEquals("level", attribute.name);
......@@ -149,7 +155,8 @@ public class GraphBackedTypeStoreTest {
@Test(dependsOnMethods = "testStore")
public void testTypeUpdate() throws Exception {
//Add enum value
EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1),
String _description = "_description_updated";
EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", "OrgLevel"+_description, new EnumValue("L1", 1),
new EnumValue("L2", 2), new EnumValue("L3", 3));
//Add attribute
......@@ -162,7 +169,7 @@ public class GraphBackedTypeStoreTest {
HierarchicalTypeDefinition<ClassType> superTypeDef = createClassTypeDef("Division", ImmutableList.<String>of(),
createOptionalAttrDef("dname", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department",
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description,
ImmutableList.of(superTypeDef.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
true, "department"));
......@@ -182,6 +189,7 @@ public class GraphBackedTypeStoreTest {
//Assert new enum value
EnumType orgLevel = ts.getDataType(EnumType.class, orgLevelEnum.name);
Assert.assertEquals(orgLevel.name, orgLevelEnum.name);
Assert.assertEquals(orgLevel.description, orgLevelEnum.description);
Assert.assertEquals(orgLevel.values().size(), orgLevelEnum.enumValues.length);
Assert.assertEquals(orgLevel.fromValue("L3").ordinal, 3);
......
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
......@@ -165,7 +165,6 @@ public class DefaultMetadataServiceTest {
entity.set("type", "VARCHAR(32)");
return entity;
}
@Test(expectedExceptions = TypeNotFoundException.class)
public void testCreateEntityWithUnknownDatatype() throws Exception {
Referenceable entity = new Referenceable("Unknown datatype");
......
repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
......@@ -78,7 +78,7 @@ object QueryTestsUtils extends GraphUtils {
new AttributeDefinition(name, dT.getName, m, isComposite, reverseAttributeName)
}
def dbClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "DB", null,
def dbClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "DB", null, null,
Array(
attrDef("name", DataTypes.STRING_TYPE),
attrDef("owner", DataTypes.STRING_TYPE),
......@@ -92,21 +92,21 @@ object QueryTestsUtils extends GraphUtils {
attrDef("order", DataTypes.INT_TYPE)
))
def storageDescClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "StorageDescriptor", null,
def storageDescClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "StorageDescriptor", null, null,
Array(
attrDef("inputFormat", DataTypes.STRING_TYPE),
attrDef("outputFormat", DataTypes.STRING_TYPE),
new AttributeDefinition("sortCols", DataTypes.arrayTypeName("HiveOrder"), Multiplicity.REQUIRED, false, null)
))
def columnClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Column", null,
def columnClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Column", null, null,
Array(
attrDef("name", DataTypes.STRING_TYPE),
attrDef("dataType", DataTypes.STRING_TYPE),
new AttributeDefinition("sd", "StorageDescriptor", Multiplicity.REQUIRED, false, null)
))
def tblClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Table", null,
def tblClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Table", null, null,
Array(
attrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("db", "DB", Multiplicity.REQUIRED, false, null),
......@@ -114,34 +114,34 @@ object QueryTestsUtils extends GraphUtils {
attrDef("created", DataTypes.DATE_TYPE)
))
def partitionClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Partition", null,
def partitionClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Partition", null, null,
Array(
new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("table", "Table", Multiplicity.REQUIRED, false, null)
))
def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null,
def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null, null,
Array(
attrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("inputTables", DataTypes.arrayTypeName("Table"), Multiplicity.COLLECTION, false, null),
new AttributeDefinition("outputTable", "Table", Multiplicity.REQUIRED, false, null)
))
def viewClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "View", null,
def viewClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "View", null, null,
Array(
attrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("inputTables", DataTypes.arrayTypeName("Table"), Multiplicity.COLLECTION, false, null)
))
def dimTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Dimension", null,
def dimTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Dimension", null, null,
Array[AttributeDefinition]())
def piiTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "PII", null,
def piiTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "PII", null, null,
Array[AttributeDefinition]())
def metricTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Metric", null,
def metricTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Metric", null, null,
Array[AttributeDefinition]())
def etlTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "ETL", null,
def etlTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "ETL", null, null,
Array[AttributeDefinition]())
def jdbcTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "JdbcAccess", null,
def jdbcTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "JdbcAccess", null, null,
Array[AttributeDefinition]())
TypeSystem.getInstance().defineTypes(ImmutableList.of[EnumTypeDefinition],
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
......@@ -19,12 +19,23 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableSortedMap;
import org.apache.atlas.AtlasException;
import java.util.Map;
abstract class AbstractDataType<T> implements IDataType<T> {
public final String name;
public final String description;
public AbstractDataType(String name, String description) {
super();
this.name = name;
this.description = description;
}
protected T convertNull(Multiplicity m) throws AtlasException {
if (!m.nullAllowed()) {
throw new ValueConversionException.NullConversionException(m);
......@@ -52,5 +63,15 @@ abstract class AbstractDataType<T> implements IDataType<T> {
throw new TypeUpdateException(newType);
}
}
@Override
public String getName() {
return name;
}
@Override
public String getDescription() {
return description;
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
......@@ -21,6 +21,7 @@ package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct;
......@@ -47,17 +48,14 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
public final Map<AttributeInfo, List<String>> infoToNameMap;
/**
* Used when creating a ClassType, to support recursive Structs.
*/
ClassType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes, int numFields) {
super(typeSystem, ClassType.class, name, superTypes, numFields);
ClassType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTypes, int numFields) {
super(typeSystem, ClassType.class, name, description, superTypes, numFields);
infoToNameMap = null;
}
ClassType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes, AttributeInfo... fields)
ClassType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTypes, AttributeInfo... fields)
throws AtlasException {
super(typeSystem, ClassType.class, name, superTypes, fields);
super(typeSystem, ClassType.class, name, description, superTypes, fields);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
......@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -57,6 +58,7 @@ public class DataTypes {
static String MAP_TYPE_SUFFIX = ">";
public static String arrayTypeName(String elemTypeName) {
assert elemTypeName != null;
return String.format("%s%s%s", ARRAY_TYPE_PREFIX, elemTypeName, ARRAY_TYPE_SUFFIX);
}
......@@ -69,6 +71,8 @@ public class DataTypes {
}
public static String mapTypeName(IDataType keyType, IDataType valueType) {
assert keyType != null;
assert valueType != null;
return mapTypeName(keyType.getName(), valueType.getName());
}
......@@ -83,6 +87,10 @@ public class DataTypes {
}
public static abstract class PrimitiveType<T> extends AbstractDataType<T> {
public PrimitiveType(String name, String description) {
super(name, description);
}
@Override
public TypeCategory getTypeCategory() {
return TypeCategory.PRIMITIVE;
......@@ -113,11 +121,7 @@ public class DataTypes {
private static final String name = "boolean".intern();
private BooleanType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -146,11 +150,7 @@ public class DataTypes {
private static final String name = "byte".intern();
private ByteType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -186,11 +186,7 @@ public class DataTypes {
private static final String name = "short".intern();
private ShortType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -219,11 +215,7 @@ public class DataTypes {
private static final String name = "int".intern();
private IntType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -252,11 +244,7 @@ public class DataTypes {
private static final String name = "long".intern();
private LongType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -285,11 +273,7 @@ public class DataTypes {
private static final String name = "float".intern();
private FloatType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -318,11 +302,7 @@ public class DataTypes {
private static final String name = "double".intern();
private DoubleType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -351,11 +331,7 @@ public class DataTypes {
private static final String name = "biginteger".intern();
private BigIntegerType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -390,11 +366,7 @@ public class DataTypes {
private static final String name = "bigdecimal".intern();
private BigDecimalType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -429,11 +401,7 @@ public class DataTypes {
private static final String name = "date".intern();
private DateType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -472,11 +440,7 @@ public class DataTypes {
private static final String name = "string".intern();
private StringType() {
}
@Override
public String getName() {
return name;
super(name, null);
}
@Override
......@@ -493,13 +457,11 @@ public class DataTypes {
}
public static class ArrayType extends AbstractDataType<ImmutableCollection<?>> {
private final String nm;
private IDataType elemType;
public ArrayType(IDataType elemType) {
assert elemType != null;
super(arrayTypeName(elemType), null);
this.elemType = elemType;
this.nm = arrayTypeName(elemType);
}
public IDataType getElemType() {
......@@ -511,11 +473,6 @@ public class DataTypes {
}
@Override
public String getName() {
return nm;
}
@Override
public ImmutableCollection<?> convert(Object val, Multiplicity m) throws AtlasException {
if (val != null) {
Iterator it = null;
......@@ -589,16 +546,13 @@ public class DataTypes {
public static class MapType extends AbstractDataType<ImmutableMap<?, ?>> {
private final String nm;
private IDataType keyType;
private IDataType valueType;
public MapType(IDataType keyType, IDataType valueType) {
assert keyType != null;
assert valueType != null;
super(mapTypeName(keyType, valueType), null);
this.keyType = keyType;
this.valueType = valueType;
this.nm = mapTypeName(keyType, valueType);
}
public IDataType getKeyType() {
......@@ -618,11 +572,6 @@ public class DataTypes {
}
@Override
public String getName() {
return nm;
}
@Override
public ImmutableMap<?, ?> convert(Object val, Multiplicity m) throws AtlasException {
if (val != null) {
Iterator<Map.Entry> it = null;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
......@@ -29,13 +29,16 @@ import java.security.MessageDigest;
public class EnumType extends AbstractDataType<EnumValue> {
public final TypeSystem typeSystem;
public final String name;
public final ImmutableMap<String, EnumValue> valueMap;
public final ImmutableMap<Integer, EnumValue> ordinalMap;
protected EnumType(TypeSystem typeSystem, String name, EnumValue... values) {
this(typeSystem, name, null, values);
}
protected EnumType(TypeSystem typeSystem, String name, String description, EnumValue... values) {
super(name, description);
this.typeSystem = typeSystem;
this.name = name;
ImmutableMap.Builder<String, EnumValue> b1 = new ImmutableMap.Builder();
ImmutableMap.Builder<Integer, EnumValue> b2 = new ImmutableMap.Builder();
for (EnumValue v : values) {
......@@ -46,12 +49,6 @@ public class EnumType extends AbstractDataType<EnumValue> {
ordinalMap = b2.build();
}
@Override
public String getName() {
return name;
}
@Override
public EnumValue convert(Object val, Multiplicity m) throws AtlasException {
if (val != null) {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
......@@ -25,10 +25,16 @@ import java.util.Arrays;
public final class EnumTypeDefinition {
public final String name;
public final String description;
public final EnumValue[] enumValues;
public EnumTypeDefinition(String name, EnumValue... enumValues) {
this(name, null, enumValues);
}
public EnumTypeDefinition(String name, String description, EnumValue... enumValues) {
this.name = ParamChecker.notEmpty(name, "Enum type name");
this.description = description;
this.enumValues = ParamChecker.notNullElements(enumValues, "Enum values");
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
......@@ -49,7 +49,6 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
public final TypeSystem typeSystem;
public final Class<ST> superTypeClass;
public final String name;
public final FieldMapping fieldMapping;
public final int numFields;
public final ImmutableList<String> superTypes;
......@@ -58,14 +57,19 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
protected ImmutableMap<String, List<Path>> superTypePaths;
protected ImmutableMap<String, Path> pathNameToPathMap;
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableList<String> superTypes,
int numFields) {
this(typeSystem, superTypeClass, name, null, superTypes, numFields);
}
/**
* Used when creating a Type, to support recursive Structs.
*/
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableList<String> superTypes,
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableList<String> superTypes,
int numFields) {
super(name, description);
this.typeSystem = typeSystem;
this.superTypeClass = superTypeClass;
this.name = name;
this.fieldMapping = null;
this.numFields = numFields;
this.superTypes = superTypes;
......@@ -74,10 +78,14 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
}
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableList<String> superTypes,
AttributeInfo... fields) throws AtlasException {
this(typeSystem, superTypeClass, name, null, superTypes, fields);
}
HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableList<String> superTypes,
AttributeInfo... fields) throws AtlasException {
super(name, description);
this.typeSystem = typeSystem;
this.superTypeClass = superTypeClass;
this.name = name;
Pair<FieldMapping, ImmutableMap<String, String>> p = constructFieldMapping(superTypes, fields);
this.fieldMapping = p.left;
this.attributeNameToType = p.right;
......@@ -86,11 +94,6 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
this.immediateAttrs = ImmutableList.copyOf(fields);
}
@Override
public String getName() {
return name;
}
public FieldMapping fieldMapping() {
return fieldMapping;
}
......@@ -490,6 +493,4 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
throw new UnsupportedOperationException();
}
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java
......@@ -31,20 +31,21 @@ public class HierarchicalTypeDefinition<T extends HierarchicalType> extends Stru
* not intended public consumption
* @param hierarchicalMetaTypeName
* @param typeName
* @param typeDescription
* @param superTypes
* @param attributeDefinitions
* @throws ClassNotFoundException
*/
@InterfaceAudience.Private
public HierarchicalTypeDefinition(String hierarchicalMetaTypeName, String typeName, String[] superTypes,
public HierarchicalTypeDefinition(String hierarchicalMetaTypeName, String typeName, String typeDescription, String[] superTypes,
AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException {
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, ImmutableList.copyOf(superTypes),
this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, ImmutableList.copyOf(superTypes),
attributeDefinitions);
}
public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, ImmutableList<String> superTypes,
AttributeDefinition[] attributeDefinitions) {
super(typeName, false, attributeDefinitions);
public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, ImmutableList<String> superTypes,
AttributeDefinition[] attributeDefinitions) {
super(typeName, typeDescription, false, attributeDefinitions);
hierarchicalMetaTypeName = hierarchicalMetaType.getName();
this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes;
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
......@@ -34,5 +34,7 @@ public interface IDataType<T> {
void validateUpdate(IDataType newType) throws TypeUpdateException;
void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException;
String getDescription();
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
......@@ -18,10 +18,6 @@
package org.apache.atlas.typesystem.types;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.HashMap;
......@@ -29,31 +25,31 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
public class StructType extends AbstractDataType<IStruct> implements IConstructableType<IStruct, ITypedStruct> {
public final TypeSystem typeSystem;
public final String name;
public final FieldMapping fieldMapping;
public final Map<AttributeInfo, List<String>> infoToNameMap;
public final int numFields;
private final TypedStructHandler handler;
/**
* Used when creating a StructType, to support recursive Structs.
*/
protected StructType(TypeSystem typeSystem, String name, int numFields) {
protected StructType(TypeSystem typeSystem, String name, String description, int numFields) {
super(name, description);
this.typeSystem = typeSystem;
this.name = name;
this.fieldMapping = null;
infoToNameMap = null;
this.numFields = numFields;
this.handler = null;
}
protected StructType(TypeSystem typeSystem, String name, AttributeInfo... fields)
protected StructType(TypeSystem typeSystem, String name, String description, AttributeInfo... fields)
throws AtlasException {
super(name, description);
this.typeSystem = typeSystem;
this.name = name;
this.fieldMapping = constructFieldMapping(fields);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping);
this.numFields = this.fieldMapping.fields.size();
......@@ -64,11 +60,6 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
return fieldMapping;
}
@Override
public String getName() {
return name;
}
/**
* Validate that current definition can be updated with the new definition
* @param newType
......@@ -216,4 +207,5 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
......@@ -25,10 +25,16 @@ import java.util.Arrays;
public class StructTypeDefinition {
public final String typeName;
public final String typeDescription;//optional field
public final AttributeDefinition[] attributeDefinitions;
protected StructTypeDefinition(String typeName, boolean validate, AttributeDefinition... attributeDefinitions) {
this(typeName, null, validate, attributeDefinitions);
}
protected StructTypeDefinition(String typeName, String typeDescription, boolean validate, AttributeDefinition... attributeDefinitions) {
this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
this.typeDescription = typeDescription;
if (attributeDefinitions != null && attributeDefinitions.length != 0) {
ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
}
......@@ -36,7 +42,13 @@ public class StructTypeDefinition {
}
public StructTypeDefinition(String typeName, AttributeDefinition[] attributeDefinitions) {
this(typeName, null, attributeDefinitions);
}
public StructTypeDefinition(String typeName, String typeDescription,
AttributeDefinition[] attributeDefinitions) {
this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
this.typeDescription = typeDescription;
this.attributeDefinitions = ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
}
......@@ -57,7 +69,6 @@ public class StructTypeDefinition {
if (!typeName.equals(that.typeName)) {
return false;
}
return true;
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
......@@ -34,18 +34,15 @@ public class TraitType extends HierarchicalType<TraitType, IStruct>
public final Map<AttributeInfo, List<String>> infoToNameMap;
private final TypedStructHandler handler;
/**
* Used when creating a TraitType, to support recursive Structs.
*/
TraitType(TypeSystem typeSystem, String name, ImmutableList<String> superTraits, int numFields) {
super(typeSystem, TraitType.class, name, superTraits, numFields);
TraitType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTraits, int numFields) {
super(typeSystem, TraitType.class, name, description, superTraits, numFields);
handler = null;
infoToNameMap = null;
}
TraitType(TypeSystem typeSystem, String name, ImmutableList<String> superTraits, AttributeInfo... fields)
TraitType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTraits, AttributeInfo... fields)
throws AtlasException {
super(typeSystem, TraitType.class, name, superTraits, fields);
super(typeSystem, TraitType.class, name, description, superTraits, fields);
handler = new TypedStructHandler(this);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -168,7 +168,12 @@ public class TypeSystem {
public StructType defineStructType(String name, boolean errorIfExists, AttributeDefinition... attrDefs)
throws AtlasException {
StructTypeDefinition structDef = new StructTypeDefinition(name, attrDefs);
return defineStructType(name, null, errorIfExists, attrDefs);
}
public StructType defineStructType(String name, String description, boolean errorIfExists, AttributeDefinition... attrDefs)
throws AtlasException {
StructTypeDefinition structDef = new StructTypeDefinition(name, description, attrDefs);
defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(structDef),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
......@@ -192,7 +197,7 @@ public class TypeSystem {
infos[i] = new AttributeInfo(this, attrDefs[i], tempTypes);
}
return new StructType(this, name, infos);
return new StructType(this, name, null, infos);
}
public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef) throws AtlasException {
......@@ -272,13 +277,17 @@ public class TypeSystem {
return defineEnumType(new EnumTypeDefinition(name, values));
}
public EnumType defineEnumType(String name, String description, EnumValue... values) throws AtlasException {
return defineEnumType(new EnumTypeDefinition(name, description, values));
}
public EnumType defineEnumType(EnumTypeDefinition eDef) throws AtlasException {
assert eDef.name != null;
if (types.containsKey(eDef.name)) {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
}
EnumType eT = new EnumType(this, eDef.name, eDef.enumValues);
EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.enumValues);
types.put(eDef.name, eT);
typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ENUM, eDef.name);
return eT;
......@@ -348,7 +357,7 @@ public class TypeSystem {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
}
EnumType eT = new EnumType(this, eDef.name, eDef.enumValues);
EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.enumValues);
transientTypes.put(eDef.name, eT);
}
......@@ -357,7 +366,7 @@ public class TypeSystem {
if (!update && (transientTypes.containsKey(sDef.typeName) || types.containsKey(sDef.typeName))) {
throw new TypeExistsException(String.format("Cannot redefine type %s", sDef.typeName));
}
StructType sT = new StructType(this, sDef.typeName, sDef.attributeDefinitions.length);
StructType sT = new StructType(this, sDef.typeName, sDef.typeDescription, sDef.attributeDefinitions.length);
structNameToDefMap.put(sDef.typeName, sDef);
transientTypes.put(sDef.typeName, sT);
}
......@@ -368,7 +377,7 @@ public class TypeSystem {
(transientTypes.containsKey(traitDef.typeName) || types.containsKey(traitDef.typeName))) {
throw new TypeExistsException(String.format("Cannot redefine type %s", traitDef.typeName));
}
TraitType tT = new TraitType(this, traitDef.typeName, traitDef.superTypes,
TraitType tT = new TraitType(this, traitDef.typeName, traitDef.typeDescription, traitDef.superTypes,
traitDef.attributeDefinitions.length);
traitNameToDefMap.put(traitDef.typeName, traitDef);
transientTypes.put(traitDef.typeName, tT);
......@@ -381,7 +390,7 @@ public class TypeSystem {
throw new TypeExistsException(String.format("Cannot redefine type %s", classDef.typeName));
}
ClassType cT = new ClassType(this, classDef.typeName, classDef.superTypes,
ClassType cT = new ClassType(this, classDef.typeName, classDef.typeDescription, classDef.superTypes,
classDef.attributeDefinitions.length);
classNameToDefMap.put(classDef.typeName, classDef);
transientTypes.put(classDef.typeName, cT);
......@@ -471,7 +480,7 @@ public class TypeSystem {
infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
}
StructType type = new StructType(this, def.typeName, infos);
StructType type = new StructType(this, def.typeName, def.typeDescription, infos);
transientTypes.put(def.typeName, type);
return type;
}
......@@ -484,9 +493,9 @@ public class TypeSystem {
}
try {
Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, ImmutableList.class,
Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, ImmutableList.class,
AttributeInfo[].class);
U type = cons.newInstance(this, def.typeName, def.superTypes, infos);
U type = cons.newInstance(this, def.typeName, def.typeDescription, def.superTypes, infos);
transientTypes.put(def.typeName, type);
return type;
} catch (Exception e) {
......@@ -682,7 +691,7 @@ public class TypeSystem {
infos[0] = new AttributeInfo(TypeSystem.this, idAttr, null);
infos[1] = new AttributeInfo(TypeSystem.this, typNmAttr, null);
StructType type = new StructType(TypeSystem.this, TYP_NAME, infos);
StructType type = new StructType(TypeSystem.this, TYP_NAME, null, infos);
TypeSystem.this.types.put(TYP_NAME, type);
} catch (AtlasException me) {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
......@@ -65,16 +65,30 @@ public class TypesUtil {
public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name,
ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(TraitType.class, name, superTypes, attrDefs);
return createTraitTypeDef(name, null, superTypes, attrDefs);
}
public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description,
ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(TraitType.class, name, description, superTypes, attrDefs);
}
public static StructTypeDefinition createStructTypeDef(String name, AttributeDefinition... attrDefs) {
return new StructTypeDefinition(name, attrDefs);
return createStructTypeDef(name, null, attrDefs);
}
public static StructTypeDefinition createStructTypeDef(String name, String description, AttributeDefinition... attrDefs) {
return new StructTypeDefinition(name, description, attrDefs);
}
public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name,
ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(ClassType.class, name, superTypes, attrDefs);
return createClassTypeDef(name, null, superTypes, attrDefs);
}
public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, String description,
ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(ClassType.class, name, description, superTypes, attrDefs);
}
public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums,
......
typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala
......@@ -131,28 +131,28 @@ object TypesSerialization {
private def convertEnumTypeToEnumTypeDef(et: EnumType) = {
val eVals: Seq[EnumValue] = et.valueMap.values().toSeq
new EnumTypeDefinition(et.name, eVals: _*)
new EnumTypeDefinition(et.name, et.description, eVals: _*)
}
private def convertStructTypeToStructDef(st: StructType): StructTypeDefinition = {
val aDefs: Iterable[AttributeDefinition] =
st.fieldMapping.fields.values().map(convertAttributeInfoToAttributeDef(_))
new StructTypeDefinition(st.name, aDefs.toArray)
new StructTypeDefinition(st.name, st.description, aDefs.toArray)
}
private def convertTraitTypeToHierarchicalTypeDefintion(tt: TraitType): HierarchicalTypeDefinition[TraitType] = {
val aDefs: Iterable[AttributeDefinition] =
tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.superTypes, aDefs.toArray)
new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.description, tt.superTypes, aDefs.toArray)
}
private def convertClassTypeToHierarchicalTypeDefintion(tt: ClassType): HierarchicalTypeDefinition[ClassType] = {
val aDefs: Iterable[AttributeDefinition] =
tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.superTypes, aDefs.toArray)
new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.description, tt.superTypes, aDefs.toArray)
}
def convertToTypesDef(ts: TypeSystem, export: IDataType[_] => Boolean): TypesDef = {
......@@ -212,8 +212,13 @@ trait TypeHelpers {
def optionalAttr(name: String, dataType: IDataType[_]) =
new AttributeDefinition(name, dataType.getName, Multiplicity.OPTIONAL, false, null)
def structDef(name: String, attrs: AttributeDefinition*) = {
new StructTypeDefinition(name, attrs.toArray)
def structDef(name: String, attrs: AttributeDefinition*):
StructTypeDefinition = {
structDef(name, None, attrs:_*)
}
def structDef(name: String, description: Option[String], attrs: AttributeDefinition*) = {
new StructTypeDefinition(name, description.getOrElse(null), attrs.toArray)
}
def defineTraits(ts: TypeSystem, tDefs: HierarchicalTypeDefinition[TraitType]*) = {
......@@ -222,15 +227,25 @@ trait TypeHelpers {
def createTraitTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[TraitType] = {
createTraitTypeDef(name, None, superTypes, attrDefs:_*)
}
def createTraitTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[TraitType] = {
val sts = ImmutableList.copyOf(superTypes.toArray)
return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name,
return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name, description.getOrElse(null),
sts, attrDefs.toArray)
}
def createClassTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[ClassType] = {
createClassTypeDef( name, None, superTypes, attrDefs:_*)
}
def createClassTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*):
HierarchicalTypeDefinition[ClassType] = {
val sts = ImmutableList.copyOf(superTypes.toArray)
return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name,
return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name, description.getOrElse(null),
sts, attrDefs.toArray)
}
......
typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
......@@ -18,16 +18,18 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
public class ClassTest extends HierarchicalTypeTest<ClassType> {
@BeforeMethod
......@@ -72,15 +74,38 @@ public class ClassTest extends HierarchicalTypeTest<ClassType> {
"}");
}
@Test
public void testSerDeWithoutDescription() throws Exception {
HierarchicalTypeDefinition<ClassType> clsType = TypesUtil
.createClassTypeDef("Random", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE));
TypesDef typesDef = getTypesDef(clsType);
String json = TypesSerialization.toJson(typesDef);
System.out.println("json " + json);
TypesSerialization.fromJson(json);
}
@Test
public void testSerDeWithDescription() throws Exception {
HierarchicalTypeDefinition<ClassType> clsType = TypesUtil
.createClassTypeDef("Random", "Random-description", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE));
TypesDef typesDef = getTypesDef(clsType);
String json = TypesSerialization.toJson(typesDef);
System.out.println("json " + json);
TypesSerialization.fromJson(json);
}
@Override
protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, AttributeDefinition... attributes) {
return new HierarchicalTypeDefinition(ClassType.class, name, null, attributes);
return new HierarchicalTypeDefinition(ClassType.class, name, null, null, attributes);
}
@Override
protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, ImmutableList<String> superTypes,
AttributeDefinition... attributes) {
return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attributes);
return new HierarchicalTypeDefinition(ClassType.class, name, null, superTypes, attributes);
}
@Override
......
typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
......@@ -220,13 +220,13 @@ public class TraitTest extends HierarchicalTypeTest<TraitType> {
@Override
protected HierarchicalTypeDefinition<TraitType> getTypeDefinition(String name, AttributeDefinition... attributes) {
return new HierarchicalTypeDefinition(TraitType.class, name, null, attributes);
return new HierarchicalTypeDefinition(TraitType.class, name, null, null, attributes);
}
@Override
protected HierarchicalTypeDefinition<TraitType> getTypeDefinition(String name, ImmutableList<String> superTypes,
AttributeDefinition... attributes) {
return new HierarchicalTypeDefinition(TraitType.class, name, superTypes, attributes);
return new HierarchicalTypeDefinition(TraitType.class, name, null, superTypes, attributes);
}
@Override
......
typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java
......@@ -27,10 +27,8 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
import java.util.Collections;
import java.util.List;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
......@@ -56,6 +54,50 @@ public class TypeSystemTest extends BaseTest {
}
@Test
public void testGetTypeDescription() throws Exception {
String typeName = "enum_type";
String description = "_description";
String typeDescription = typeName + description;
getTypeSystem().defineEnumType(typeName, typeDescription, new EnumValue("0", 0), new EnumValue("1", 1), new EnumValue("2", 2),
new EnumValue("3", 3));
Assert.assertTrue(getTypeSystem().getTypeNames().contains(typeName));
IDataType type = getTypeSystem().getDataType(EnumType.class, typeName);
Assert.assertNotNull(type);
Assert.assertEquals(type.getDescription(), typeDescription);
typeName = "trait_type";
typeDescription = typeName + description;
HierarchicalTypeDefinition<TraitType> trait = TypesUtil
.createTraitTypeDef(typeName, typeDescription, ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
getTypeSystem().defineTraitType(trait);
Assert.assertTrue(getTypeSystem().getTypeNames().contains(typeName));
type = getTypeSystem().getDataType(TraitType.class, typeName);
Assert.assertNotNull(type);
Assert.assertEquals(type.getDescription(), typeDescription);
typeName = "class_type";
typeDescription = typeName + description;
HierarchicalTypeDefinition<ClassType> classType = TypesUtil
.createClassTypeDef(typeName, typeDescription, ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
getTypeSystem().defineClassType(classType);
Assert.assertTrue(getTypeSystem().getTypeNames().contains(typeName));
type = getTypeSystem().getDataType(ClassType.class, typeName);
Assert.assertNotNull(type);
Assert.assertEquals(type.getDescription(), typeDescription);
typeName = "struct_type";
typeDescription = typeName + description;
getTypeSystem().defineStructType(typeName, typeDescription, true, createRequiredAttrDef("a", DataTypes.INT_TYPE));
Assert.assertTrue(getTypeSystem().getTypeNames().contains(typeName));
type = getTypeSystem().getDataType(StructType.class, typeName);
Assert.assertNotNull(type);
Assert.assertEquals(type.getDescription(), typeDescription);
}
@Test
public void testIsRegistered() throws Exception {
getTypeSystem().defineEnumType("enum_test", new EnumValue("0", 0), new EnumValue("1", 1), new EnumValue("2", 2),
new EnumValue("3", 3));
......
typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
......@@ -86,6 +86,11 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
new EnumValue("DB", 1),
new EnumValue("TABLE", 2),
new EnumValue("PARTITION", 3))
ts.defineEnumType("TestType", "TestType-description",
new EnumValue("A", 1),
new EnumValue("B", 2),
new EnumValue("C", 3))
defineClassType(ts, createClassTypeDef("t4", List(),
requiredAttr("a", DataTypes.INT_TYPE),
......@@ -124,8 +129,12 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance", List(),
requiredAttr("level", DataTypes.INT_TYPE)
)
val securityClearanceTypeDefWithDesc: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance2", Some("SecurityClearance-Description"), List(),
requiredAttr("level", DataTypes.INT_TYPE)
)
ts.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.of[StructTypeDefinition],
ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef, securityClearanceTypeDefWithDesc),
ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef))
val ser = TypesSerialization.toJson(ts, _ => true)
......@@ -191,12 +200,32 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
val ser = TypesSerialization.toJson(sDef)
val typesDef2 = TypesSerialization.fromJson(ser)
Assert.assertEquals(sDef, typesDef2.structTypes(0))
//Now with description
val sDef2 = structDef("ts1", Some("ts1-description"), requiredAttr("a", DataTypes.INT_TYPE),
optionalAttr("b", DataTypes.BOOLEAN_TYPE),
optionalAttr("c", DataTypes.BYTE_TYPE),
optionalAttr("d", DataTypes.SHORT_TYPE),
optionalAttr("e", DataTypes.INT_TYPE),
optionalAttr("f", DataTypes.INT_TYPE),
optionalAttr("g", DataTypes.LONG_TYPE),
optionalAttr("h", DataTypes.FLOAT_TYPE),
optionalAttr("i", DataTypes.DOUBLE_TYPE),
optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
optionalAttr("l", DataTypes.DATE_TYPE),
optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
val ser2 = TypesSerialization.toJson(sDef)
val typesDef2 = TypesSerialization.fromJson(ser2)
val ser2 = TypesSerialization.toJson(sDef)
val typesDef3 = TypesSerialization.fromJson(ser2)
Assert.assertEquals(sDef, typesDef3.structTypes(0))
Assert.assertEquals(sDef, typesDef2.structTypes(0))
}
@Test def test4 : Unit = {
......@@ -212,8 +241,9 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
val D: HierarchicalTypeDefinition[TraitType] =
createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
val typDefs = Seq(A,B,C,D)
val E: HierarchicalTypeDefinition[TraitType] =
createTraitTypeDef("E", Some("E-description"), Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
val typDefs = Seq(A,B,C,D,E)
typDefs.foreach { tDef =>
val ser2 = TypesSerialization.toJson(tDef, true)
val typesDef2 = TypesSerialization.fromJson(ser2)
......@@ -245,7 +275,12 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
new EnumValue("TABLE", 2),
new EnumValue("PARTITION", 3))
val typDefs = Seq(e1,e2,e3,e4)
val e5 = new EnumTypeDefinition("LockLevel", "LockLevel-description",
new EnumValue("DB", 1),
new EnumValue("TABLE", 2),
new EnumValue("PARTITION", 3))
val typDefs = Seq(e1,e2,e3,e4,e5)
typDefs.foreach { tDef =>
val ser2 = TypesSerialization.toJson(tDef)
val typesDef2 = TypesSerialization.fromJson(ser2)
......@@ -290,12 +325,18 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
Multiplicity.COLLECTION, false, "manager")
)
val typDefs = Seq(typDef, deptTypeDef, personTypeDef, managerTypeDef)
val managerTypeDefWithDesc: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", Some("Manager-description"), List("Person"),
new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
Multiplicity.COLLECTION, false, "manager")
)
val typDefs = Seq(typDef, deptTypeDef, personTypeDef, managerTypeDef, managerTypeDefWithDesc)
typDefs.foreach { tDef =>
val ser2 = TypesSerialization.toJson(tDef, false)
val typesDef2 = TypesSerialization.fromJson(ser2)
Assert.assertEquals(tDef, typesDef2.classTypes(0))
}
}
}
webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
......@@ -128,22 +128,22 @@ public class QuickStart {
TypesDef createTypeDefinitions() throws Exception {
HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil
.createClassTypeDef(DATABASE_TYPE, null,
.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE, null,
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE),
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE));
HierarchicalTypeDefinition<ClassType> storageDescClsDef = TypesUtil
.createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE),
.createClassTypeDef(STORAGE_DESC_TYPE, STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE),
attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE),
attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null));
HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil
.createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
.createClassTypeDef(COLUMN_TYPE, COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil
.createClassTypeDef(TABLE_TYPE, ImmutableList.of("DataSet"),
.createClassTypeDef(TABLE_TYPE, TABLE_TYPE, ImmutableList.of("DataSet"),
new AttributeDefinition(DB_ATTRIBUTE, DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true, null),
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE),
......@@ -155,7 +155,7 @@ public class QuickStart {
Multiplicity.COLLECTION, true, null));
HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil
.createClassTypeDef(LOAD_PROCESS_TYPE, ImmutableList.of("Process"),
.createClassTypeDef(LOAD_PROCESS_TYPE, LOAD_PROCESS_TYPE, ImmutableList.of("Process"),
attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE),
attrDef("endTime", DataTypes.LONG_TYPE),
attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
......@@ -164,25 +164,25 @@ public class QuickStart {
attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED));
HierarchicalTypeDefinition<ClassType> viewClsDef = TypesUtil
.createClassTypeDef(VIEW_TYPE, null,
.createClassTypeDef(VIEW_TYPE, VIEW_TYPE, null,
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
new AttributeDefinition("inputTables", DataTypes.arrayTypeName(TABLE_TYPE),
Multiplicity.COLLECTION, false, null));
HierarchicalTypeDefinition<TraitType> dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null);
HierarchicalTypeDefinition<TraitType> dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", "Dimension Trait", null);
HierarchicalTypeDefinition<TraitType> factTraitDef = TypesUtil.createTraitTypeDef("Fact", null);
HierarchicalTypeDefinition<TraitType> factTraitDef = TypesUtil.createTraitTypeDef("Fact", "Fact Trait", null);
HierarchicalTypeDefinition<TraitType> piiTraitDef = TypesUtil.createTraitTypeDef("PII", null);
HierarchicalTypeDefinition<TraitType> piiTraitDef = TypesUtil.createTraitTypeDef("PII", "PII Trait", null);
HierarchicalTypeDefinition<TraitType> metricTraitDef = TypesUtil.createTraitTypeDef("Metric", null);
HierarchicalTypeDefinition<TraitType> metricTraitDef = TypesUtil.createTraitTypeDef("Metric", "Metric Trait", null);
HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL", null);
HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL", "ETL Trait", null);
HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null);
HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", "JdbcAccess Trait", null);
HierarchicalTypeDefinition<TraitType> logTraitDef = TypesUtil.createTraitTypeDef("Log Data", null);
HierarchicalTypeDefinition<TraitType> logTraitDef = TypesUtil.createTraitTypeDef("Log Data", "LogData Trait", null);
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef, logTraitDef),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment