Skip to content

A
atlas
Commit 6b0f7d82 by Suma Shivaprasad
Options

ATLAS-553 Entity mutation - Fix issue with reordering of elements in…

ATLAS-553 Entity mutation - Fix issue with reordering of elements in array<class> with composite references ( sumasai via shwethags)
parent 0fc20719
Showing with 285 additions and 150 deletions
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
...@@ -22,7 +22,6 @@ import com.google.common.base.Joiner; ...@@ -22,7 +22,6 @@ import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import org.apache.atlas.ApplicationProperties; import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hive.model.HiveDataTypes;
...@@ -365,14 +364,16 @@ public class HiveHookIT { ...@@ -365,14 +364,16 @@ public class HiveHookIT {
@Test @Test
public void testAlterTableDropColumn() throws Exception { public void testAlterTableDropColumn() throws Exception {
String tableName = createTable(); String tableName = createTable();
final String colDropped = "name"; final String colDropped = "id";
String query = "alter table " + tableName + " replace columns (id int)"; String query = "alter table " + tableName + " replace columns (name string)";
runCommand(query); runCommand(query);
assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), colDropped)); assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), colDropped));
//Verify the number of columns present in the table //Verify the number of columns present in the table
final List<Referenceable> columns = getColumns(DEFAULT_DB, tableName); final List<Referenceable> columns = getColumns(DEFAULT_DB, tableName);
Assert.assertEquals(columns.size(), 1); Assert.assertEquals(columns.size(), 1);
Assert.assertEquals(columns.get(0).get(HiveDataModelGenerator.NAME), "name");
} }
@Test @Test
...@@ -420,6 +421,39 @@ public class HiveHookIT { ...@@ -420,6 +421,39 @@ public class HiveHookIT {
assertColumnIsRegistered(newColQualifiedName); assertColumnIsRegistered(newColQualifiedName);
Assert.assertEquals(columns.get(1).get(HiveDataModelGenerator.COMMENT), comment); Assert.assertEquals(columns.get(1).get(HiveDataModelGenerator.COMMENT), comment);
//Change column position
oldColName = "name3";
newColName = "name4";
query = String.format("alter table %s change column %s %s %s first", tableName, oldColName, newColName, newColType);
runCommand(query);
columns = getColumns(DEFAULT_DB, tableName);
Assert.assertEquals(columns.size(), 2);
assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), oldColName));
newColQualifiedName = HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), newColName);
assertColumnIsRegistered(newColQualifiedName);
//Change col position again
Assert.assertEquals(columns.get(0).get(HiveDataModelGenerator.NAME), newColName);
Assert.assertEquals(columns.get(1).get(HiveDataModelGenerator.NAME), "id");
oldColName = "name4";
newColName = "name5";
query = String.format("alter table %s change column %s %s %s after id", tableName, oldColName, newColName, newColType);
runCommand(query);
columns = getColumns(DEFAULT_DB, tableName);
Assert.assertEquals(columns.size(), 2);
assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), oldColName));
newColQualifiedName = HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), newColName);
assertColumnIsRegistered(newColQualifiedName);
//Check col position
Assert.assertEquals(columns.get(1).get(HiveDataModelGenerator.NAME), newColName);
Assert.assertEquals(columns.get(0).get(HiveDataModelGenerator.NAME), "id");
} }
@Test @Test
......
release-log.txt
...@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ...@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES: ALL CHANGES:
ATLAS-553 Entity mutation - Fix issue with reordering of elements in array<class> with composite references (sumasai via shwethags)
ATLAS-513 Admin support for HA (yhemanth via sumasai) ATLAS-513 Admin support for HA (yhemanth via sumasai)
ATLAS-511 Ability to run multiple instances of Atlas Server with automatic failover to one active server (yhemanth via shwethags) ATLAS-511 Ability to run multiple instances of Atlas Server with automatic failover to one active server (yhemanth via shwethags)
ATLAS-577 Integrate entity audit with DefaultMetadataService (shwethags) ATLAS-577 Integrate entity audit with DefaultMetadataService (shwethags)
......
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
...@@ -57,14 +57,18 @@ import java.util.Collection; ...@@ -57,14 +57,18 @@ import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
public final class TypedInstanceToGraphMapper { public final class TypedInstanceToGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class); private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class);
private final Map<Id, Vertex> idToVertexMap = new HashMap<>(); private final Map<Id, Vertex> idToVertexMap = new HashMap<>();
//Maintains a set of Guid based Ids that are referenced/created during graph walk
private final Set<Id> referencedIds = new HashSet<>();
private final TypeSystem typeSystem = TypeSystem.getInstance(); private final TypeSystem typeSystem = TypeSystem.getInstance();
private final List<String> deletedEntityGuids = new ArrayList<>(); private final List<String> deletedEntityGuids = new ArrayList<>();
private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>(); private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>();
...@@ -301,10 +305,13 @@ public final class TypedInstanceToGraphMapper { ...@@ -301,10 +305,13 @@ public final class TypedInstanceToGraphMapper {
newInstance = (ITypedReferenceableInstance) instance; newInstance = (ITypedReferenceableInstance) instance;
instancesToUpdate.add(newInstance); instancesToUpdate.add(newInstance);
} }
} }
//Set the id in the new instance //Set the id in the new instance
idToVertexMap.put(id, instanceVertex); idToVertexMap.put(id, instanceVertex);
referencedIds.add(GraphHelper.getIdFromVertex(instance.getTypeName(), instanceVertex));
} }
} }
return TypeUtils.Pair.of(instancesToCreate, instancesToUpdate); return TypeUtils.Pair.of(instancesToCreate, instancesToUpdate);
...@@ -395,7 +402,14 @@ public final class TypedInstanceToGraphMapper { ...@@ -395,7 +402,14 @@ public final class TypedInstanceToGraphMapper {
if (currentEntries != null) { if (currentEntries != null) {
if (index < currentEntries.size()) { if (index < currentEntries.size()) {
for (; index < currentEntries.size(); index++) { for (; index < currentEntries.size(); index++) {
removeUnusedReference(currentEntries.get(index), attributeInfo, elementType); if (elementType.getTypeCategory() == TypeCategory.CLASS) {
final String edgeId = currentEntries.get(index);
final Pair<Edge, Vertex> edgeAndTargetVertex = GraphHelper.getInstance().getEdgeAndTargetVertex(edgeId);
Id guid = GraphHelper.getIdFromVertex(elementType.getName(), edgeAndTargetVertex.right);
removeUnusedClassReference(edgeId, attributeInfo, elementType, !referencedIds.contains(guid));
} else if (elementType.getTypeCategory() == TypeCategory.STRUCT) {
removeUnusedStructReference(currentEntries.get(index), attributeInfo, elementType);
}
} }
} }
} }
...@@ -403,8 +417,12 @@ public final class TypedInstanceToGraphMapper { ...@@ -403,8 +417,12 @@ public final class TypedInstanceToGraphMapper {
else if (operation == Operation.UPDATE_FULL || operation == Operation.DELETE) { else if (operation == Operation.UPDATE_FULL || operation == Operation.DELETE) {
// Clear all existing entries // Clear all existing entries
if (currentEntries != null) { if (currentEntries != null) {
for (String edgeId : currentEntries) { for (String entry : currentEntries) {
removeUnusedReference(edgeId, attributeInfo, elementType); if (elementType.getTypeCategory() == TypeCategory.CLASS) {
removeUnusedClassReference(entry, attributeInfo, elementType, true);
} else if(elementType.getTypeCategory() == TypeCategory.STRUCT) {
removeUnusedStructReference(entry, attributeInfo, elementType);
}
} }
} }
} }
...@@ -451,8 +469,13 @@ public final class TypedInstanceToGraphMapper { ...@@ -451,8 +469,13 @@ public final class TypedInstanceToGraphMapper {
String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo) + "." + unusedKey; String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo) + "." + unusedKey;
if (instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().hasNext()) { if (instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().hasNext()) {
Edge edge = instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().next(); Edge edge = instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().next();
removeUnusedReference(edge.getId().toString(), attributeInfo, if (TypeCategory.STRUCT.equals(((DataTypes.MapType) attributeInfo.dataType()).getValueType().getTypeCategory())) {
((DataTypes.MapType) attributeInfo.dataType()).getValueType()); removeUnusedStructReference(edge.getId().toString(), attributeInfo,
((DataTypes.MapType) attributeInfo.dataType()).getValueType());
} else if(TypeCategory.CLASS.equals(((DataTypes.MapType) attributeInfo.dataType()).getValueType().getTypeCategory())){
final Vertex targetVertex = edge.getVertex(Direction.OUT);
Id guid = GraphHelper.getIdFromVertex(elementType.getName(), targetVertex);
removeUnusedClassReference(edge.getId().toString(), attributeInfo, elementType, !referencedIds.contains(guid)); }
} }
} }
} }
...@@ -499,7 +522,7 @@ public final class TypedInstanceToGraphMapper { ...@@ -499,7 +522,7 @@ public final class TypedInstanceToGraphMapper {
TypeUtils.Pair<Vertex, Edge> vertexEdgePair = null; TypeUtils.Pair<Vertex, Edge> vertexEdgePair = null;
if (curVal != null && structAttr == null) { if (curVal != null && structAttr == null) {
//remove edge //remove edge
removeUnusedReference(curVal, attributeInfo, elementType); removeUnusedStructReference(curVal, attributeInfo, elementType);
} else if (curVal != null && structAttr != null) { } else if (curVal != null && structAttr != null) {
//update //update
Edge edge = graphHelper.getOutGoingEdgeById(curVal); Edge edge = graphHelper.getOutGoingEdgeById(curVal);
...@@ -524,7 +547,7 @@ public final class TypedInstanceToGraphMapper { ...@@ -524,7 +547,7 @@ public final class TypedInstanceToGraphMapper {
TypeUtils.Pair<Vertex, Edge> vertexEdgePair = null; TypeUtils.Pair<Vertex, Edge> vertexEdgePair = null;
if (curVal != null && newVal == null) { if (curVal != null && newVal == null) {
//remove edge //remove edge
removeUnusedReference(curVal, attributeInfo, elementType); removeUnusedClassReference(curVal, attributeInfo, elementType, true);
} else if (curVal != null && newVal != null) { } else if (curVal != null && newVal != null) {
Edge edge = graphHelper.getOutGoingEdgeById(curVal); Edge edge = graphHelper.getOutGoingEdgeById(curVal);
Id classRefId = getId(newVal); Id classRefId = getId(newVal);
...@@ -586,14 +609,17 @@ public final class TypedInstanceToGraphMapper { ...@@ -586,14 +609,17 @@ public final class TypedInstanceToGraphMapper {
String currentGUID = invertex.getProperty(Constants.GUID_PROPERTY_KEY); String currentGUID = invertex.getProperty(Constants.GUID_PROPERTY_KEY);
Id currentId = new Id(currentGUID, 0, (String) invertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY)); Id currentId = new Id(currentGUID, 0, (String) invertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
if (!currentId.equals(id)) { if (!currentId.equals(id)) {
// add an edge to the class vertex from the instance // add an edge to the class vertex from the instance
if(toVertex != null) { if (toVertex != null) {
newEdge = graphHelper.addEdge(instanceVertex, toVertex, edgeLabel); newEdge = graphHelper.addEdge(instanceVertex, toVertex, edgeLabel);
result = TypeUtils.Pair.of(toVertex, newEdge); result = TypeUtils.Pair.of(toVertex, newEdge);
} }
removeUnusedReference(edge.getId().toString(), attributeInfo, dataType);
//Delete vertex only if the IdtoVertex map doesnt have it in future references
removeUnusedClassReference(edge.getId().toString(), attributeInfo, dataType, !referencedIds.contains(currentId));
} }
if (attributeInfo.isComposite) { if (attributeInfo.isComposite) {
//Update the attributes also if composite //Update the attributes also if composite
if (typedInstance.fieldMapping() != null) { if (typedInstance.fieldMapping() != null) {
...@@ -670,59 +696,62 @@ public final class TypedInstanceToGraphMapper { ...@@ -670,59 +696,62 @@ public final class TypedInstanceToGraphMapper {
GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue); GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
} }
private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) throws AtlasException { private Edge removeUnusedClassReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType, boolean deleteReferredVertex) throws AtlasException {
TypeCategory typeCategory = elementType.getTypeCategory();
if (typeCategory != TypeCategory.STRUCT && elementType.getTypeCategory() != TypeCategory.CLASS) {
// Only class and struct references have edges.
return null;
}
// Remove edge to disconnect struct or class reference. // Remove edge to disconnect struct or class reference.
// For struct or composite class reference, also delete the target instance. // For struct or composite class reference, also delete the target instance.
Edge removedRelation = null; Edge removedRelation = null;
TypeUtils.Pair<Edge, Vertex> edgeAndVertex = graphHelper.getEdgeAndTargetVertex(edgeId); TypeUtils.Pair<Edge, Vertex> edgeAndVertex = graphHelper.getEdgeAndTargetVertex(edgeId);
if (typeCategory == TypeCategory.STRUCT) { if (attributeInfo.isComposite) {
graphHelper.removeEdge(edgeAndVertex.left); // For uni-directional reference, remove the edge.
removedRelation = edgeAndVertex.left; // For bi-directional reference, the edges are removed
// when the composite entity is deleted.
if (attributeInfo.reverseAttributeName == null) {
graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left;
}
// Create an empty instance to use for clearing all struct attributes. // Delete the contained entity.
StructType structType = (StructType) elementType; if (deleteReferredVertex) {
ITypedStruct typedInstance = structType.createInstance();
// Delete target vertex and any underlying structs and composite entities owned by this struct.
mapInstanceToVertex(typedInstance, edgeAndVertex.right, structType.fieldMapping().fields, false, Operation.DELETE);
}
else {
// Class reference
if (attributeInfo.isComposite) {
// For uni-directional reference, remove the edge.
// For bi-directional reference, the edges are removed
// when the composite entity is deleted.
if (attributeInfo.reverseAttributeName == null) {
graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left;
}
// Delete the contained entity.
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
Vertex sourceVertex = edgeAndVertex.left.getVertex(Direction.OUT); Vertex sourceVertex = edgeAndVertex.left.getVertex(Direction.OUT);
String sourceTypeName = GraphHelper.getTypeName(sourceVertex); String sourceTypeName = GraphHelper.getTypeName(sourceVertex);
LOG.debug("Deleting composite entity {}:{} contained by {}:{} through reference {}", LOG.debug("Deleting composite entity {}:{} contained by {}:{} through reference {}",
elementType.getName(), GraphHelper.getIdFromVertex(elementType.getName(), edgeAndVertex.right)._getId(), elementType.getName(), GraphHelper.getIdFromVertex(elementType.getName(), edgeAndVertex.right)._getId(),
sourceTypeName, GraphHelper.getIdFromVertex(sourceTypeName, sourceVertex)._getId(), sourceTypeName, GraphHelper.getIdFromVertex(sourceTypeName, sourceVertex)._getId(),
attributeInfo.name); attributeInfo.name);
} }
deleteEntity(elementType.getName(), edgeAndVertex.right); deleteEntity(elementType.getName(), edgeAndVertex.right);
} }
else { }
if (attributeInfo.reverseAttributeName != null) { else {
// Disconnect both ends of the bi-directional reference if (attributeInfo.reverseAttributeName != null) {
removeReverseReference(edgeAndVertex, attributeInfo); // Disconnect both ends of the bi-directional reference
} removeReverseReference(edgeAndVertex, attributeInfo);
graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left;
} }
graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left;
return removedRelation;
} }
return removedRelation; return removedRelation;
}
private Edge removeUnusedStructReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) throws AtlasException {
// Remove edge to disconnect struct or class reference.
// For struct or composite class reference, also delete the target instance.
Edge removedRelation = null;
TypeUtils.Pair<Edge, Vertex> edgeAndVertex = graphHelper.getEdgeAndTargetVertex(edgeId);
graphHelper.removeEdge(edgeAndVertex.left);
removedRelation = edgeAndVertex.left;
// Create an empty instance to use for clearing all struct attributes.
StructType structType = (StructType) elementType;
ITypedStruct typedInstance = structType.createInstance();
// Delete target vertex and any underlying structs and composite entities owned by this struct.
mapInstanceToVertex(typedInstance, edgeAndVertex.right, structType.fieldMapping().fields, false, Operation.DELETE);
return removedRelation;
} }
/** /**
......
repository/src/test/java/org/apache/atlas/TestUtils.java
...@@ -199,6 +199,7 @@ public final class TestUtils { ...@@ -199,6 +199,7 @@ public final class TestUtils {
public static final String PARTITION_STRUCT_TYPE = "partition_struct_type"; public static final String PARTITION_STRUCT_TYPE = "partition_struct_type";
public static final String PARTITION_CLASS_TYPE = "partition_class_type"; public static final String PARTITION_CLASS_TYPE = "partition_class_type";
public static final String SERDE_TYPE = "serdeType"; public static final String SERDE_TYPE = "serdeType";
public static final String COLUMNS_MAP = "columnsMap";
public static TypesDef defineHiveTypes() { public static TypesDef defineHiveTypes() {
String _description = "_description"; String _description = "_description";
...@@ -290,7 +291,7 @@ public final class TestUtils { ...@@ -290,7 +291,7 @@ public final class TestUtils {
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, true, null), Multiplicity.OPTIONAL, true, null),
//map of classes - //map of classes -
new AttributeDefinition("columnsMap", new AttributeDefinition(COLUMNS_MAP,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
COLUMN_TYPE), COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null), Multiplicity.OPTIONAL, true, null),
......
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
...@@ -94,6 +94,9 @@ public class DefaultMetadataServiceTest { ...@@ -94,6 +94,9 @@ public class DefaultMetadataServiceTest {
private Referenceable table; private Referenceable table;
private Id tableId; private Id tableId;
private final String NAME = "name";
private final String COLUMNS_ATTR_NAME = "columns";
@BeforeTest @BeforeTest
...@@ -116,7 +119,7 @@ public class DefaultMetadataServiceTest { ...@@ -116,7 +119,7 @@ public class DefaultMetadataServiceTest {
table = createTableEntity(dbId); table = createTableEntity(dbId);
String tableGuid = createInstance(table); String tableGuid = createInstance(table);
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
table = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); table = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
tableId = new Id(tableGuid, 0, TestUtils.TABLE_TYPE); tableId = new Id(tableGuid, 0, TestUtils.TABLE_TYPE);
} }
...@@ -167,7 +170,7 @@ public class DefaultMetadataServiceTest { ...@@ -167,7 +170,7 @@ public class DefaultMetadataServiceTest {
private Referenceable createDBEntity() { private Referenceable createDBEntity() {
Referenceable entity = new Referenceable(TestUtils.DATABASE_TYPE); Referenceable entity = new Referenceable(TestUtils.DATABASE_TYPE);
String dbName = RandomStringUtils.randomAlphanumeric(10); String dbName = RandomStringUtils.randomAlphanumeric(10);
entity.set("name", dbName); entity.set(NAME, dbName);
entity.set("description", "us db"); entity.set("description", "us db");
return entity; return entity;
} }
...@@ -175,7 +178,7 @@ public class DefaultMetadataServiceTest { ...@@ -175,7 +178,7 @@ public class DefaultMetadataServiceTest {
private Referenceable createTableEntity(Id dbId) { private Referenceable createTableEntity(Id dbId) {
Referenceable entity = new Referenceable(TestUtils.TABLE_TYPE); Referenceable entity = new Referenceable(TestUtils.TABLE_TYPE);
String tableName = RandomStringUtils.randomAlphanumeric(10); String tableName = RandomStringUtils.randomAlphanumeric(10);
entity.set("name", tableName); entity.set(NAME, tableName);
entity.set("description", "random table"); entity.set("description", "random table");
entity.set("type", "type"); entity.set("type", "type");
entity.set("tableType", "MANAGED"); entity.set("tableType", "MANAGED");
...@@ -186,7 +189,7 @@ public class DefaultMetadataServiceTest { ...@@ -186,7 +189,7 @@ public class DefaultMetadataServiceTest {
private Referenceable createColumnEntity() { private Referenceable createColumnEntity() {
Referenceable entity = new Referenceable(TestUtils.COLUMN_TYPE); Referenceable entity = new Referenceable(TestUtils.COLUMN_TYPE);
entity.set("name", RandomStringUtils.randomAlphanumeric(10)); entity.set(NAME, RandomStringUtils.randomAlphanumeric(10));
entity.set("type", "VARCHAR(32)"); entity.set("type", "VARCHAR(32)");
return entity; return entity;
} }
...@@ -195,7 +198,7 @@ public class DefaultMetadataServiceTest { ...@@ -195,7 +198,7 @@ public class DefaultMetadataServiceTest {
public void testCreateEntityWithUnknownDatatype() throws Exception { public void testCreateEntityWithUnknownDatatype() throws Exception {
Referenceable entity = new Referenceable("Unknown datatype"); Referenceable entity = new Referenceable("Unknown datatype");
String dbName = RandomStringUtils.randomAlphanumeric(10); String dbName = RandomStringUtils.randomAlphanumeric(10);
entity.set("name", dbName); entity.set(NAME, dbName);
entity.set("description", "us db"); entity.set("description", "us db");
createInstance(entity); createInstance(entity);
Assert.fail(TypeNotFoundException.class.getSimpleName() + " was expected but none thrown."); Assert.fail(TypeNotFoundException.class.getSimpleName() + " was expected but none thrown.");
...@@ -213,7 +216,7 @@ public class DefaultMetadataServiceTest { ...@@ -213,7 +216,7 @@ public class DefaultMetadataServiceTest {
assertNull(newId); assertNull(newId);
//Same entity, but different qualified name should succeed //Same entity, but different qualified name should succeed
entity.set("name", TestUtils.randomString()); entity.set(NAME, TestUtils.randomString());
newId = createInstance(entity); newId = createInstance(entity);
Assert.assertNotEquals(newId, id); Assert.assertNotEquals(newId, id);
} }
...@@ -253,7 +256,7 @@ public class DefaultMetadataServiceTest { ...@@ -253,7 +256,7 @@ public class DefaultMetadataServiceTest {
String dbId = createInstance(db); String dbId = createInstance(db);
Referenceable table = new Referenceable(TestUtils.TABLE_TYPE); Referenceable table = new Referenceable(TestUtils.TABLE_TYPE);
table.set("name", TestUtils.randomString()); table.set(NAME, TestUtils.randomString());
table.set("description", "random table"); table.set("description", "random table");
table.set("type", "type"); table.set("type", "type");
table.set("tableType", "MANAGED"); table.set("tableType", "MANAGED");
...@@ -263,7 +266,7 @@ public class DefaultMetadataServiceTest { ...@@ -263,7 +266,7 @@ public class DefaultMetadataServiceTest {
//table create should re-use the db instance created earlier //table create should re-use the db instance created earlier
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Referenceable actualDb = (Referenceable) tableDefinition.get("databaseComposite"); Referenceable actualDb = (Referenceable) tableDefinition.get("databaseComposite");
Assert.assertEquals(actualDb.getId().id, dbId); Assert.assertEquals(actualDb.getId().id, dbId);
...@@ -275,10 +278,10 @@ public class DefaultMetadataServiceTest { ...@@ -275,10 +278,10 @@ public class DefaultMetadataServiceTest {
Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{ Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("columnNames", colNameList); put("columnNames", colNameList);
}}); }});
metadataService.updateEntityByUniqueAttribute(table.getTypeName(), "name", (String) table.get("name"), tableUpdated); metadataService.updateEntityByUniqueAttribute(table.getTypeName(), NAME, (String) table.get(NAME), tableUpdated);
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
List<String> actualColumns = (List) tableDefinition.get("columnNames"); List<String> actualColumns = (List) tableDefinition.get("columnNames");
Assert.assertEquals(actualColumns, colNameList); Assert.assertEquals(actualColumns, colNameList);
...@@ -290,14 +293,14 @@ public class DefaultMetadataServiceTest { ...@@ -290,14 +293,14 @@ public class DefaultMetadataServiceTest {
final Map<String, Struct> partsMap = new HashMap<>(); final Map<String, Struct> partsMap = new HashMap<>();
partsMap.put("part0", new Struct(TestUtils.PARTITION_STRUCT_TYPE, partsMap.put("part0", new Struct(TestUtils.PARTITION_STRUCT_TYPE,
new HashMap<String, Object>() {{ new HashMap<String, Object>() {{
put("name", "test"); put(NAME, "test");
}})); }}));
table.set("partitionsMap", partsMap); table.set("partitionsMap", partsMap);
updateInstance(table); updateInstance(table);
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertTrue(partsMap.get("part0").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"))); Assert.assertTrue(partsMap.get("part0").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0")));
...@@ -305,13 +308,13 @@ public class DefaultMetadataServiceTest { ...@@ -305,13 +308,13 @@ public class DefaultMetadataServiceTest {
//update map - add a map key //update map - add a map key
partsMap.put("part1", new Struct(TestUtils.PARTITION_STRUCT_TYPE, partsMap.put("part1", new Struct(TestUtils.PARTITION_STRUCT_TYPE,
new HashMap<String, Object>() {{ new HashMap<String, Object>() {{
put("name", "test1"); put(NAME, "test1");
}})); }}));
table.set("partitionsMap", partsMap); table.set("partitionsMap", partsMap);
updateInstance(table); updateInstance(table);
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
...@@ -321,31 +324,88 @@ public class DefaultMetadataServiceTest { ...@@ -321,31 +324,88 @@ public class DefaultMetadataServiceTest {
partsMap.remove("part0"); partsMap.remove("part0");
partsMap.put("part2", new Struct(TestUtils.PARTITION_STRUCT_TYPE, partsMap.put("part2", new Struct(TestUtils.PARTITION_STRUCT_TYPE,
new HashMap<String, Object>() {{ new HashMap<String, Object>() {{
put("name", "test2"); put(NAME, "test2");
}})); }}));
table.set("partitionsMap", partsMap); table.set("partitionsMap", partsMap);
updateInstance(table); updateInstance(table);
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0")); Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"));
Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2"))); Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2")));
//update struct value for existing map key //update struct value for existing map key
Struct partition2 = (Struct)partsMap.get("part2"); Struct partition2 = (Struct)partsMap.get("part2");
partition2.set("name", "test2Updated"); partition2.set(NAME, "test2Updated");
updateInstance(table); updateInstance(table);
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2); Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0")); Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"));
Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2"))); Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2")));
//Test map pointing to a class
final Map<String, Struct> columnsMap = new HashMap<>();
Referenceable col0Type = new Referenceable(TestUtils.COLUMN_TYPE,
new HashMap<String, Object>() {{
put(NAME, "test1");
put("type", "string");
}});
columnsMap.put("col0", col0Type);
Referenceable col1Type = new Referenceable(TestUtils.COLUMN_TYPE,
new HashMap<String, Object>() {{
put(NAME, "test2");
put("type", "string");
}});
columnsMap.put("col1", col1Type);
table.set(TestUtils.COLUMNS_MAP, columnsMap);
updateInstance(table);
verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columnsMap, TestUtils.COLUMNS_MAP);
//Swap elements
columnsMap.clear();
columnsMap.put("col0", col1Type);
columnsMap.put("col1", col0Type);
table.set(TestUtils.COLUMNS_MAP, columnsMap);
updateInstance(table);
verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columnsMap, TestUtils.COLUMNS_MAP);
//Drop the first key and change the class type as well to col0
columnsMap.clear();
columnsMap.put("col0", col0Type);
table.set(TestUtils.COLUMNS_MAP, columnsMap);
updateInstance(table);
verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columnsMap, TestUtils.COLUMNS_MAP);
//Clear state
table.setNull(TestUtils.COLUMNS_MAP);
updateInstance(table);
verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), null, TestUtils.COLUMNS_MAP);
}
private void verifyMapUpdates(String typeName, String uniqAttrName, String uniqAttrValue, Map<String, Struct> expectedMap, String mapAttrName) throws AtlasException {
String json =
metadataService.getEntityDefinition(typeName, uniqAttrName, uniqAttrValue);
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(json, true);
if(expectedMap == null) {
Assert.assertNull(tableDefinition.get(TestUtils.COLUMNS_MAP));
} else {
Assert.assertEquals(((Map<String, Referenceable>)tableDefinition.get(mapAttrName)).size(), expectedMap.size());
for (String key : expectedMap.keySet()) {
Assert.assertTrue(((Map<String, Referenceable>) tableDefinition.get(mapAttrName)).get(key).equalsContents(expectedMap.get(key)));
}
}
} }
@Test @Test
...@@ -359,7 +419,7 @@ public class DefaultMetadataServiceTest { ...@@ -359,7 +419,7 @@ public class DefaultMetadataServiceTest {
metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated); metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
List<String> actualColumns = (List) tableDefinition.get("columnNames"); List<String> actualColumns = (List) tableDefinition.get("columnNames");
Assert.assertEquals(actualColumns, colNameList); Assert.assertEquals(actualColumns, colNameList);
...@@ -372,7 +432,7 @@ public class DefaultMetadataServiceTest { ...@@ -372,7 +432,7 @@ public class DefaultMetadataServiceTest {
metadataService.updateEntityPartialByGuid(tableId.getId()._getId(), tableUpdated); metadataService.updateEntityPartialByGuid(tableId.getId()._getId(), tableUpdated);
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
actualColumns = (List) tableDefinition.get("columnNames"); actualColumns = (List) tableDefinition.get("columnNames");
Assert.assertEquals(actualColumns, updatedColNameList); Assert.assertEquals(actualColumns, updatedColNameList);
...@@ -380,29 +440,26 @@ public class DefaultMetadataServiceTest { ...@@ -380,29 +440,26 @@ public class DefaultMetadataServiceTest {
@Test @Test
public void testUpdateEntityArrayOfClass() throws Exception { public void testUpdateEntityArrayOfClass() throws Exception {
//test array of class with id //test array of class with id
final List<Referenceable> columns = new ArrayList<>(); final List<Referenceable> columns = new ArrayList<>();
Map<String, Object> values = new HashMap<>(); Map<String, Object> values = new HashMap<>();
values.put("name", "col1"); values.put(NAME, "col1");
values.put("type", "type"); values.put("type", "type");
Referenceable col1 = new Referenceable("column_type", values); Referenceable col1 = new Referenceable(TestUtils.COLUMN_TYPE, values);
columns.add(col1); columns.add(col1);
Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{ Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("columns", columns); put(COLUMNS_ATTR_NAME, columns);
}}); }});
metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated); metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
String tableDefinitionJson = verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
final List<Referenceable> arrClsColumns = (List) tableDefinition.get("columns");
assertReferenceables(arrClsColumns.get(0), columns.get(0));
//Partial update. Add col5 But also update col1 //Partial update. Add col5 But also update col1
Map<String, Object> valuesCol5 = new HashMap<>(); Map<String, Object> valuesCol5 = new HashMap<>();
valuesCol5.put("name", "col5"); valuesCol5.put(NAME, "col5");
valuesCol5.put("type", "type"); valuesCol5.put("type", "type");
Referenceable col2 = new Referenceable("column_type", valuesCol5); Referenceable col2 = new Referenceable(TestUtils.COLUMN_TYPE, valuesCol5);
//update col1 //update col1
col1.set("type", "type1"); col1.set("type", "type1");
...@@ -410,70 +467,84 @@ public class DefaultMetadataServiceTest { ...@@ -410,70 +467,84 @@ public class DefaultMetadataServiceTest {
final List<Referenceable> updateColumns = Arrays.asList(col1, col2); final List<Referenceable> updateColumns = Arrays.asList(col1, col2);
tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{ tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
put("columns", updateColumns); put(COLUMNS_ATTR_NAME, updateColumns);
}}); }});
metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated); metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
tableDefinitionJson = verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), updateColumns, COLUMNS_ATTR_NAME);
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
List<Referenceable> arrColumnsList = (List) tableDefinition.get("columns");
Assert.assertEquals(arrColumnsList.size(), 2);
assertReferenceables(arrColumnsList.get(0), updateColumns.get(0));
assertReferenceables(arrColumnsList.get(1), updateColumns.get(1));
//Complete update. Add array elements - col3,4 //Complete update. Add array elements - col3,4
Map<String, Object> values1 = new HashMap<>(); Map<String, Object> values1 = new HashMap<>();
values1.put("name", "col3"); values1.put(NAME, "col3");
values1.put("type", "type"); values1.put("type", "type");
Referenceable ref1 = new Referenceable("column_type", values1); Referenceable ref1 = new Referenceable(TestUtils.COLUMN_TYPE, values1);
columns.add(ref1); columns.add(ref1);
Map<String, Object> values2 = new HashMap<>(); Map<String, Object> values2 = new HashMap<>();
values2.put("name", "col4"); values2.put(NAME, "col4");
values2.put("type", "type"); values2.put("type", "type");
Referenceable ref2 = new Referenceable("column_type", values2); Referenceable ref2 = new Referenceable(TestUtils.COLUMN_TYPE, values2);
columns.add(ref2); columns.add(ref2);
table.set("columns", columns); table.set(COLUMNS_ATTR_NAME, columns);
updateInstance(table); updateInstance(table);
tableDefinitionJson = verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); //Swap elements
arrColumnsList = (List) tableDefinition.get("columns"); columns.clear();
Assert.assertEquals(arrColumnsList.size(), columns.size()); columns.add(ref2);
assertReferenceables(arrColumnsList.get(1), columns.get(1)); columns.add(ref1);
assertReferenceables(arrColumnsList.get(2), columns.get(2));
table.set(COLUMNS_ATTR_NAME, columns);
updateInstance(table);
verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
//drop a single column
columns.clear();
columns.add(ref1);
table.set(COLUMNS_ATTR_NAME, columns);
updateInstance(table);
verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
//Remove a class reference/Id and insert another reference //Remove a class reference/Id and insert another reference
//Also covers isComposite case since columns is a composite //Also covers isComposite case since columns is a composite
values.clear(); values.clear();
columns.clear(); columns.clear();
values.put("name", "col2"); values.put(NAME, "col2");
values.put("type", "type"); values.put("type", "type");
col1 = new Referenceable("column_type", values); col1 = new Referenceable(TestUtils.COLUMN_TYPE, values);
columns.add(col1); columns.add(col1);
table.set("columns", columns); table.set(COLUMNS_ATTR_NAME, columns);
updateInstance(table); updateInstance(table);
tableDefinitionJson = verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
arrColumnsList = (List) tableDefinition.get("columns");
Assert.assertEquals(arrColumnsList.size(), columns.size());
assertReferenceables(arrColumnsList.get(0), columns.get(0));
//Update array column to null //Update array column to null
table.setNull("columns"); table.setNull(COLUMNS_ATTR_NAME);
String newtableId = updateInstance(table); String newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson = verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), null, COLUMNS_ATTR_NAME);
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); }
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNull(tableDefinition.get("columns")); private void verifyArrayUpdates(String typeName, String uniqAttrName, String uniqAttrValue, List<? extends Struct> expectedArray, String arrAttrName) throws AtlasException {
String json =
metadataService.getEntityDefinition(typeName, uniqAttrName, uniqAttrValue);
Referenceable entityDefinition = InstanceSerialization.fromJsonReferenceable(json, true);
if (expectedArray == null) {
Assert.assertNull(entityDefinition.get(arrAttrName));
} else {
Assert.assertEquals(((List<Referenceable>)entityDefinition.get(arrAttrName)).size(), expectedArray.size());
for (int index = 0; index < expectedArray.size(); index++) {
Assert.assertTrue(((List<Referenceable>) entityDefinition.get(arrAttrName)).get(index).equalsContents(expectedArray.get(index)));
}
}
} }
private void assertReferenceables(Referenceable r1, Referenceable r2) { private void assertReferenceables(Referenceable r1, Referenceable r2) {
...@@ -488,7 +559,7 @@ public class DefaultMetadataServiceTest { ...@@ -488,7 +559,7 @@ public class DefaultMetadataServiceTest {
@Test @Test
public void testStructs() throws Exception { public void testStructs() throws Exception {
Struct serdeInstance = new Struct(TestUtils.SERDE_TYPE); Struct serdeInstance = new Struct(TestUtils.SERDE_TYPE);
serdeInstance.set("name", "serde1Name"); serdeInstance.set(NAME, "serde1Name");
serdeInstance.set("serde", "test"); serdeInstance.set("serde", "test");
serdeInstance.set("description", "testDesc"); serdeInstance.set("description", "testDesc");
table.set("serde1", serdeInstance); table.set("serde1", serdeInstance);
...@@ -497,7 +568,7 @@ public class DefaultMetadataServiceTest { ...@@ -497,7 +568,7 @@ public class DefaultMetadataServiceTest {
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("serde1")); Assert.assertNotNull(tableDefinition.get("serde1"));
Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1"))); Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1")));
...@@ -506,7 +577,7 @@ public class DefaultMetadataServiceTest { ...@@ -506,7 +577,7 @@ public class DefaultMetadataServiceTest {
serdeInstance.set("serde", "testUpdated"); serdeInstance.set("serde", "testUpdated");
updateInstance(table); updateInstance(table);
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1"))); Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1")));
...@@ -523,7 +594,6 @@ public class DefaultMetadataServiceTest { ...@@ -523,7 +594,6 @@ public class DefaultMetadataServiceTest {
@Test @Test
public void testCreateEntityWithReferenceableHavingIdNoValue() throws Exception { public void testCreateEntityWithReferenceableHavingIdNoValue() throws Exception {
//ATLAS-383 Test //ATLAS-383 Test
Referenceable sdReferenceable = new Referenceable(TestUtils.STORAGE_DESC_TYPE); Referenceable sdReferenceable = new Referenceable(TestUtils.STORAGE_DESC_TYPE);
sdReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, TestUtils.randomString()); sdReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, TestUtils.randomString());
...@@ -547,7 +617,7 @@ public class DefaultMetadataServiceTest { ...@@ -547,7 +617,7 @@ public class DefaultMetadataServiceTest {
public void testClassUpdate() throws Exception { public void testClassUpdate() throws Exception {
//Create new db instance //Create new db instance
final Referenceable databaseInstance = new Referenceable(TestUtils.DATABASE_TYPE); final Referenceable databaseInstance = new Referenceable(TestUtils.DATABASE_TYPE);
databaseInstance.set("name", TestUtils.randomString()); databaseInstance.set(NAME, TestUtils.randomString());
databaseInstance.set("description", "new database"); databaseInstance.set("description", "new database");
String dbId = createInstance(databaseInstance); String dbId = createInstance(databaseInstance);
...@@ -564,7 +634,7 @@ public class DefaultMetadataServiceTest { ...@@ -564,7 +634,7 @@ public class DefaultMetadataServiceTest {
/* Update with referenceable - TODO - Fails . Need to fix this */ /* Update with referenceable - TODO - Fails . Need to fix this */
/*final String dbName = TestUtils.randomString(); /*final String dbName = TestUtils.randomString();
final Referenceable databaseInstance2 = new Referenceable(TestUtils.DATABASE_TYPE); final Referenceable databaseInstance2 = new Referenceable(TestUtils.DATABASE_TYPE);
databaseInstance2.set("name", dbName); databaseInstance2.set(NAME, dbName);
databaseInstance2.set("description", "new database 2"); databaseInstance2.set("description", "new database 2");
Referenceable updateTable = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{ Referenceable updateTable = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
...@@ -576,7 +646,7 @@ public class DefaultMetadataServiceTest { ...@@ -576,7 +646,7 @@ public class DefaultMetadataServiceTest {
metadataService.getEntityDefinition(tableId._getId()); metadataService.getEntityDefinition(tableId._getId());
Referenceable tableDefinitionActual = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinitionActual = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
String dbDefJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "name", dbName); String dbDefJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, NAME, dbName);
Referenceable dbDef = InstanceSerialization.fromJsonReferenceable(dbDefJson, true); Referenceable dbDef = InstanceSerialization.fromJsonReferenceable(dbDefJson, true);
Assert.assertNotEquals(dbId, (((Id) tableDefinitionActual.get("database"))._getId())); Assert.assertNotEquals(dbId, (((Id) tableDefinitionActual.get("database"))._getId()));
...@@ -590,10 +660,10 @@ public class DefaultMetadataServiceTest { ...@@ -590,10 +660,10 @@ public class DefaultMetadataServiceTest {
TestUtils.dumpGraph(graphProvider.get()); TestUtils.dumpGraph(graphProvider.get());
final Struct partition1 = new Struct(TestUtils.PARTITION_STRUCT_TYPE); final Struct partition1 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
partition1.set("name", "part1"); partition1.set(NAME, "part1");
final Struct partition2 = new Struct(TestUtils.PARTITION_STRUCT_TYPE); final Struct partition2 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
partition2.set("name", "part2"); partition2.set(NAME, "part2");
List<Struct> partitions = new ArrayList<Struct>(){{ add(partition1); add(partition2); }}; List<Struct> partitions = new ArrayList<Struct>(){{ add(partition1); add(partition2); }};
table.set("partitions", partitions); table.set("partitions", partitions);
...@@ -602,7 +672,7 @@ public class DefaultMetadataServiceTest { ...@@ -602,7 +672,7 @@ public class DefaultMetadataServiceTest {
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions")); Assert.assertNotNull(tableDefinition.get("partitions"));
...@@ -612,14 +682,14 @@ public class DefaultMetadataServiceTest { ...@@ -612,14 +682,14 @@ public class DefaultMetadataServiceTest {
//add a new element to array of struct //add a new element to array of struct
final Struct partition3 = new Struct(TestUtils.PARTITION_STRUCT_TYPE); final Struct partition3 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
partition3.set("name", "part3"); partition3.set(NAME, "part3");
partitions.add(partition3); partitions.add(partition3);
table.set("partitions", partitions); table.set("partitions", partitions);
newtableId = updateInstance(table); newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions")); Assert.assertNotNull(tableDefinition.get("partitions"));
...@@ -634,7 +704,7 @@ public class DefaultMetadataServiceTest { ...@@ -634,7 +704,7 @@ public class DefaultMetadataServiceTest {
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions")); Assert.assertNotNull(tableDefinition.get("partitions"));
...@@ -644,12 +714,12 @@ public class DefaultMetadataServiceTest { ...@@ -644,12 +714,12 @@ public class DefaultMetadataServiceTest {
Assert.assertTrue(partitions.get(1).equalsContents(partitionsActual.get(1))); Assert.assertTrue(partitions.get(1).equalsContents(partitionsActual.get(1)));
//Update struct value within array of struct //Update struct value within array of struct
partition1.set("name", "part4"); partition1.set(NAME, "part4");
newtableId = updateInstance(table); newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions")); Assert.assertNotNull(tableDefinition.get("partitions"));
...@@ -659,21 +729,21 @@ public class DefaultMetadataServiceTest { ...@@ -659,21 +729,21 @@ public class DefaultMetadataServiceTest {
//add a repeated element to array of struct //add a repeated element to array of struct
final Struct partition4 = new Struct(TestUtils.PARTITION_STRUCT_TYPE); final Struct partition4 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
partition4.set("name", "part4"); partition4.set(NAME, "part4");
partitions.add(partition4); partitions.add(partition4);
table.set("partitions", partitions); table.set("partitions", partitions);
newtableId = updateInstance(table); newtableId = updateInstance(table);
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNotNull(tableDefinition.get("partitions")); Assert.assertNotNull(tableDefinition.get("partitions"));
partitionsActual = (List<Struct>) tableDefinition.get("partitions"); partitionsActual = (List<Struct>) tableDefinition.get("partitions");
Assert.assertEquals(partitionsActual.size(), 3); Assert.assertEquals(partitionsActual.size(), 3);
Assert.assertEquals(partitionsActual.get(2).get("name"), "part4"); Assert.assertEquals(partitionsActual.get(2).get(NAME), "part4");
Assert.assertEquals(partitionsActual.get(0).get("name"), "part4"); Assert.assertEquals(partitionsActual.get(0).get(NAME), "part4");
Assert.assertTrue(partitions.get(2).equalsContents(partitionsActual.get(2))); Assert.assertTrue(partitions.get(2).equalsContents(partitionsActual.get(2)));
...@@ -683,7 +753,7 @@ public class DefaultMetadataServiceTest { ...@@ -683,7 +753,7 @@ public class DefaultMetadataServiceTest {
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNull(tableDefinition.get("partitions")); Assert.assertNull(tableDefinition.get("partitions"));
...@@ -694,7 +764,7 @@ public class DefaultMetadataServiceTest { ...@@ -694,7 +764,7 @@ public class DefaultMetadataServiceTest {
public void testUpdateRequiredAttrToNull() throws Exception { public void testUpdateRequiredAttrToNull() throws Exception {
//Update required attribute //Update required attribute
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertEquals(tableDefinition.get("description"), "random table"); Assert.assertEquals(tableDefinition.get("description"), "random table");
...@@ -708,7 +778,7 @@ public class DefaultMetadataServiceTest { ...@@ -708,7 +778,7 @@ public class DefaultMetadataServiceTest {
public void testUpdateOptionalAttrToNull() throws Exception { public void testUpdateOptionalAttrToNull() throws Exception {
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
//Update optional Attribute //Update optional Attribute
...@@ -720,7 +790,7 @@ public class DefaultMetadataServiceTest { ...@@ -720,7 +790,7 @@ public class DefaultMetadataServiceTest {
Assert.assertEquals(newtableId, tableId._getId()); Assert.assertEquals(newtableId, tableId._getId());
tableDefinitionJson = tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Assert.assertNull(tableDefinition.get("created")); Assert.assertNull(tableDefinition.get("created"));
} }
...@@ -728,7 +798,7 @@ public class DefaultMetadataServiceTest { ...@@ -728,7 +798,7 @@ public class DefaultMetadataServiceTest {
@Test @Test
public void testCreateEntityWithEnum() throws Exception { public void testCreateEntityWithEnum() throws Exception {
String tableDefinitionJson = String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name")); metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
EnumValue tableType = (EnumValue) tableDefinition.get("tableType"); EnumValue tableType = (EnumValue) tableDefinition.get("tableType");
...@@ -741,15 +811,15 @@ public class DefaultMetadataServiceTest { ...@@ -741,15 +811,15 @@ public class DefaultMetadataServiceTest {
createInstance(entity); createInstance(entity);
//get entity by valid qualified name //get entity by valid qualified name
String entityJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "name", String entityJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, NAME,
(String) entity.get("name")); (String) entity.get(NAME));
Assert.assertNotNull(entityJson); Assert.assertNotNull(entityJson);
Referenceable referenceable = InstanceSerialization.fromJsonReferenceable(entityJson, true); Referenceable referenceable = InstanceSerialization.fromJsonReferenceable(entityJson, true);
Assert.assertEquals(referenceable.get("name"), entity.get("name")); Assert.assertEquals(referenceable.get(NAME), entity.get(NAME));
//get entity by invalid qualified name //get entity by invalid qualified name
try { try {
metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "name", "random"); metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, NAME, "random");
Assert.fail("Expected EntityNotFoundException"); Assert.fail("Expected EntityNotFoundException");
} catch (EntityNotFoundException e) { } catch (EntityNotFoundException e) {
//expected //expected
...@@ -776,25 +846,25 @@ public class DefaultMetadataServiceTest { ...@@ -776,25 +846,25 @@ public class DefaultMetadataServiceTest {
Referenceable col1 = createColumnEntity(); Referenceable col1 = createColumnEntity();
Referenceable col2 = createColumnEntity(); Referenceable col2 = createColumnEntity();
Referenceable col3 = createColumnEntity(); Referenceable col3 = createColumnEntity();
table1Entity.set("columns", ImmutableList.of(col1, col2, col3)); table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col3));
table2Entity.set("columns", ImmutableList.of(col1, col2, col3)); table2Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col3));
createInstance(table1Entity); createInstance(table1Entity);
createInstance(table2Entity); createInstance(table2Entity);
// Retrieve the table entities from the repository, // Retrieve the table entities from the repository,
// to get their guids and the composite column guids. // to get their guids and the composite column guids.
String entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, String entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE,
"name", (String)table1Entity.get("name")); NAME, (String)table1Entity.get(NAME));
Assert.assertNotNull(entityJson); Assert.assertNotNull(entityJson);
table1Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true); table1Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true);
Object val = table1Entity.get("columns"); Object val = table1Entity.get(COLUMNS_ATTR_NAME);
Assert.assertTrue(val instanceof List); Assert.assertTrue(val instanceof List);
List<IReferenceableInstance> table1Columns = (List<IReferenceableInstance>) val; List<IReferenceableInstance> table1Columns = (List<IReferenceableInstance>) val;
entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE,
"name", (String)table2Entity.get("name")); NAME, (String)table2Entity.get(NAME));
Assert.assertNotNull(entityJson); Assert.assertNotNull(entityJson);
table2Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true); table2Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true);
val = table2Entity.get("columns"); val = table2Entity.get(COLUMNS_ATTR_NAME);
Assert.assertTrue(val instanceof List); Assert.assertTrue(val instanceof List);
List<IReferenceableInstance> table2Columns = (List<IReferenceableInstance>) val; List<IReferenceableInstance> table2Columns = (List<IReferenceableInstance>) val;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment