Skip to content

A
atlas
Commit de243153 by Venkatesh Seetharam
Options

Fix Map type mapping with tests. Contributed by Venkatesh Seetharam

parent d97a7236
Showing with 133 additions and 52 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
...@@ -18,7 +18,6 @@ ...@@ -18,7 +18,6 @@
package org.apache.hadoop.metadata.repository.graph; package org.apache.hadoop.metadata.repository.graph;
import com.google.common.collect.ImmutableCollection;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanProperty; import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex; import com.thinkaurelius.titan.core.TitanVertex;
...@@ -209,6 +208,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -209,6 +208,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Map<String,String> oRow = new HashMap<>(); Map<String,String> oRow = new HashMap<>();
if ( r instanceof Map ) { if ( r instanceof Map ) {
@SuppressWarnings("unchecked")
Map<Object,Object> iRow = (Map) r; Map<Object,Object> iRow = (Map) r;
for(Map.Entry e : iRow.entrySet()) { for(Map.Entry e : iRow.entrySet()) {
Object k = e.getKey(); Object k = e.getKey();
...@@ -289,9 +289,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -289,9 +289,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
final Id id = typedInstance.getId(); final Id id = typedInstance.getId();
if (!idToVertexMap.containsKey(id)) { if (!idToVertexMap.containsKey(id)) {
Vertex instanceVertex; Vertex instanceVertex;
if (id.isAssigned()) { if (id.isAssigned()) { // has a GUID
instanceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id); instanceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
} else { } else {
instanceVertex = GraphHelper.createVertex(titanGraph, typedInstance); instanceVertex = GraphHelper.createVertex(titanGraph, typedInstance);
} }
...@@ -429,14 +428,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -429,14 +428,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break; break;
case MAP: case MAP:
/*
mapMapCollectionToVertex( mapMapCollectionToVertex(
id, typedInstance, instanceVertex, attributeInfo, idToVertexMap); id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
*/
break; break;
case STRUCT: case STRUCT:
Vertex structInstanceVertex = mapStructInstanceToVertex(id, typedInstance, Vertex structInstanceVertex = mapStructInstanceToVertex(id,
(ITypedStruct) typedInstance.get(attributeInfo.name), (ITypedStruct) typedInstance.get(attributeInfo.name),
attributeInfo, idToVertexMap); attributeInfo, idToVertexMap);
// add an edge to the newly created vertex from the parent // add an edge to the newly created vertex from the parent
...@@ -450,8 +447,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -450,8 +447,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
case CLASS: case CLASS:
Id referenceId = (Id) typedInstance.get(attributeInfo.name); Id referenceId = (Id) typedInstance.get(attributeInfo.name);
mapClassReferenceAsEdge( mapClassReferenceAsEdge(
instanceVertex, typedInstance, idToVertexMap, propertyName, referenceId, instanceVertex, idToVertexMap, propertyName, referenceId
attributeInfo.dataType().getName()); );
break; break;
default: default:
...@@ -465,7 +462,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -465,7 +462,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Map<Id, Vertex> idToVertexMap) throws MetadataException { Map<Id, Vertex> idToVertexMap) throws MetadataException {
LOG.debug("Mapping instance {} to vertex {} for name {}", LOG.debug("Mapping instance {} to vertex {} for name {}",
typedInstance.getTypeName(), instanceVertex, attributeInfo.name); typedInstance.getTypeName(), instanceVertex, attributeInfo.name);
ImmutableCollection list = (ImmutableCollection) typedInstance.get(attributeInfo.name); List list = (List) typedInstance.get(attributeInfo.name);
if (list == null) { if (list == null) {
return; return;
} }
...@@ -481,14 +478,57 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -481,14 +478,57 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
for (int index = 0; index < array.length; index++) { for (int index = 0; index < array.length; index++) {
String propertyNameWithSuffix = propertyName + "." + index; String propertyNameWithSuffix = propertyName + "." + index;
buffer.append(propertyNameWithSuffix).append(","); buffer.append(propertyNameWithSuffix).append(",");
final Object value = array[index]; mapCollectionEntryToVertex(id, instanceVertex, attributeInfo,
idToVertexMap, elementType, array[index], propertyNameWithSuffix);
}
buffer.setLength(buffer.length() - 1);
// for dereference on way out
instanceVertex.setProperty(propertyName, buffer.toString());
}
private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance,
Vertex instanceVertex,
AttributeInfo attributeInfo,
Map<Id, Vertex> idToVertexMap) throws MetadataException {
LOG.debug("Mapping instance {} to vertex {} for name {}",
typedInstance.getTypeName(), instanceVertex, attributeInfo.name);
@SuppressWarnings("unchecked")
Map<Object, Object> collection = (Map<Object, Object>) typedInstance.get(attributeInfo.name);
if (collection == null) {
return;
}
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
// todo: move this to the indexer
GraphHelper.createPropertyKey(titanGraph.getManagementSystem(), propertyName);
StringBuilder buffer = new StringBuilder();
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
for (Map.Entry entry : collection.entrySet()) {
String propertyNameWithSuffix = propertyName + "." + entry.getKey();
buffer.append(propertyNameWithSuffix).append(",");
mapCollectionEntryToVertex(id, instanceVertex, attributeInfo,
idToVertexMap, elementType, entry.getValue(), propertyNameWithSuffix);
}
buffer.setLength(buffer.length() - 1);
// for dereference on way out
instanceVertex.setProperty(propertyName, buffer.toString());
}
private void mapCollectionEntryToVertex(Id id, Vertex instanceVertex,
AttributeInfo attributeInfo,
Map<Id, Vertex> idToVertexMap,
IDataType elementType, Object value,
String propertyName) throws MetadataException {
switch (elementType.getTypeCategory()) { switch (elementType.getTypeCategory()) {
case PRIMITIVE: case PRIMITIVE:
instanceVertex.setProperty(propertyNameWithSuffix, value); instanceVertex.setProperty(propertyName, value);
break; break;
case ENUM: case ENUM:
instanceVertex.setProperty(propertyNameWithSuffix, value); instanceVertex.setProperty(propertyName, value);
break; break;
case ARRAY: case ARRAY:
...@@ -498,18 +538,18 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -498,18 +538,18 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break; break;
case STRUCT: case STRUCT:
Vertex structInstanceVertex = mapStructInstanceToVertex(id, typedInstance, Vertex structInstanceVertex = mapStructInstanceToVertex(id,
(ITypedStruct) value, attributeInfo, idToVertexMap); (ITypedStruct) value, attributeInfo, idToVertexMap);
// add an edge to the newly created vertex from the parent // add an edge to the newly created vertex from the parent
GraphHelper.addEdge(instanceVertex, structInstanceVertex, GraphHelper.addEdge(instanceVertex, structInstanceVertex,
propertyNameWithSuffix); propertyName);
break; break;
case CLASS: case CLASS:
Id referenceId = (Id) value; Id referenceId = (Id) value;
mapClassReferenceAsEdge( mapClassReferenceAsEdge(
instanceVertex, typedInstance, idToVertexMap, instanceVertex, idToVertexMap,
propertyNameWithSuffix, referenceId, elementType.getName()); propertyName, referenceId);
break; break;
default: default:
...@@ -517,15 +557,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -517,15 +557,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} }
} }
buffer.setLength(buffer.length() - 1); private void mapClassReferenceAsEdge(Vertex instanceVertex,
// for dereference on way out
instanceVertex.setProperty(propertyName, buffer.toString());
}
private void mapClassReferenceAsEdge(Vertex instanceVertex, ITypedInstance typedInstance,
Map<Id, Vertex> idToVertexMap, Map<Id, Vertex> idToVertexMap,
String propertyKey, Id id, String typeName) String propertyKey, Id id) throws MetadataException {
throws MetadataException {
if (id != null) { if (id != null) {
Vertex referenceVertex; Vertex referenceVertex;
...@@ -533,9 +567,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -533,9 +567,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id); referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
} else { } else {
referenceVertex = idToVertexMap.get(id); referenceVertex = idToVertexMap.get(id);
/*
ClassType classType = typeSystem.getDataType(ClassType.class, typeName); ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
mapInstanceToVertex(id, typedInstance, referenceVertex, mapInstanceToVertex(id, typedInstance, referenceVertex,
classType.fieldMapping().fields, idToVertexMap); classType.fieldMapping().fields, idToVertexMap);
*/
} }
if (referenceVertex != null) { if (referenceVertex != null) {
...@@ -547,19 +583,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -547,19 +583,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} }
} }
/* private Vertex mapStructInstanceToVertex(Id id, ITypedStruct structInstance,
private void mapMapCollectionToVertex(ITypedInstance typedInstance,
Vertex instanceVertex,
AttributeInfo attributeInfo,
Map<Id, Vertex> idToVertexMap) throws MetadataException {
DataTypes.MapType collection = (DataTypes.MapType) typedInstance.get(attributeInfo.name);
ImmutableMap map = collection.convert(collection, Multiplicity.COLLECTION);
}
*/
private Vertex mapStructInstanceToVertex(Id id, ITypedInstance typedInstance,
ITypedStruct structInstance,
AttributeInfo attributeInfo, AttributeInfo attributeInfo,
Map<Id, Vertex> idToVertexMap) throws MetadataException { Map<Id, Vertex> idToVertexMap) throws MetadataException {
// add a new vertex for the struct or trait instance // add a new vertex for the struct or trait instance
...@@ -698,7 +722,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -698,7 +722,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break; break;
case MAP: case MAP:
// todo - Add to/from json for collections mapVertexToMapInstance(instanceVertex, typedInstance, attributeInfo);
break; break;
case STRUCT: case STRUCT:
...@@ -707,15 +731,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -707,15 +731,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
case TRAIT: case TRAIT:
// do NOTHING - handled in class // do NOTHING - handled in class
// mapVertexToCompositeInstance(instanceVertex, typedInstance, attributeInfo);
break; break;
case CLASS: case CLASS:
// todo - use ObjectWalker here instead else it can be an infinite loop
// for cross references
String relationshipLabel = typedInstance.getTypeName() + "." + attributeInfo.name; String relationshipLabel = typedInstance.getTypeName() + "." + attributeInfo.name;
Object idOrInstance = mapClassReferenceToVertex( Object idOrInstance = mapClassReferenceToVertex(instanceVertex,
instanceVertex, attributeInfo, relationshipLabel); attributeInfo, relationshipLabel, attributeInfo.dataType());
typedInstance.set(attributeInfo.name, idOrInstance); typedInstance.set(attributeInfo.name, idOrInstance);
break; break;
...@@ -727,7 +748,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -727,7 +748,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private Object mapClassReferenceToVertex(Vertex instanceVertex, private Object mapClassReferenceToVertex(Vertex instanceVertex,
AttributeInfo attributeInfo, AttributeInfo attributeInfo,
String relationshipLabel) throws MetadataException { String relationshipLabel,
IDataType dataType) throws MetadataException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel); LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
Iterator<Edge> results = instanceVertex.getEdges( Iterator<Edge> results = instanceVertex.getEdges(
Direction.OUT, relationshipLabel).iterator(); Direction.OUT, relationshipLabel).iterator();
...@@ -743,7 +765,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -743,7 +765,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} else { } else {
Id referenceId = new Id(guid, Id referenceId = new Id(guid,
referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
attributeInfo.name); dataType.getName());
LOG.debug("Found non-composite, adding id {} ", referenceId); LOG.debug("Found non-composite, adding id {} ", referenceId);
return referenceId; return referenceId;
} }
...@@ -753,23 +775,81 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -753,23 +775,81 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return null; return null;
} }
@SuppressWarnings("unchecked")
private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance, private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException { AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name); LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
String keys = instanceVertex.getProperty(propertyName);
DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType(); DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
final IDataType elementType = arrayType.getElemType();
ArrayList values = new ArrayList();
for (String propertyNameWithSuffix : keys.split(",")) {
values.add(mapVertexToCollectionEntry(
instanceVertex, attributeInfo, elementType, propertyNameWithSuffix));
}
typedInstance.set(attributeInfo.name, values);
}
private Object mapVertexToCollectionEntry(Vertex instanceVertex,
AttributeInfo attributeInfo,
IDataType elementType,
String propertyNameWithSuffix) throws MetadataException {
switch (elementType.getTypeCategory()) {
case PRIMITIVE:
return instanceVertex.getProperty(propertyNameWithSuffix);
case ENUM:
return instanceVertex.<Integer>getProperty(propertyNameWithSuffix);
case ARRAY:
case MAP:
case TRAIT:
// do nothing
break;
case STRUCT:
return getStructInstanceFromVertex(instanceVertex,
elementType, attributeInfo.name, propertyNameWithSuffix);
case CLASS:
return mapClassReferenceToVertex(
instanceVertex, attributeInfo, propertyNameWithSuffix, elementType);
default:
break;
}
throw new IllegalArgumentException();
}
@SuppressWarnings("unchecked")
private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name; String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
String keys = instanceVertex.getProperty(propertyName); String keys = instanceVertex.getProperty(propertyName);
ArrayList values = new ArrayList(); DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
final IDataType elementType = mapType.getValueType();
HashMap values = new HashMap();
for (String propertyNameWithSuffix : keys.split(",")) { for (String propertyNameWithSuffix : keys.split(",")) {
switch (arrayType.getElemType().getTypeCategory()) { final String key = propertyNameWithSuffix.substring(
propertyNameWithSuffix.lastIndexOf("."), propertyNameWithSuffix.length());
values.put(key, mapVertexToCollectionEntry(
instanceVertex, attributeInfo, elementType, propertyNameWithSuffix));
/*
switch (valueType.getTypeCategory()) {
case PRIMITIVE: case PRIMITIVE:
values.add(instanceVertex.getProperty(propertyNameWithSuffix)); values.put(key, instanceVertex.getProperty(propertyNameWithSuffix));
break; break;
case ENUM: case ENUM:
values.add(instanceVertex.<Integer>getProperty(propertyNameWithSuffix)); values.put(key, instanceVertex.getProperty(propertyNameWithSuffix));
break; break;
case ARRAY: case ARRAY:
...@@ -780,19 +860,20 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -780,19 +860,20 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
case STRUCT: case STRUCT:
ITypedStruct structInstance = getStructInstanceFromVertex(instanceVertex, ITypedStruct structInstance = getStructInstanceFromVertex(instanceVertex,
arrayType.getElemType(), attributeInfo.name, propertyNameWithSuffix); valueType, attributeInfo.name, propertyNameWithSuffix);
values.add(structInstance); values.put(key, structInstance);
break; break;
case CLASS: case CLASS:
Object idOrInstance = mapClassReferenceToVertex( Object idOrInstance = mapClassReferenceToVertex(
instanceVertex, attributeInfo, propertyNameWithSuffix); instanceVertex, attributeInfo, propertyNameWithSuffix);
values.add(idOrInstance); values.put(key, idOrInstance);
break; break;
default: default:
break; break;
} }
*/
} }
typedInstance.set(attributeInfo.name, values); typedInstance.set(attributeInfo.name, values);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment