add ObjectGraphWalker and DiscoverInstances NodeProcessor

src/main/java/org/apache/metadata/storage/ReferenceableInstance.java

@@ -35,7 +35,7 @@ import java.util.Date;
  */
 public class ReferenceableInstance extends StructInstance implements ITypedReferenceableInstance {
 
-    private final Id id;
+    private Id id;
     private final ImmutableMap<String, ITypedStruct> traits;
     private final ImmutableList<String> traitNames;
 
@@ -75,6 +75,14 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
         return traits.get(typeName);
     }
 
+    /**
+     * @nopub
+     * @param id
+     */
+    void replaceWithNewId(Id id) {
+        this.id = id;
+    }
+
     @Override
     public String toString()  {
         try {

src/main/java/org/apache/metadata/storage/memory/DiscoverInstances.java

+package org.apache.metadata.storage.memory;
+
+
+import org.apache.metadata.IReferenceableInstance;
+import org.apache.metadata.MetadataException;
+import org.apache.metadata.storage.Id;
+import org.apache.metadata.storage.RepositoryException;
+import org.apache.metadata.types.DataTypes;
+import org.apache.metadata.types.ObjectGraphWalker;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DiscoverInstances implements ObjectGraphWalker.NodeProcessor {
+
+    final MemRepository memRepository;
+    final Map<Id, Id> idToNewIdMap;
+    final Map<Id, IReferenceableInstance> idToInstanceMap;
+
+    public DiscoverInstances(MemRepository memRepository) {
+        this.memRepository = memRepository;
+        idToNewIdMap = new HashMap<Id, Id>();
+        idToInstanceMap = new HashMap<Id, IReferenceableInstance>();
+    }
+
+    @Override
+    public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
+
+        IReferenceableInstance ref =  null;
+        Id id = null;
+
+        if ( nd.attributeName == null ) {
+            ref = (IReferenceableInstance) nd.instance;
+            id = ref.getId();
+        } else if ( nd.dataType.getTypeCategory() == DataTypes.TypeCategory.CLASS ) {
+            if ( nd.value != null && (nd.value instanceof  Id)) {
+                id = (Id) nd.value;
+            }
+        }
+
+        if ( id != null ) {
+            if ( id.isUnassigned() ) {
+                if ( !idToNewIdMap.containsKey(id)) {
+                    idToNewIdMap.put(id, memRepository.newId(id.className));
+                }
+                if ( idToInstanceMap.containsKey(ref)) {
+                    // Oops
+                    throw new RepositoryException(
+                            String.format("Unexpected internal error: Id %s processed again", id));
+                }
+                idToInstanceMap.put(id, ref);
+            }
+        }
+    }
+}

src/main/java/org/apache/metadata/storage/memory/MemRepository.java

@@ -18,22 +18,32 @@
 
 package org.apache.metadata.storage.memory;
 
-import org.apache.metadata.IInstance;
-import org.apache.metadata.IReferenceableInstance;
-import org.apache.metadata.ITypedInstance;
-import org.apache.metadata.ITypedReferenceableInstance;
+import org.apache.metadata.*;
 import org.apache.metadata.storage.IRepository;
 import org.apache.metadata.storage.Id;
 import org.apache.metadata.storage.RepositoryException;
+import org.apache.metadata.types.ObjectGraphTraversal;
+import org.apache.metadata.types.ObjectGraphWalker;
+import org.apache.metadata.types.TypeSystem;
 
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 public class MemRepository implements IRepository {
 
     private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
     private static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
+    final TypeSystem typeSystem;
+    final AtomicInteger ID_SEQ = new AtomicInteger(0);
+
+    public MemRepository(TypeSystem typeSystem) {
+        this.typeSystem = typeSystem;
+    }
+
     @Override
     public DateFormat getDateFormat() {
         return dateFormat;
@@ -49,12 +59,17 @@ public class MemRepository implements IRepository {
         return false;
     }
 
+    Id newId(String typeName) {
+        return new Id(ID_SEQ.incrementAndGet(), 0, typeName);
+    }
+
     /**
      * 1. traverse the Object Graph from  i and create idToNewIdMap : Map[Id, Id],
      *    also create old Id to Instance Map: oldIdToInstance : Map[Id, IInstance]
      *   - traverse reference Attributes, List[ClassType], Maps where Key/value is ClassType
      *   - traverse Structs
      *   - traverse Traits.
+     * 1b. Ensure that every newId has an associated Instance.
      * 2. Traverse oldIdToInstance map create newInstances : List[ITypedReferenceableInstance]
      *    - create a ITypedReferenceableInstance.
      *      replace any old References ( ids or object references) with new Ids.
@@ -71,6 +86,16 @@ public class MemRepository implements IRepository {
      * @throws RepositoryException
      */
     public ITypedReferenceableInstance create(IReferenceableInstance i) throws RepositoryException {
+
+        DiscoverInstances discoverInstances = new DiscoverInstances(this);
+
+        try {
+            new ObjectGraphWalker(typeSystem, discoverInstances, i).walk();
+        } catch (MetadataException me) {
+            throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
+        }
+
+
         throw new RepositoryException("not implemented");
     }
 

src/main/java/org/apache/metadata/types/ClassType.java

@@ -82,6 +82,7 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
             if ( val instanceof Struct) {
                 Struct s = (Struct) val;
                 Referenceable r = null;
+                Id id = null;
 
                 if ( s.typeName != getName() ) {
                     /*
@@ -96,10 +97,11 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
 
                 if ( val instanceof Referenceable ) {
                      r = (Referenceable)val;
+                    id = r.getId();
                 }
 
                 ITypedReferenceableInstance tr = r != null ?
-                        createInstanceWithTraits(r, r.getTraits().toArray(new String[0])) : createInstance();
+                        createInstanceWithTraits(id, r, r.getTraits().toArray(new String[0])) : createInstance(id);
 
                 for(Map.Entry<String,AttributeInfo> e : fieldMapping.fields.entrySet() ) {
                     String attrKey = e.getKey();
@@ -134,10 +136,14 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
 
     @Override
     public ITypedReferenceableInstance createInstance() throws MetadataException {
-        return createInstanceWithTraits(null);
+        return createInstance(null);
     }
 
-    public ITypedReferenceableInstance createInstanceWithTraits(Referenceable r, String... traitNames)
+    public ITypedReferenceableInstance createInstance(Id id) throws MetadataException {
+        return createInstanceWithTraits(id, null);
+    }
+
+    public ITypedReferenceableInstance createInstanceWithTraits(Id id, Referenceable r, String... traitNames)
     throws MetadataException {
 
         ImmutableMap.Builder<String, ITypedStruct> b = new ImmutableBiMap.Builder<String, ITypedStruct>();
@@ -149,7 +155,7 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
             b.put(t, trait);
         }
 
-        return new ReferenceableInstance(new Id(getName()),
+        return new ReferenceableInstance(id == null ? new Id(getName()) : id,
                 getName(),
                 fieldMapping,
                 new boolean[fieldMapping.fields.size()],

src/main/java/org/apache/metadata/types/ObjectGraphTraversal.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metadata.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.metadata.IReferenceableInstance;
+import org.apache.metadata.IStruct;
+import org.apache.metadata.MetadataException;
+import org.apache.metadata.storage.Id;
+
+
+import java.util.*;
+
+public class ObjectGraphTraversal implements Iterator<ObjectGraphTraversal.InstanceTuple> {
+
+    final Queue<InstanceTuple> queue;
+    final TypeSystem typeSystem;
+    Set<Id> processedIds;
+
+    public ObjectGraphTraversal(TypeSystem typeSystem, IReferenceableInstance start)  throws MetadataException {
+        this.typeSystem = typeSystem;
+        queue = new LinkedList<InstanceTuple>();
+        processedIds = new HashSet<Id>();
+        processReferenceableInstance(start);
+    }
+
+    void processValue(IDataType dT, Object val) throws MetadataException {
+        if ( val != null ) {
+            if ( dT.getTypeCategory() == DataTypes.TypeCategory.ARRAY ) {
+                IDataType elemType = ((DataTypes.ArrayType)dT).getElemType();
+                processCollection(elemType, val);
+            }  else if (dT.getTypeCategory() == DataTypes.TypeCategory.MAP) {
+                IDataType keyType = ((DataTypes.MapType)dT).getKeyType();
+                IDataType valueType = ((DataTypes.MapType)dT).getKeyType();
+                processMap(keyType, valueType, val);
+            }   else if (dT.getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
+                    dT.getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+                processStruct(val);
+            } else if (dT.getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+                processReferenceableInstance(val);
+            }
+        }
+    }
+
+    void processMap(IDataType keyType, IDataType valueType, Object val) throws MetadataException {
+        if ( keyType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE &&
+                valueType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE ) {
+            return;
+        }
+
+        if ( val != null ) {
+            Iterator<Map.Entry> it = null;
+            if ( Map.class.isAssignableFrom(val.getClass())) {
+                it = ((Map)val).entrySet().iterator();
+                ImmutableMap.Builder b = ImmutableMap.builder();
+                while (it.hasNext() ) {
+                    Map.Entry e = it.next();
+                    processValue(keyType, e.getKey());
+                    processValue(valueType, e.getValue());
+                }
+            }
+        }
+    }
+
+    void processCollection(IDataType elemType, Object val) throws MetadataException {
+
+        if ( elemType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE ) {
+            return;
+        }
+
+        if ( val != null ) {
+            Iterator it = null;
+            if (val instanceof Collection) {
+                it = ((Collection) val).iterator();
+            } else if (val instanceof Iterable) {
+                it = ((Iterable) val).iterator();
+            } else if (val instanceof Iterator) {
+                it = (Iterator) val;
+            }
+            if (it != null) {
+                DataTypes.TypeCategory elemCategory = elemType.getTypeCategory();
+                while (it.hasNext()) {
+                    Object elem = it.next();
+                    processValue(elemType, elem);
+                }
+            }
+        }
+    }
+
+    void processStruct(Object val) throws MetadataException {
+
+        if ( val == null || !(val instanceof  IStruct) ) {
+            return;
+        }
+
+        IStruct i = (IStruct) val;
+
+        IConstructableType type = typeSystem.getDataType(IConstructableType.class, i.getTypeName());
+
+        for(Map.Entry<String,AttributeInfo> e : type.fieldMapping().fields.entrySet()) {
+            AttributeInfo aInfo = e.getValue();
+            String attrName = e.getKey();
+            if ( aInfo.dataType().getTypeCategory() != DataTypes.TypeCategory.PRIMITIVE ) {
+                processValue(aInfo.dataType(), i.get(attrName));
+            }
+        }
+    }
+
+    void processReferenceableInstance(Object val) throws MetadataException {
+
+        if ( val == null || !(val instanceof  IReferenceableInstance || val instanceof Id)  ) {
+            return;
+        }
+
+        if ( val instanceof  Id) {
+            Id id = (Id) val;
+            if ( id.isUnassigned() ) {
+                add(id, null);
+            }
+            return;
+        }
+
+        IReferenceableInstance ref = (IReferenceableInstance) val;
+        Id id = ref.getId();
+        if ( id.isUnassigned() ) {
+            add(id, ref);
+            if (!processedIds.contains(id)) {
+                processedIds.add(id);
+                processStruct(val);
+
+                ImmutableList<String> traits = ref.getTraits();
+                for(String trait : traits) {
+                    processStruct(ref.getTrait(trait));
+                }
+            }
+        }
+    }
+
+    void add(Id id, IReferenceableInstance ref) {
+        queue.add(new InstanceTuple(id, ref));
+    }
+
+
+    @Override
+    public boolean hasNext() {
+        return !queue.isEmpty();
+    }
+
+    @Override
+    public InstanceTuple next() {
+        try {
+            InstanceTuple t = queue.poll();
+            processReferenceableInstance(t.instance);
+            return t;
+        } catch(MetadataException me) {
+            throw new RuntimeException(me);
+        }
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+    public static class InstanceTuple {
+        public final Id id;
+        public final IReferenceableInstance instance;
+
+        public InstanceTuple(Id id, IReferenceableInstance instance) {
+            this.id = id;
+            this.instance = instance;
+        }
+    }
+
+}

src/main/java/org/apache/metadata/types/ObjectGraphWalker.java

+package org.apache.metadata.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.metadata.IReferenceableInstance;
+import org.apache.metadata.IStruct;
+import org.apache.metadata.MetadataException;
+import org.apache.metadata.storage.Id;
+import org.apache.metadata.storage.RepositoryException;
+
+import java.util.*;
+
+/**
+ * Given a IReferenceableInstance, a Walker will traverse the Object Graph
+ * reachable form the instance. It will invoke the process call on the provided NodeProcessor
+ * for each non-primitive attribute (Structs, Traits, References, Arrays of Non-Primitives, Maps of Non-Primitives)
+ */
+public class ObjectGraphWalker {
+
+    final Queue<IReferenceableInstance> queue;
+    final TypeSystem typeSystem;
+    final NodeProcessor nodeProcessor;
+    Set<Id> processedIds;
+
+    public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor, IReferenceableInstance start)
+            throws MetadataException {
+        this.typeSystem = typeSystem;
+        this.nodeProcessor = nodeProcessor;
+        queue = new LinkedList<IReferenceableInstance>();
+        processedIds = new HashSet<Id>();
+        queue.add(start);
+    }
+
+    public void walk() throws MetadataException {
+        while (!queue.isEmpty()) {
+            IReferenceableInstance r = queue.poll();
+            processReferenceableInstance(r);
+        }
+    }
+
+    void traverseValue(IDataType dT, Object val) throws MetadataException {
+        if (val != null) {
+            if (dT.getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+                IDataType elemType = ((DataTypes.ArrayType) dT).getElemType();
+                visitCollection(elemType, val);
+            } else if (dT.getTypeCategory() == DataTypes.TypeCategory.MAP) {
+                IDataType keyType = ((DataTypes.MapType) dT).getKeyType();
+                IDataType valueType = ((DataTypes.MapType) dT).getKeyType();
+                visitMap(keyType, valueType, val);
+            } else if (dT.getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
+                    dT.getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+                visitStruct(val);
+            } else if (dT.getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+                visitReferenceableInstance(val);
+            }
+        }
+    }
+
+    void visitMap(IDataType keyType, IDataType valueType, Object val) throws MetadataException {
+        if (keyType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE &&
+                valueType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
+            return;
+        }
+
+        if (val != null) {
+            Iterator<Map.Entry> it = null;
+            if (Map.class.isAssignableFrom(val.getClass())) {
+                it = ((Map) val).entrySet().iterator();
+                ImmutableMap.Builder b = ImmutableMap.builder();
+                while (it.hasNext()) {
+                    Map.Entry e = it.next();
+                    traverseValue(keyType, e.getKey());
+                    traverseValue(valueType, e.getValue());
+                }
+            }
+        }
+    }
+
+    void visitCollection(IDataType elemType, Object val) throws MetadataException {
+
+        if (elemType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
+            return;
+        }
+
+        if (val != null) {
+            Iterator it = null;
+            if (val instanceof Collection) {
+                it = ((Collection) val).iterator();
+            } else if (val instanceof Iterable) {
+                it = ((Iterable) val).iterator();
+            } else if (val instanceof Iterator) {
+                it = (Iterator) val;
+            }
+            if (it != null) {
+                DataTypes.TypeCategory elemCategory = elemType.getTypeCategory();
+                while (it.hasNext()) {
+                    Object elem = it.next();
+                    traverseValue(elemType, elem);
+                }
+            }
+        }
+    }
+
+    void visitStruct(Object val) throws MetadataException {
+
+        if (val == null || !(val instanceof IStruct)) {
+            return;
+        }
+
+        IStruct i = (IStruct) val;
+
+        IConstructableType type = typeSystem.getDataType(IConstructableType.class, i.getTypeName());
+
+        for (Map.Entry<String, AttributeInfo> e : type.fieldMapping().fields.entrySet()) {
+            AttributeInfo aInfo = e.getValue();
+            String attrName = e.getKey();
+            if (aInfo.dataType().getTypeCategory() != DataTypes.TypeCategory.PRIMITIVE) {
+                Object aVal = i.get(attrName);
+                nodeProcessor.processNode(new Node(i, attrName, aInfo.dataType(), aVal));
+                traverseValue(aInfo.dataType(), aVal);
+            }
+        }
+    }
+
+    void visitReferenceableInstance(Object val) throws MetadataException {
+
+        if (val == null || !(val instanceof IReferenceableInstance)) {
+            return;
+        }
+
+        IReferenceableInstance ref = (IReferenceableInstance) val;
+
+        if (!processedIds.contains(ref.getId())) {
+            queue.add(ref);
+        }
+    }
+
+    void processReferenceableInstance(IReferenceableInstance ref) throws MetadataException {
+
+        nodeProcessor.processNode(new Node(ref, null, null, null));
+        visitStruct(ref);
+        ImmutableList<String> traits = ref.getTraits();
+        for (String trait : traits) {
+            visitStruct(ref.getTrait(trait));
+        }
+    }
+
+    public static interface NodeProcessor {
+
+        void processNode(Node nd) throws MetadataException;
+    }
+
+    /**
+     * Represents a non-primitive value of an instance.
+     */
+    public static class Node {
+        public final IStruct instance;
+        public final String attributeName;
+        public final IDataType dataType;
+        public final Object value;
+
+        public Node(IStruct instance, String attributeName, IDataType dataType, Object value) {
+            this.instance = instance;
+            this.attributeName = attributeName;
+            this.dataType = dataType;
+            this.value = value;
+        }
+    }
+}

src/main/scala/org/apache/metadata/cli/Console.scala

@@ -46,7 +46,7 @@ class SampleILoop extends ILoop {
   //intp = Console.in
 
   val ts: TypeSystem = new TypeSystem
-  val mr: MemRepository = new MemRepository
+  val mr: MemRepository = new MemRepository(ts)
   val ms : MetadataService = new MetadataService(mr, ts)
   MetadataService.setCurrentService(ms)
 

src/test/java/org/apache/metadata/BaseTest.java

@@ -41,7 +41,7 @@ public abstract class BaseTest {
     public void setup() throws MetadataException {
 
         TypeSystem ts = new TypeSystem();
-        MemRepository mr = new MemRepository();
+        MemRepository mr = new MemRepository(ts);
         ms = new MetadataService(mr, ts);
         MetadataService.setCurrentService(ms);