flush out MemRepository create call

src/main/java/org/apache/metadata/storage/memory/DiscoverInstances.java → src/main/java/org/apache/metadata/storage/DiscoverInstances.java

-package org.apache.metadata.storage.memory;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
+package org.apache.metadata.storage;
 
 import org.apache.metadata.IReferenceableInstance;
 import org.apache.metadata.MetadataException;
 import org.apache.metadata.storage.Id;
 import org.apache.metadata.storage.RepositoryException;
+import org.apache.metadata.storage.memory.MemRepository;
 import org.apache.metadata.types.DataTypes;
 import org.apache.metadata.types.ObjectGraphWalker;
 
@@ -13,12 +31,12 @@ import java.util.Map;
 
 public class DiscoverInstances implements ObjectGraphWalker.NodeProcessor {
 
-    final MemRepository memRepository;
-    final Map<Id, Id> idToNewIdMap;
-    final Map<Id, IReferenceableInstance> idToInstanceMap;
+    final IRepository repository;
+    public final Map<Id, Id> idToNewIdMap;
+    public final Map<Id, IReferenceableInstance> idToInstanceMap;
 
-    public DiscoverInstances(MemRepository memRepository) {
-        this.memRepository = memRepository;
+    public DiscoverInstances(IRepository repository) {
+        this.repository = repository;
         idToNewIdMap = new HashMap<Id, Id>();
         idToInstanceMap = new HashMap<Id, IReferenceableInstance>();
     }
@@ -32,7 +50,7 @@ public class DiscoverInstances implements ObjectGraphWalker.NodeProcessor {
         if ( nd.attributeName == null ) {
             ref = (IReferenceableInstance) nd.instance;
             id = ref.getId();
-        } else if ( nd.dataType.getTypeCategory() == DataTypes.TypeCategory.CLASS ) {
+        } else if ( nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS ) {
             if ( nd.value != null && (nd.value instanceof  Id)) {
                 id = (Id) nd.value;
             }
@@ -41,7 +59,7 @@ public class DiscoverInstances implements ObjectGraphWalker.NodeProcessor {
         if ( id != null ) {
             if ( id.isUnassigned() ) {
                 if ( !idToNewIdMap.containsKey(id)) {
-                    idToNewIdMap.put(id, memRepository.newId(id.className));
+                    idToNewIdMap.put(id, repository.newId(id.className));
                 }
                 if ( idToInstanceMap.containsKey(ref)) {
                     // Oops

src/main/java/org/apache/metadata/storage/IRepository.java

@@ -33,5 +33,7 @@ public interface IRepository {
 
     void delete(ITypedReferenceableInstance i) throws RepositoryException;
 
+    Id newId(String typeName);
+
     ITypedInstance get(Id id) throws RepositoryException;
 }

src/main/java/org/apache/metadata/storage/Id.java

@@ -18,7 +18,12 @@
 
 package org.apache.metadata.storage;
 
-public class Id {
+import com.google.common.collect.ImmutableList;
+import org.apache.metadata.IReferenceableInstance;
+import org.apache.metadata.IStruct;
+import org.apache.metadata.MetadataException;
+
+public class Id implements IReferenceableInstance {
 
     public final long id;
     public final String className;
@@ -41,4 +46,34 @@ public class Id {
     public String toString() {
         return String.format("(type: %s, id: %s)", className, isUnassigned() ? "<unassigned>" : "" + id);
     }
+
+    @Override
+    public ImmutableList<String> getTraits() {
+        return null;
+    }
+
+    @Override
+    public Id getId() {
+        return this;
+    }
+
+    @Override
+    public IStruct getTrait(String typeName) {
+        return null;
+    }
+
+    @Override
+    public String getTypeName() {
+        return className;
+    }
+
+    @Override
+    public Object get(String attrName) throws MetadataException {
+        throw new MetadataException("Get/Set not supported on an Id object");
+    }
+
+    @Override
+    public void set(String attrName, Object val) throws MetadataException {
+        throw new MetadataException("Get/Set not supported on an Id object");
+    }
 }

src/main/java/org/apache/metadata/storage/MapIds.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metadata.storage;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMap;
+import org.apache.metadata.IReferenceableInstance;
+import org.apache.metadata.MetadataException;
+import org.apache.metadata.types.DataTypes;
+import org.apache.metadata.types.ObjectGraphWalker;
+
+import java.util.Map;
+
+public class MapIds implements ObjectGraphWalker.NodeProcessor {
+
+    final IRepository repository;
+    final Map<Id, Id> idToNewIdMap;
+
+    public MapIds(IRepository repository, Map<Id, Id> idToNewIdMap) {
+        this.repository = repository;
+        this.idToNewIdMap = idToNewIdMap;
+    }
+
+    @Override
+    public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
+
+        IReferenceableInstance ref =  null;
+        Id id = null;
+
+        if ( nd.attributeName == null ) {
+            ref = (IReferenceableInstance) nd.instance;
+            Id newId = idToNewIdMap.get(ref.getId());
+            if ( newId != null ) {
+                ((ReferenceableInstance)ref).replaceWithNewId(newId);
+            }
+        } else if ( nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS ) {
+            if ( nd.value != null && nd.value instanceof  IReferenceableInstance ) {
+                Id oldId = ((IReferenceableInstance)nd.value).getId();
+                Id newId = idToNewIdMap.get(ref.getId());
+                /*
+                 * Replace Instances with Ids, irrespective of whether they map to newIds or not.
+                 */
+                newId = newId == null ? oldId : newId;
+                nd.instance.set(nd.attributeName, newId);
+            }
+        } else if ( nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY ) {
+            DataTypes.ArrayType aT = (DataTypes.ArrayType) nd.aInfo.dataType();
+            Object v = aT.mapIds((ImmutableCollection)nd.value, nd.aInfo.multiplicity, idToNewIdMap);
+            nd.instance.set(nd.attributeName, v);
+        }  else if ( nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP ) {
+            DataTypes.MapType mT = (DataTypes.MapType) nd.aInfo.dataType();
+            Object v = mT.mapIds((ImmutableMap)nd.value, nd.aInfo.multiplicity, idToNewIdMap);
+            nd.instance.set(nd.attributeName, v);
+        }
+    }
+}

src/main/java/org/apache/metadata/storage/memory/AttributeStores.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.metadata.storage.memory;
 
 import it.unimi.dsi.fastutil.booleans.BooleanArrayList;

src/main/java/org/apache/metadata/storage/memory/HierarchicalTypeStore.java

-package org.apache.metadata.storage.memory;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
+package org.apache.metadata.storage.memory;
 
 import com.google.common.collect.ImmutableBiMap;
 import com.google.common.collect.ImmutableList;
@@ -71,8 +88,8 @@ public class HierarchicalTypeStore {
      * - add to freePositions.
      * @throws RepositoryException
      */
-    void releaseId() throws RepositoryException {
-        throw new RepositoryException("Not implemented");
+    void releaseId(Id id) {
+        throw new RuntimeException("Not implemented");
     }
 
     /**

src/main/java/org/apache/metadata/storage/memory/MemRepository.java

@@ -19,16 +19,14 @@
 package org.apache.metadata.storage.memory;
 
 import org.apache.metadata.*;
-import org.apache.metadata.storage.IRepository;
-import org.apache.metadata.storage.Id;
-import org.apache.metadata.storage.RepositoryException;
-import org.apache.metadata.types.ObjectGraphTraversal;
-import org.apache.metadata.types.ObjectGraphWalker;
-import org.apache.metadata.types.TypeSystem;
+import org.apache.metadata.storage.*;
+import org.apache.metadata.types.*;
 
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -38,10 +36,15 @@ public class MemRepository implements IRepository {
     private static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
     final TypeSystem typeSystem;
+    /*
+     * A Store for each Class and Trait.
+     */
+    final Map<String, HierarchicalTypeStore> typeStores;
     final AtomicInteger ID_SEQ = new AtomicInteger(0);
 
     public MemRepository(TypeSystem typeSystem) {
         this.typeSystem = typeSystem;
+        this.typeStores = new HashMap<String, HierarchicalTypeStore>();
     }
 
     @Override
@@ -59,7 +62,8 @@ public class MemRepository implements IRepository {
         return false;
     }
 
-    Id newId(String typeName) {
+    @Override
+    public Id newId(String typeName) {
         return new Id(ID_SEQ.incrementAndGet(), 0, typeName);
     }
 
@@ -89,14 +93,81 @@ public class MemRepository implements IRepository {
 
         DiscoverInstances discoverInstances = new DiscoverInstances(this);
 
+        /*
+         * Step 1: traverse the Object Graph from  i and create idToNewIdMap : Map[Id, Id],
+         *    also create old Id to Instance Map: oldIdToInstance : Map[Id, IInstance]
+         *   - traverse reference Attributes, List[ClassType], Maps where Key/value is ClassType
+         *   - traverse Structs
+         *   - traverse Traits.
+         */
         try {
             new ObjectGraphWalker(typeSystem, discoverInstances, i).walk();
         } catch (MetadataException me) {
             throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
         }
 
+        /*
+         * Step 1b: Ensure that every newId has an associated Instance.
+         */
+        for(Id oldId : discoverInstances.idToNewIdMap.keySet()) {
+            if ( !discoverInstances.idToInstanceMap.containsKey(oldId)) {
+                throw new RepositoryException(String.format("Invalid Object Graph: " +
+                        "Encountered an unassignedId %s that is not associated with an Instance", oldId));
+            }
+        }
 
-        throw new RepositoryException("not implemented");
+        /* Step 2: Traverse oldIdToInstance map create newInstances : List[ITypedReferenceableInstance]
+         * - create a ITypedReferenceableInstance.
+         *   replace any old References ( ids or object references) with new Ids.
+        */
+        List<ITypedReferenceableInstance> newInstances = new ArrayList<ITypedReferenceableInstance>();
+        ITypedReferenceableInstance retInstance = null;
+        for(IReferenceableInstance transientInstance : discoverInstances.idToInstanceMap.values()) {
+            try {
+                ClassType cT = typeSystem.getDataType(ClassType.class, transientInstance.getTypeName());
+                ITypedReferenceableInstance newInstance = cT.convert(transientInstance, Multiplicity.REQUIRED);
+                newInstances.add(newInstance);
+                if (newInstance.getId() == i.getId()) {
+                    retInstance = newInstance;
+                }
+
+                /*
+                 * Now replace old references with new Ids
+                 */
+                MapIds mapIds = new MapIds(this, discoverInstances.idToNewIdMap);
+                new ObjectGraphWalker(typeSystem, mapIds, newInstances).walk();
+
+            } catch (MetadataException me) {
+                throw new RepositoryException(
+                        String.format("Failed to create Instance(id = %s", transientInstance.getId()), me);
+            }
+        }
+
+        /*
+         * 3. Traverse over newInstances
+         *    - ask ClassStore to assign a position to the Id.
+         *      - for Instances with Traits, assign a position for each Trait
+         *    - invoke store on the nwInstance.
+         */
+        try {
+            for (ITypedReferenceableInstance instance : newInstances) {
+                HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
+                st.assignPosition(instance.getId());
+            }
+
+            for (ITypedReferenceableInstance instance : newInstances) {
+                HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
+                st.store(instance);
+            }
+        } catch(RepositoryException re) {
+            for (ITypedReferenceableInstance instance : newInstances) {
+                HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
+                st.releaseId(instance.getId());
+            }
+            throw re;
+        }
+
+        return retInstance;
     }
 
     public ITypedReferenceableInstance update(ITypedReferenceableInstance i) throws RepositoryException {

src/main/java/org/apache/metadata/storage/memory/StructStore.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.metadata.storage.memory;
 
 import com.google.common.collect.ImmutableBiMap;

src/main/java/org/apache/metadata/types/ClassType.java

@@ -79,7 +79,21 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
     public ITypedReferenceableInstance convert(Object val, Multiplicity m) throws MetadataException {
 
         if ( val != null ) {
-            if ( val instanceof Struct) {
+            if ( val instanceof  ITypedReferenceableInstance) {
+                ITypedReferenceableInstance tr = (ITypedReferenceableInstance) val;
+                if ( tr.getTypeName() != getName() ) {
+                     /*
+                     * If val is a subType instance; invoke convert on it.
+                     */
+                    ClassType valType = typeSystem.getDataType(superTypeClass, tr.getTypeName());
+                    if ( valType.superTypePaths.containsKey(name) ) {
+                        return valType.convert(val, m);
+                    }
+                    throw new ValueConversionException(this, val);
+                }
+                return tr;
+            }
+            else if ( val instanceof Struct) {
                 Struct s = (Struct) val;
                 Referenceable r = null;
                 Id id = null;

src/main/java/org/apache/metadata/types/DataTypes.java

@@ -23,9 +23,11 @@ import com.google.common.collect.ImmutableCollection.Builder;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import org.apache.metadata.IReferenceableInstance;
 import org.apache.metadata.storage.IRepository;
 import org.apache.metadata.MetadataException;
 import org.apache.metadata.MetadataService;
+import org.apache.metadata.storage.Id;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -439,6 +441,27 @@ public class DataTypes {
             return null;
         }
 
+        public ImmutableCollection<?> mapIds(ImmutableCollection<?> val, Multiplicity m, Map<Id, Id> transientToNewIds)
+                throws MetadataException {
+
+            if ( val == null || elemType.getTypeCategory() != TypeCategory.CLASS ) {
+                return val;
+            }
+            ImmutableCollection.Builder<?> b = m.isUnique ? ImmutableSet.builder() : ImmutableList.builder();
+            Iterator it = val.iterator();
+            while(it.hasNext()) {
+                Object elem = it.next();
+                if ( elem instanceof IReferenceableInstance) {
+                    Id oldId = ((IReferenceableInstance)elem).getId();
+                    Id newId = transientToNewIds.get(oldId);
+                    b.add(newId == null ? oldId : newId);
+                } else {
+                    b.add(elem);
+                }
+            }
+            return b.build();
+        }
+
         @Override
         public TypeCategory getTypeCategory() {
             return TypeCategory.ARRAY;
@@ -510,6 +533,39 @@ public class DataTypes {
             return null;
         }
 
+        public ImmutableMap<?, ?> mapIds(ImmutableMap val, Multiplicity m, Map<Id, Id> transientToNewIds)
+                throws MetadataException {
+
+            if ( val == null || (keyType.getTypeCategory() != TypeCategory.CLASS &&
+                    valueType.getTypeCategory() != TypeCategory.CLASS) ) {
+                return val;
+            }
+            ImmutableMap.Builder b = ImmutableMap.builder();
+            Iterator<Map.Entry> it = val.entrySet().iterator();
+            while(it.hasNext()) {
+                Map.Entry elem = it.next();
+                Object oldKey = elem.getKey();
+                Object oldValue = elem.getValue();
+                Object newKey = oldKey;
+                Object newValue = oldValue;
+
+                if ( oldKey instanceof IReferenceableInstance ) {
+                    Id oldId = ((IReferenceableInstance)oldKey).getId();
+                    Id newId = transientToNewIds.get(oldId);
+                    newKey = newId == null ? oldId : newId;
+                }
+
+                if ( oldValue instanceof IReferenceableInstance ) {
+                    Id oldId = ((IReferenceableInstance)oldValue).getId();
+                    Id newId = transientToNewIds.get(oldId);
+                    newValue = newId == null ? oldId : newId;
+                }
+
+                b.put(newKey, newValue);
+            }
+            return b.build();
+        }
+
         @Override
         public TypeCategory getTypeCategory() {
             return TypeCategory.MAP;

src/main/java/org/apache/metadata/types/IDataType.java

@@ -17,9 +17,12 @@
  */
 package org.apache.metadata.types;
 
+import org.apache.metadata.storage.Id;
 import org.apache.metadata.types.DataTypes.TypeCategory;
 import org.apache.metadata.MetadataException;
 
+import java.util.Map;
+
 public interface IDataType<T> {
     String getName();
     T convert(Object val, Multiplicity m) throws MetadataException;

src/main/java/org/apache/metadata/types/ObjectGraphWalker.java

@@ -31,6 +31,16 @@ public class ObjectGraphWalker {
         queue.add(start);
     }
 
+    public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor,
+                             List<? extends IReferenceableInstance> roots)
+            throws MetadataException {
+        this.typeSystem = typeSystem;
+        this.nodeProcessor = nodeProcessor;
+        queue = new LinkedList<IReferenceableInstance>();
+        processedIds = new HashSet<Id>();
+        queue.addAll(roots);
+    }
+
     public void walk() throws MetadataException {
         while (!queue.isEmpty()) {
             IReferenceableInstance r = queue.poll();
@@ -116,7 +126,7 @@ public class ObjectGraphWalker {
             String attrName = e.getKey();
             if (aInfo.dataType().getTypeCategory() != DataTypes.TypeCategory.PRIMITIVE) {
                 Object aVal = i.get(attrName);
-                nodeProcessor.processNode(new Node(i, attrName, aInfo.dataType(), aVal));
+                nodeProcessor.processNode(new Node(i, attrName, aInfo, aVal));
                 traverseValue(aInfo.dataType(), aVal);
             }
         }
@@ -156,13 +166,13 @@ public class ObjectGraphWalker {
     public static class Node {
         public final IStruct instance;
         public final String attributeName;
-        public final IDataType dataType;
+        public final AttributeInfo aInfo;
         public final Object value;
 
-        public Node(IStruct instance, String attributeName, IDataType dataType, Object value) {
+        public Node(IStruct instance, String attributeName, AttributeInfo aInfo, Object value) {
             this.instance = instance;
             this.attributeName = attributeName;
-            this.dataType = dataType;
+            this.aInfo = aInfo;
             this.value = value;
         }
     }

src/main/java/org/apache/metadata/types/TypedStructHandler.java

@@ -45,7 +45,14 @@ public class TypedStructHandler {
 
     public ITypedStruct convert(Object val, Multiplicity m) throws MetadataException {
         if ( val != null ) {
-            if ( val instanceof Struct) {
+            if ( val instanceof  ITypedStruct ) {
+                ITypedStruct ts = (ITypedStruct) val;
+                if ( ts.getTypeName() != structType.getName() ) {
+                    throw new ValueConversionException(structType, val);
+                }
+                return ts;
+            }
+            else if ( val instanceof Struct) {
                 Struct s = (Struct) val;
                 if ( s.typeName != structType.getName() ) {
                     throw new ValueConversionException(structType, val);