complete wiring of createInstance flow; add StorageTest

typesystem/src/main/java/org/apache/hadoop/metadata/storage/IRepository.java

@@ -20,8 +20,12 @@ package org.apache.hadoop.metadata.storage;
 import org.apache.hadoop.metadata.IReferenceableInstance;
 import org.apache.hadoop.metadata.ITypedInstance;
 import org.apache.hadoop.metadata.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.types.ClassType;
+import org.apache.hadoop.metadata.types.HierarchicalType;
+import org.apache.hadoop.metadata.types.TraitType;
 
 import java.text.DateFormat;
+import java.util.List;
 
 public interface IRepository {
 
@@ -38,4 +42,8 @@ public interface IRepository {
     Id newId(String typeName);
 
     ITypedInstance get(Id id) throws RepositoryException;
+
+    void defineClass(ClassType type) throws RepositoryException;
+    void defineTrait(TraitType type) throws RepositoryException;
+    void defineTypes(List<HierarchicalType> types) throws RepositoryException;
 }

typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/AttributeStores.java

@@ -31,6 +31,7 @@ import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.shorts.ShortArrayList;
 import org.apache.hadoop.metadata.ITypedStruct;
 import org.apache.hadoop.metadata.MetadataException;
+import org.apache.hadoop.metadata.storage.Id;
 import org.apache.hadoop.metadata.storage.StructInstance;
 import org.apache.hadoop.metadata.ITypedInstance;
 import org.apache.hadoop.metadata.storage.RepositoryException;
@@ -73,6 +74,8 @@ public class AttributeStores {
                     new DateStore(i);
                 } else if ( i.dataType() == DataTypes.STRING_TYPE ) {
                     new StringStore(i);
+                }  else if ( i.dataType() == DataTypes.STRING_TYPE ) {
+                    new StringStore(i);
                 } else {
                     throw new RepositoryException(String.format("Unknown datatype %s", i.dataType()));
                 }
@@ -80,6 +83,10 @@ public class AttributeStores {
                 return new ImmutableListStore(i);
             case MAP:
                 return new ImmutableMapStore(i);
+            case STRUCT:
+                return new StructStore(i);
+            case CLASS:
+                return new IdStore(i);
             default:
                 throw new RepositoryException(String.format("Unknown Category for datatype %s", i.dataType()));
         }
@@ -539,6 +546,30 @@ public class AttributeStores {
 
     }
 
+    static class IdStore extends ObjectAttributeStore<Id> {
+
+        public IdStore( AttributeInfo attrInfo) {
+            super(Id.class, attrInfo);
+        }
+
+        protected void store(StructInstance instance, int colPos, int pos) {
+            list.set(pos, instance.ids[colPos]);
+        }
+
+        protected void load(StructInstance instance, int colPos, int pos) {
+            instance.ids[colPos] = list.get(pos);
+        }
+
+        protected void store(StructInstance instance, int colPos, String attrName, Map<String, Object> m) {
+            m.put(attrName, instance.ids[colPos]);
+        }
+
+        protected void load(StructInstance instance, int colPos, Object val) {
+            instance.ids[colPos] = (Id) val;
+        }
+
+    }
+
     static class ImmutableListStore extends ObjectAttributeStore<ImmutableList> {
 
         public ImmutableListStore( AttributeInfo attrInfo) {

typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/HierarchicalTypeStore.java

@@ -128,6 +128,22 @@ public class HierarchicalTypeStore {
         }
     }
 
+    void acquireReadLock() {
+        lock.readLock().lock();
+    }
+
+    void acquireWriteLock() {
+        lock.writeLock().lock();
+    }
+
+    void releaseReadLock() {
+        lock.readLock().unlock();
+    }
+
+    void releaseWriteLock() {
+        lock.readLock().unlock();
+    }
+
     /**
      * - store the typeName
      * - store the immediate attributes in the respective IAttributeStore

typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/MemRepository.java

@@ -22,17 +22,11 @@ import org.apache.hadoop.metadata.IReferenceableInstance;
 import org.apache.hadoop.metadata.ITypedReferenceableInstance;
 import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.storage.*;
-import org.apache.hadoop.metadata.types.ClassType;
-import org.apache.hadoop.metadata.types.Multiplicity;
-import org.apache.hadoop.metadata.types.ObjectGraphWalker;
-import org.apache.hadoop.metadata.types.TypeSystem;
+import org.apache.hadoop.metadata.types.*;
 
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class MemRepository implements IRepository {
@@ -127,11 +121,20 @@ public class MemRepository implements IRepository {
         */
         List<ITypedReferenceableInstance> newInstances = new ArrayList<ITypedReferenceableInstance>();
         ITypedReferenceableInstance retInstance = null;
+        Set<ClassType> classTypes = new TreeSet<ClassType>();
+        Set<TraitType> traitTypes = new TreeSet<TraitType>();
         for(IReferenceableInstance transientInstance : discoverInstances.idToInstanceMap.values()) {
             try {
                 ClassType cT = typeSystem.getDataType(ClassType.class, transientInstance.getTypeName());
                 ITypedReferenceableInstance newInstance = cT.convert(transientInstance, Multiplicity.REQUIRED);
                 newInstances.add(newInstance);
+
+                classTypes.add(cT);
+                for(String traitName : newInstance.getTraits()) {
+                    TraitType tT = typeSystem.getDataType(TraitType.class, traitName);
+                    traitTypes.add(tT);
+                }
+
                 if (newInstance.getId() == i.getId()) {
                     retInstance = newInstance;
                 }
@@ -149,7 +152,22 @@ public class MemRepository implements IRepository {
         }
 
         /*
-         * 3. Traverse over newInstances
+         * 3. Acquire Class and Trait Storage locks.
+         * - acquire them in a stable order (super before subclass, classes before traits
+         */
+        for(ClassType cT : classTypes) {
+            HierarchicalTypeStore st = typeStores.get(cT.getName());
+            st.acquireWriteLock();
+        }
+
+        for(TraitType tT : traitTypes) {
+            HierarchicalTypeStore st = typeStores.get(tT.getName());
+            st.acquireWriteLock();
+        }
+
+
+        /*
+         * 4. Traverse over newInstances
          *    - ask ClassStore to assign a position to the Id.
          *      - for Instances with Traits, assign a position for each Trait
          *    - invoke store on the nwInstance.
@@ -170,6 +188,16 @@ public class MemRepository implements IRepository {
                 st.releaseId(instance.getId());
             }
             throw re;
+        } finally {
+            for(ClassType cT : classTypes) {
+                HierarchicalTypeStore st = typeStores.get(cT.getName());
+                st.releaseWriteLock();
+            }
+
+            for(TraitType tT : traitTypes) {
+                HierarchicalTypeStore st = typeStores.get(tT.getName());
+                st.releaseWriteLock();
+            }
         }
 
         return retInstance;
@@ -188,6 +216,40 @@ public class MemRepository implements IRepository {
     }
 
     HierarchicalTypeStore getStore(String typeName) {
-        return null;
+        return typeStores.get(typeName);
+    }
+
+    public void defineClass(ClassType type) throws RepositoryException {
+        HierarchicalTypeStore s = new HierarchicalTypeStore(this, type);
+        typeStores.put(type.getName(), s);
+    }
+
+    public void defineTrait(TraitType type) throws RepositoryException {
+        HierarchicalTypeStore s = new HierarchicalTypeStore(this, type);
+        typeStores.put(type.getName(), s);
+    }
+
+    public void defineTypes(List<HierarchicalType> types) throws RepositoryException {
+        List<TraitType> tTypes = new ArrayList<TraitType>();
+        List<ClassType> cTypes = new ArrayList<ClassType>();
+
+        for(HierarchicalType h : types) {
+            if ( h.getTypeCategory() == DataTypes.TypeCategory.TRAIT ) {
+                tTypes.add((TraitType) h);
+            } else {
+                cTypes.add((ClassType)h);
+            }
+        }
+
+        Collections.sort(tTypes);
+        Collections.sort(cTypes);
+
+        for(TraitType tT : tTypes) {
+            defineTrait(tT);
+        }
+
+        for(ClassType cT : cTypes) {
+            defineClass(cT);
+        }
     }
 }

typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/StructStore.java

@@ -38,7 +38,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-public abstract class StructStore extends AttributeStores.AbstractAttributeStore implements IAttributeStore {
+public class StructStore extends AttributeStores.AbstractAttributeStore implements IAttributeStore {
 
     final StructType structType;
     final ImmutableMap<AttributeInfo, IAttributeStore> attrStores;
@@ -56,6 +56,7 @@ public abstract class StructStore extends AttributeStores.AbstractAttributeStore
 
     }
 
+    @Override
     protected void store(StructInstance instance, int colPos, int pos) throws RepositoryException {
         StructInstance s = instance.structs[colPos];
         for(Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
@@ -64,6 +65,7 @@ public abstract class StructStore extends AttributeStores.AbstractAttributeStore
         }
     }
 
+    @Override
     protected void load(StructInstance instance, int colPos, int pos) throws RepositoryException {
         for(Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
             IAttributeStore attributeStore = e.getValue();
@@ -71,10 +73,12 @@ public abstract class StructStore extends AttributeStores.AbstractAttributeStore
         }
     }
 
+    @Override
     protected void store(StructInstance instance, int colPos, String attrName, Map<String, Object> m) {
         m.put(attrName, instance.structs[colPos]);
     }
 
+    @Override
     protected void load(StructInstance instance, int colPos, Object val) {
         instance.structs[colPos] = (StructInstance) val;
     }

typesystem/src/test/java/org/apache/hadoop/metadata/BaseTest.java

@@ -140,4 +140,77 @@ public abstract class BaseTest {
         return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);
     }
 
+    /*
+     * Class Hierarchy is:
+     *   Department(name : String, employees : Array[Person])
+     *   Person(name : String, department : Department, manager : Manager)
+     *   Manager(subordinates : Array[Person]) extends Person
+     *
+     * Persons can have SecurityClearance(level : Int) clearance.
+     */
+    protected void defineDeptEmployeeTypes(TypeSystem ts) throws MetadataException {
+
+        HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", ImmutableList.<String>of(),
+                createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                new AttributeDefinition("employees",
+                        String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")
+        );
+        HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableList.<String>of(),
+                createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                new AttributeDefinition("department",
+                        "Department", Multiplicity.REQUIRED, false, "employees"),
+                new AttributeDefinition("manager",
+                        "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+        );
+        HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager",
+                ImmutableList.<String>of("Person"),
+                new AttributeDefinition("subordinates",
+                        String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")
+        );
+
+        HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef("SecurityClearance",
+                ImmutableList.<String>of(),
+                createRequiredAttrDef("level", DataTypes.INT_TYPE)
+        );
+
+        ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
+                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
+                ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef, managerTypeDef));
+
+        ImmutableList<HierarchicalType> types = ImmutableList.of(
+                ts.getDataType(HierarchicalType.class, "SecurityClearance"),
+                ts.getDataType(ClassType.class, "Department"),
+                ts.getDataType(ClassType.class, "Person"),
+                ts.getDataType(ClassType.class, "Manager")
+        );
+
+        ms.getRepository().defineTypes(types);
+
+    }
+
+    protected Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
+        Referenceable hrDept = new Referenceable("Department");
+        Referenceable john = new Referenceable("Person");
+        Referenceable jane = new Referenceable("Manager", "SecurityClearance");
+
+        hrDept.set("name", "hr");
+        john.set("name", "John");
+        john.set("department", hrDept);
+        jane.set("name", "Jane");
+        jane.set("department", hrDept);
+
+        john.set("manager", jane);
+
+        hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
+
+        jane.set("subordinates", ImmutableList.<Referenceable>of(john));
+
+        jane.getTrait("SecurityClearance").set("level", 1);
+
+        ClassType deptType = ts.getDataType(ClassType.class, "Department");
+        ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+        return hrDept;
+    }
+
 }

typesystem/src/test/java/org/apache/hadoop/metadata/ClassTest.java

@@ -9,73 +9,22 @@ import org.junit.Test;
 
 public class ClassTest extends BaseTest {
 
-
     @Before
     public void setup() throws MetadataException {
         super.setup();
     }
 
-    /*
-     * Class Hierarchy is:
-     *   Department(name : String, employees : Array[Person])
-     *   Person(name : String, department : Department, manager : Manager)
-     *   Manager(subordinates : Array[Person]) extends Person
-     *
-     * Persons can have SecurityClearance(level : Int) clearance.
-     */
     @Test
     public void test1() throws MetadataException {
 
         TypeSystem ts = ms.getTypeSystem();
 
-        HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", ImmutableList.<String>of(),
-                createRequiredAttrDef("name", DataTypes.STRING_TYPE),
-                new AttributeDefinition("employees",
-                        String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")
-        );
-        HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableList.<String>of(),
-                createRequiredAttrDef("name", DataTypes.STRING_TYPE),
-                new AttributeDefinition("department",
-                        "Department", Multiplicity.REQUIRED, false, "employees"),
-                new AttributeDefinition("manager",
-                        "Manager", Multiplicity.OPTIONAL, false, "subordinates")
-        );
-        HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager",
-                ImmutableList.<String>of("Person"),
-                new AttributeDefinition("subordinates",
-                        String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")
-        );
-
-        HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef("SecurityClearance",
-                ImmutableList.<String>of(),
-                createRequiredAttrDef("level", DataTypes.INT_TYPE)
-        );
-
-        ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
-                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
-                ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef, managerTypeDef));
-
-        Referenceable hrDept = new Referenceable("Department");
-        Referenceable john = new Referenceable("Person");
-        Referenceable jane = new Referenceable("Manager", "SecurityClearance");
-
-        hrDept.set("name", "hr");
-        john.set("name", "John");
-        john.set("department", hrDept);
-        jane.set("name", "Jane");
-        jane.set("department", hrDept);
-
-        john.set("manager", jane);
-
-        hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
-
-        jane.set("subordinates", ImmutableList.<Referenceable>of(john));
-
-        jane.getTrait("SecurityClearance").set("level", 1);
-
+        defineDeptEmployeeTypes(ts);
+        Referenceable hrDept = createDeptEg1(ts);
         ClassType deptType = ts.getDataType(ClassType.class, "Department");
         ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
 
+
         Assert.assertEquals(hrDept2.toString(), "{\n" +
                 "\tid : (type: Department, id: <unassigned>)\n" +
                 "\tname : \thr\n" +

typesystem/src/test/java/org/apache/hadoop/metadata/StorageTest.java

+package org.apache.hadoop.metadata;
+
+import org.apache.hadoop.metadata.types.TypeSystem;
+import org.junit.Before;
+import org.junit.Test;
+
+public class StorageTest extends BaseTest {
+
+    @Before
+    public void setup() throws MetadataException {
+        super.setup();
+    }
+
+    @Test
+    public void test1() throws MetadataException {
+
+        TypeSystem ts = ms.getTypeSystem();
+
+        defineDeptEmployeeTypes(ts);
+
+
+        Referenceable hrDept = createDeptEg1(ts);
+
+        //ITypedReferenceableInstance hrDept2 = ms.getRepository().create(hrDept);
+
+    }
+
+}
\ No newline at end of file