Skip to content

A
atlas
Commit 2810310f by Harish Butani
Options

complete wiring of createInstance flow; add StorageTest

parent bf00ea91
Showing with 237 additions and 65 deletions
typesystem/src/main/java/org/apache/hadoop/metadata/storage/IRepository.java
......@@ -20,8 +20,12 @@ package org.apache.hadoop.metadata.storage;
import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.ITypedInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.HierarchicalType;
import org.apache.hadoop.metadata.types.TraitType;
import java.text.DateFormat;
import java.util.List;
public interface IRepository {
......@@ -38,4 +42,8 @@ public interface IRepository {
Id newId(String typeName);
ITypedInstance get(Id id) throws RepositoryException;
void defineClass(ClassType type) throws RepositoryException;
void defineTrait(TraitType type) throws RepositoryException;
void defineTypes(List<HierarchicalType> types) throws RepositoryException;
}
typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/AttributeStores.java
......@@ -31,6 +31,7 @@ import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import org.apache.hadoop.metadata.ITypedStruct;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.storage.Id;
import org.apache.hadoop.metadata.storage.StructInstance;
import org.apache.hadoop.metadata.ITypedInstance;
import org.apache.hadoop.metadata.storage.RepositoryException;
......@@ -73,6 +74,8 @@ public class AttributeStores {
new DateStore(i);
} else if ( i.dataType() == DataTypes.STRING_TYPE ) {
new StringStore(i);
} else if ( i.dataType() == DataTypes.STRING_TYPE ) {
new StringStore(i);
} else {
throw new RepositoryException(String.format("Unknown datatype %s", i.dataType()));
}
......@@ -80,6 +83,10 @@ public class AttributeStores {
return new ImmutableListStore(i);
case MAP:
return new ImmutableMapStore(i);
case STRUCT:
return new StructStore(i);
case CLASS:
return new IdStore(i);
default:
throw new RepositoryException(String.format("Unknown Category for datatype %s", i.dataType()));
}
......@@ -539,6 +546,30 @@ public class AttributeStores {
}
static class IdStore extends ObjectAttributeStore<Id> {
public IdStore( AttributeInfo attrInfo) {
super(Id.class, attrInfo);
}
protected void store(StructInstance instance, int colPos, int pos) {
list.set(pos, instance.ids[colPos]);
}
protected void load(StructInstance instance, int colPos, int pos) {
instance.ids[colPos] = list.get(pos);
}
protected void store(StructInstance instance, int colPos, String attrName, Map<String, Object> m) {
m.put(attrName, instance.ids[colPos]);
}
protected void load(StructInstance instance, int colPos, Object val) {
instance.ids[colPos] = (Id) val;
}
}
static class ImmutableListStore extends ObjectAttributeStore<ImmutableList> {
public ImmutableListStore( AttributeInfo attrInfo) {
......
typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/HierarchicalTypeStore.java
......@@ -128,6 +128,22 @@ public class HierarchicalTypeStore {
}
}
void acquireReadLock() {
lock.readLock().lock();
}
void acquireWriteLock() {
lock.writeLock().lock();
}
void releaseReadLock() {
lock.readLock().unlock();
}
void releaseWriteLock() {
lock.readLock().unlock();
}
/**
* - store the typeName
* - store the immediate attributes in the respective IAttributeStore
......
typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/MemRepository.java
......@@ -22,17 +22,11 @@ import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.storage.*;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.ObjectGraphWalker;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.apache.hadoop.metadata.types.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
public class MemRepository implements IRepository {
......@@ -127,11 +121,20 @@ public class MemRepository implements IRepository {
*/
List<ITypedReferenceableInstance> newInstances = new ArrayList<ITypedReferenceableInstance>();
ITypedReferenceableInstance retInstance = null;
Set<ClassType> classTypes = new TreeSet<ClassType>();
Set<TraitType> traitTypes = new TreeSet<TraitType>();
for(IReferenceableInstance transientInstance : discoverInstances.idToInstanceMap.values()) {
try {
ClassType cT = typeSystem.getDataType(ClassType.class, transientInstance.getTypeName());
ITypedReferenceableInstance newInstance = cT.convert(transientInstance, Multiplicity.REQUIRED);
newInstances.add(newInstance);
classTypes.add(cT);
for(String traitName : newInstance.getTraits()) {
TraitType tT = typeSystem.getDataType(TraitType.class, traitName);
traitTypes.add(tT);
}
if (newInstance.getId() == i.getId()) {
retInstance = newInstance;
}
......@@ -149,7 +152,22 @@ public class MemRepository implements IRepository {
}
/*
* 3. Traverse over newInstances
* 3. Acquire Class and Trait Storage locks.
* - acquire them in a stable order (super before subclass, classes before traits
*/
for(ClassType cT : classTypes) {
HierarchicalTypeStore st = typeStores.get(cT.getName());
st.acquireWriteLock();
}
for(TraitType tT : traitTypes) {
HierarchicalTypeStore st = typeStores.get(tT.getName());
st.acquireWriteLock();
}
/*
* 4. Traverse over newInstances
* - ask ClassStore to assign a position to the Id.
* - for Instances with Traits, assign a position for each Trait
* - invoke store on the nwInstance.
......@@ -170,6 +188,16 @@ public class MemRepository implements IRepository {
st.releaseId(instance.getId());
}
throw re;
} finally {
for(ClassType cT : classTypes) {
HierarchicalTypeStore st = typeStores.get(cT.getName());
st.releaseWriteLock();
}
for(TraitType tT : traitTypes) {
HierarchicalTypeStore st = typeStores.get(tT.getName());
st.releaseWriteLock();
}
}
return retInstance;
......@@ -188,6 +216,40 @@ public class MemRepository implements IRepository {
}
HierarchicalTypeStore getStore(String typeName) {
return null;
return typeStores.get(typeName);
}
public void defineClass(ClassType type) throws RepositoryException {
HierarchicalTypeStore s = new HierarchicalTypeStore(this, type);
typeStores.put(type.getName(), s);
}
public void defineTrait(TraitType type) throws RepositoryException {
HierarchicalTypeStore s = new HierarchicalTypeStore(this, type);
typeStores.put(type.getName(), s);
}
public void defineTypes(List<HierarchicalType> types) throws RepositoryException {
List<TraitType> tTypes = new ArrayList<TraitType>();
List<ClassType> cTypes = new ArrayList<ClassType>();
for(HierarchicalType h : types) {
if ( h.getTypeCategory() == DataTypes.TypeCategory.TRAIT ) {
tTypes.add((TraitType) h);
} else {
cTypes.add((ClassType)h);
}
}
Collections.sort(tTypes);
Collections.sort(cTypes);
for(TraitType tT : tTypes) {
defineTrait(tT);
}
for(ClassType cT : cTypes) {
defineClass(cT);
}
}
}
typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/StructStore.java
......@@ -38,7 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
public abstract class StructStore extends AttributeStores.AbstractAttributeStore implements IAttributeStore {
public class StructStore extends AttributeStores.AbstractAttributeStore implements IAttributeStore {
final StructType structType;
final ImmutableMap<AttributeInfo, IAttributeStore> attrStores;
......@@ -56,6 +56,7 @@ public abstract class StructStore extends AttributeStores.AbstractAttributeStore
}
@Override
protected void store(StructInstance instance, int colPos, int pos) throws RepositoryException {
StructInstance s = instance.structs[colPos];
for(Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
......@@ -64,6 +65,7 @@ public abstract class StructStore extends AttributeStores.AbstractAttributeStore
}
}
@Override
protected void load(StructInstance instance, int colPos, int pos) throws RepositoryException {
for(Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
IAttributeStore attributeStore = e.getValue();
......@@ -71,10 +73,12 @@ public abstract class StructStore extends AttributeStores.AbstractAttributeStore
}
}
@Override
protected void store(StructInstance instance, int colPos, String attrName, Map<String, Object> m) {
m.put(attrName, instance.structs[colPos]);
}
@Override
protected void load(StructInstance instance, int colPos, Object val) {
instance.structs[colPos] = (StructInstance) val;
}
......
typesystem/src/test/java/org/apache/hadoop/metadata/BaseTest.java
......@@ -140,4 +140,77 @@ public abstract class BaseTest {
return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);
}
/*
* Class Hierarchy is:
* Department(name : String, employees : Array[Person])
* Person(name : String, department : Department, manager : Manager)
* Manager(subordinates : Array[Person]) extends Person
*
* Persons can have SecurityClearance(level : Int) clearance.
*/
protected void defineDeptEmployeeTypes(TypeSystem ts) throws MetadataException {
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees",
String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")
);
HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("department",
"Department", Multiplicity.REQUIRED, false, "employees"),
new AttributeDefinition("manager",
"Manager", Multiplicity.OPTIONAL, false, "subordinates")
);
HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager",
ImmutableList.<String>of("Person"),
new AttributeDefinition("subordinates",
String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")
);
HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef("SecurityClearance",
ImmutableList.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE)
);
ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef, managerTypeDef));
ImmutableList<HierarchicalType> types = ImmutableList.of(
ts.getDataType(HierarchicalType.class, "SecurityClearance"),
ts.getDataType(ClassType.class, "Department"),
ts.getDataType(ClassType.class, "Person"),
ts.getDataType(ClassType.class, "Manager")
);
ms.getRepository().defineTypes(types);
}
protected Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
Referenceable hrDept = new Referenceable("Department");
Referenceable john = new Referenceable("Person");
Referenceable jane = new Referenceable("Manager", "SecurityClearance");
hrDept.set("name", "hr");
john.set("name", "John");
john.set("department", hrDept);
jane.set("name", "Jane");
jane.set("department", hrDept);
john.set("manager", jane);
hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
jane.set("subordinates", ImmutableList.<Referenceable>of(john));
jane.getTrait("SecurityClearance").set("level", 1);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
return hrDept;
}
}
typesystem/src/test/java/org/apache/hadoop/metadata/ClassTest.java
......@@ -9,73 +9,22 @@ import org.junit.Test;
public class ClassTest extends BaseTest {
@Before
public void setup() throws MetadataException {
super.setup();
}
/*
* Class Hierarchy is:
* Department(name : String, employees : Array[Person])
* Person(name : String, department : Department, manager : Manager)
* Manager(subordinates : Array[Person]) extends Person
*
* Persons can have SecurityClearance(level : Int) clearance.
*/
@Test
public void test1() throws MetadataException {
TypeSystem ts = ms.getTypeSystem();
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees",
String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")
);
HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("department",
"Department", Multiplicity.REQUIRED, false, "employees"),
new AttributeDefinition("manager",
"Manager", Multiplicity.OPTIONAL, false, "subordinates")
);
HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager",
ImmutableList.<String>of("Person"),
new AttributeDefinition("subordinates",
String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")
);
HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef("SecurityClearance",
ImmutableList.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE)
);
ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef, managerTypeDef));
Referenceable hrDept = new Referenceable("Department");
Referenceable john = new Referenceable("Person");
Referenceable jane = new Referenceable("Manager", "SecurityClearance");
hrDept.set("name", "hr");
john.set("name", "John");
john.set("department", hrDept);
jane.set("name", "Jane");
jane.set("department", hrDept);
john.set("manager", jane);
hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
jane.set("subordinates", ImmutableList.<Referenceable>of(john));
jane.getTrait("SecurityClearance").set("level", 1);
defineDeptEmployeeTypes(ts);
Referenceable hrDept = createDeptEg1(ts);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
Assert.assertEquals(hrDept2.toString(), "{\n" +
"\tid : (type: Department, id: <unassigned>)\n" +
"\tname : \thr\n" +
......
typesystem/src/test/java/org/apache/hadoop/metadata/StorageTest.java 0 → 100644
package org.apache.hadoop.metadata;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.junit.Before;
import org.junit.Test;
public class StorageTest extends BaseTest {
@Before
public void setup() throws MetadataException {
super.setup();
}
@Test
public void test1() throws MetadataException {
TypeSystem ts = ms.getTypeSystem();
defineDeptEmployeeTypes(ts);
Referenceable hrDept = createDeptEg1(ts);
//ITypedReferenceableInstance hrDept2 = ms.getRepository().create(hrDept);
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment