Introduce FieldMapping to get ready for Trait and Class Types

src/main/java/org/apache/metadata/storage/TypedStruct.java → src/main/java/org/apache/metadata/storage/StructInstance.java

@@ -5,14 +5,17 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.apache.metadata.IStruct;
 import org.apache.metadata.MetadataException;
-import org.apache.metadata.types.StructType;
+import org.apache.metadata.types.AttributeInfo;
+import org.apache.metadata.types.FieldMapping;
+import org.apache.metadata.types.TypeUtils;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.Date;
 
-public class TypedStruct implements IStruct {
-    public final StructType dataType;
+public class StructInstance implements IStruct {
+    public final String dataTypeName;
+    public final FieldMapping fieldMapping;
     public final boolean nullFlags[];
     public final boolean[] bools;
     public final byte[] bytes;
@@ -27,14 +30,16 @@ public class TypedStruct implements IStruct {
     public final String[] strings;
     public final ImmutableList<Object>[] arrays;
     public final ImmutableMap<Object,Object>[] maps;
-    public final TypedStruct[] structs;
+    public final StructInstance[] structs;
 
-    public TypedStruct(StructType dataType, boolean[] nullFlags, boolean[] bools, byte[] bytes, short[] shorts, int[] ints,
-                       long[] longs, float[] floats, double[] doubles,
-                       BigDecimal[] bigDecimals, BigInteger[] bigIntegers, Date[] dates, String[] strings,
-                       ImmutableList<Object>[] arrays, ImmutableMap<Object, Object>[] maps, TypedStruct[] structs) {
-        assert dataType != null;
-        this.dataType = dataType;
+    public StructInstance(String dataTypeName, FieldMapping fieldMapping,
+                          boolean[] nullFlags, boolean[] bools, byte[] bytes, short[] shorts, int[] ints,
+                          long[] longs, float[] floats, double[] doubles,
+                          BigDecimal[] bigDecimals, BigInteger[] bigIntegers, Date[] dates, String[] strings,
+                          ImmutableList<Object>[] arrays, ImmutableMap<Object, Object>[] maps, StructInstance[] structs) {
+        assert dataTypeName != null;
+        this.dataTypeName = dataTypeName;
+        this.fieldMapping = fieldMapping;
         this.nullFlags = nullFlags;
         this.bools = bools;
         this.bytes = bytes;
@@ -58,25 +63,54 @@ public class TypedStruct implements IStruct {
 
     @Override
     public String getTypeName() {
-        return dataType.getName();
+        return dataTypeName;
     }
 
     @Override
     public Object get(String attrName) throws MetadataException {
-        return dataType.get(this, attrName);
+        return fieldMapping.get(this, attrName);
     }
 
     @Override
     public void set(String attrName, Object val) throws MetadataException {
-        dataType.set(this, attrName, val);
+        fieldMapping.set(this, attrName, val);
+    }
+
+    public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
+        TypeUtils.outputVal("{", buf, prefix);
+        if ( s == null ) {
+            TypeUtils.outputVal("<null>\n", buf, "");
+            return;
+        }
+        TypeUtils.outputVal("\n", buf, "");
+        String fieldPrefix = prefix + "\t";
+        for(AttributeInfo i : fieldMapping.fields.values()) {
+            Object aVal = s.get(i.name);
+            TypeUtils.outputVal(i.name + " : ", buf, fieldPrefix);
+            i.dataType().output(aVal, buf, "");
+            TypeUtils.outputVal("\n", buf, "");
+        }
+        TypeUtils.outputVal("\n}\n", buf, "");
     }
 
     @Override
     public String toString()  {
         try {
-            StringBuilder b = new StringBuilder();
-            dataType.output(this, b, "");
-            return b.toString();
+            StringBuilder buf = new StringBuilder();
+            String prefix = "";
+
+            TypeUtils.outputVal("{", buf, prefix);
+            TypeUtils.outputVal("\n", buf, "");
+            String fieldPrefix = prefix + "\t";
+            for(AttributeInfo i : fieldMapping.fields.values()) {
+                Object aVal = get(i.name);
+                TypeUtils.outputVal(i.name + " : ", buf, fieldPrefix);
+                i.dataType().output(aVal, buf, "");
+                TypeUtils.outputVal("\n", buf, "");
+            }
+            TypeUtils.outputVal("\n}\n", buf, "");
+            return buf.toString();
+
         } catch(MetadataException me) {
             throw new RuntimeException(me);
         }

src/main/java/org/apache/metadata/types/AbstractDataType.java

@@ -13,18 +13,9 @@ abstract class AbstractDataType<T> implements IDataType<T> {
         return null;
     }
 
-    protected void outputVal(String val, Appendable buf, String prefix) throws MetadataException {
-        try {
-            buf.append(prefix).append(val);
-        } catch(IOException ie) {
-            throw new MetadataException(ie);
-        }
-    }
-
     @Override
     public void output(T val, Appendable buf, String prefix) throws MetadataException {
-        outputVal(val == null ? "<null>" : val.toString(), buf, prefix);
+        TypeUtils.outputVal(val == null ? "<null>" : val.toString(), buf, prefix);
     }
-
 }
 

src/main/java/org/apache/metadata/types/FieldMapping.java

+package org.apache.metadata.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.metadata.MetadataException;
+import org.apache.metadata.storage.StructInstance;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class FieldMapping {
+
+    public final Map<String,AttributeInfo> fields;
+    private final Map<String, Integer> fieldPos;
+    private final Map<String, Integer> fieldNullPos;
+    public final int numBools;
+    public final int numBytes;
+    public final int numShorts;
+    public final int numInts;
+    public final int numLongs;
+    public final int numFloats;
+    public final int numDoubles;
+    public final int numBigInts;
+    public final int numBigDecimals;
+    public final int numDates;
+    public final int numStrings;
+    public final int numArrays;
+    public final int numMaps;
+    public final int numStructs;
+
+    public FieldMapping(Map<String, AttributeInfo> fields, Map<String, Integer> fieldPos,
+                        Map<String, Integer> fieldNullPos, int numBools, int numBytes, int numShorts,
+                        int numInts, int numLongs, int numFloats, int numDoubles, int numBigInts, int numBigDecimals,
+                        int numDates, int numStrings, int numArrays, int numMaps, int numStructs) {
+        this.fields = fields;
+        this.fieldPos = fieldPos;
+        this.fieldNullPos = fieldNullPos;
+        this.numBools = numBools;
+        this.numBytes = numBytes;
+        this.numShorts = numShorts;
+        this.numInts = numInts;
+        this.numLongs = numLongs;
+        this.numFloats = numFloats;
+        this.numDoubles = numDoubles;
+        this.numBigInts = numBigInts;
+        this.numBigDecimals = numBigDecimals;
+        this.numDates = numDates;
+        this.numStrings = numStrings;
+        this.numArrays = numArrays;
+        this.numMaps = numMaps;
+        this.numStructs = numStructs;
+    }
+
+    public void set(StructInstance s, String attrName, Object val) throws MetadataException {
+        AttributeInfo i = fields.get(attrName);
+        if ( i == null ) {
+            throw new ValueConversionException(s.getTypeName(), val, "Unknown field " + attrName);
+        }
+        int pos = fieldPos.get(attrName);
+        int nullPos = fieldNullPos.get(attrName);
+        Object cVal = i.dataType().convert(val, i.multiplicity);
+        if ( cVal == null ) {
+            s.nullFlags[nullPos] = true;
+            return;
+        }
+        s.nullFlags[nullPos] = false;
+        if ( i.dataType() == DataTypes.BOOLEAN_TYPE ) {
+            s.bools[pos] = ((Boolean)cVal).booleanValue();
+        } else if ( i.dataType() == DataTypes.BYTE_TYPE ) {
+            s.bytes[pos] = ((Byte)cVal).byteValue();
+        } else if ( i.dataType() == DataTypes.SHORT_TYPE ) {
+            s.shorts[pos] = ((Short)cVal).shortValue();
+        } else if ( i.dataType() == DataTypes.INT_TYPE ) {
+            s.ints[pos] = ((Integer)cVal).intValue();
+        } else if ( i.dataType() == DataTypes.LONG_TYPE ) {
+            s.longs[pos] = ((Long)cVal).longValue();
+        } else if ( i.dataType() == DataTypes.FLOAT_TYPE ) {
+            s.floats[pos] = ((Float)cVal).floatValue();
+        } else if ( i.dataType() == DataTypes.DOUBLE_TYPE ) {
+            s.doubles[pos] = ((Double)cVal).doubleValue();
+        } else if ( i.dataType() == DataTypes.BIGINTEGER_TYPE ) {
+            s.bigIntegers[pos] = (BigInteger) cVal;
+        } else if ( i.dataType() == DataTypes.BIGDECIMAL_TYPE ) {
+            s.bigDecimals[pos] = (BigDecimal) cVal;
+        } else if ( i.dataType() == DataTypes.DATE_TYPE ) {
+            s.dates[pos] = (Date) cVal;
+        } else if ( i.dataType() == DataTypes.STRING_TYPE ) {
+            s.strings[pos] = (String) cVal;
+        } else if ( i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY ) {
+            s.arrays[pos] = (ImmutableList) cVal;
+        } else if ( i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP ) {
+            s.maps[pos] = (ImmutableMap) cVal;
+        } else if ( i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ) {
+            s.structs[pos] = (StructInstance) cVal;
+        } else {
+            throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
+        }
+    }
+
+    public Object get(StructInstance s, String attrName) throws MetadataException {
+        AttributeInfo i = fields.get(attrName);
+        if ( i == null ) {
+            throw new MetadataException(String.format("Unknown field %s for Struct %s", attrName, s.getTypeName()));
+        }
+        int pos = fieldPos.get(attrName);
+        int nullPos = fieldNullPos.get(attrName);
+
+        if ( s.nullFlags[nullPos]) {
+            return null;
+        }
+
+        if ( i.dataType() == DataTypes.BOOLEAN_TYPE ) {
+            return s.bools[pos];
+        } else if ( i.dataType() == DataTypes.BYTE_TYPE ) {
+            return s.bytes[pos];
+        } else if ( i.dataType() == DataTypes.SHORT_TYPE ) {
+            return s.shorts[pos];
+        } else if ( i.dataType() == DataTypes.INT_TYPE ) {
+            return s.ints[pos];
+        } else if ( i.dataType() == DataTypes.LONG_TYPE ) {
+            return s.longs[pos];
+        } else if ( i.dataType() == DataTypes.FLOAT_TYPE ) {
+            return s.floats[pos];
+        } else if ( i.dataType() == DataTypes.DOUBLE_TYPE ) {
+            return s.doubles[pos];
+        } else if ( i.dataType() == DataTypes.BIGINTEGER_TYPE ) {
+            return s.bigIntegers[pos];
+        } else if ( i.dataType() == DataTypes.BIGDECIMAL_TYPE ) {
+            return s.bigDecimals[pos];
+        } else if ( i.dataType() == DataTypes.DATE_TYPE ) {
+            return s.dates[pos];
+        } else if ( i.dataType() == DataTypes.STRING_TYPE ) {
+            return s.strings[pos];
+        } else if ( i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY ) {
+            return s.arrays[pos];
+        } else if ( i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP ) {
+            return s.maps[pos];
+        } else if ( i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ) {
+            return s.structs[pos];
+        } else {
+            throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
+        }
+    }
+
+}

src/main/java/org/apache/metadata/types/TypeUtils.java

+package org.apache.metadata.types;
+
+
+import org.apache.metadata.MetadataException;
+
+import java.io.IOException;
+
+public class TypeUtils {
+
+    public static void outputVal(String val, Appendable buf, String prefix) throws MetadataException {
+        try {
+            buf.append(prefix).append(val);
+        } catch(IOException ie) {
+            throw new MetadataException(ie);
+        }
+    }
+}

src/main/java/org/apache/metadata/types/ValueConversionException.java

@@ -17,6 +17,11 @@ public class ValueConversionException  extends MetadataException {
                 val.toString(), typ.getName(), msg));
     }
 
+    public ValueConversionException(String typeName, Object val, String msg) {
+        super(String.format("Cannot convert value '%s' to datatype %s because: %s",
+                val.toString(), typeName, msg));
+    }
+
     protected ValueConversionException(String msg) {
         super(msg);
     }

src/main/scala/org/apache/metadata/dsl/DynamicTypedStruct.scala

 package org.apache.metadata.dsl
 
-import org.apache.metadata.storage.TypedStruct
+import org.apache.metadata.MetadataService
+import org.apache.metadata.storage.StructInstance
+import org.apache.metadata.types.TypeSystem
 import scala.language.dynamics
 
-class DynamicTypedStruct(val ts : TypedStruct) extends Dynamic {
+class DynamicTypedStruct(val ts : StructInstance) extends Dynamic {
   def selectDynamic(name: String) = ts.get(name)
   def updateDynamic(name: String)(value: Any) {
     var value1 = value
@@ -12,5 +14,5 @@ class DynamicTypedStruct(val ts : TypedStruct) extends Dynamic {
     }
     ts.set(name, value1)
   }
-  def dataType = ts.dataType
+  def dataType = MetadataService.getCurrentTypeSystem.getDataType(ts.getTypeName)
 }

src/main/scala/org/apache/metadata/dsl/package.scala

 package org.apache.metadata
 
 import org.apache.metadata.json.{BigIntegerSerializer, BigDecimalSerializer, TypedStructSerializer, Serialization}
-import org.apache.metadata.storage.TypedStruct
+import org.apache.metadata.storage.StructInstance
 import org.apache.metadata.types._
 
 import scala.collection.JavaConverters._
@@ -59,7 +59,7 @@ package object dsl {
     assert(j.isInstanceOf[JObject])
     var j1 = j.asInstanceOf[JObject]
     j1 = JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(typeName)) :: j1.obj)
-    new DynamicTypedStruct(Extraction.extract[TypedStruct](j1))
+    new DynamicTypedStruct(Extraction.extract[StructInstance](j1))
   }
 
   def createInstance(typeName : String) = {

src/main/scala/org/apache/metadata/json/Serialization.scala

@@ -9,7 +9,7 @@ import org.json4s.native.Serialization.{read, write => swrite}
 import org.json4s.reflect.{ScalaType, Reflector}
 import java.util.regex.Pattern
 import java.util.Date
-import org.apache.metadata.storage.TypedStruct
+import org.apache.metadata.storage.StructInstance
 import collection.JavaConversions._
 import scala.collection.JavaConverters._
 
@@ -27,7 +27,7 @@ class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format
 }
   ))
 
-class TypedStructSerializer extends Serializer[TypedStruct] {
+class TypedStructSerializer extends Serializer[StructInstance] {
 
   def extractList(lT : ArrayType, value : JArray)(implicit format: Formats) : Any = {
     val dT = lT.getElemType
@@ -51,11 +51,11 @@ class TypedStructSerializer extends Serializer[TypedStruct] {
     case value : JObject if dT.getTypeCategory eq TypeCategory.MAP =>
       extractMap(dT.asInstanceOf[MapType], value.asInstanceOf[JObject])
     case value : JObject  =>
-      Extraction.extract[TypedStruct](value)
+      Extraction.extract[StructInstance](value)
   }
 
   def deserialize(implicit format: Formats) = {
-    case (TypeInfo(clazz, ptype), json) if classOf[TypedStruct].isAssignableFrom(clazz) => json match {
+    case (TypeInfo(clazz, ptype), json) if classOf[StructInstance].isAssignableFrom(clazz) => json match {
       case JObject(fs) =>
         val(typ, fields) = fs.partition(f => f._1 == Serialization.STRUCT_TYPE_FIELD_NAME)
         val typName = typ(0)._2.asInstanceOf[JString].s
@@ -63,7 +63,7 @@ class TypedStructSerializer extends Serializer[TypedStruct] {
         val s = sT.createInstance()
         fields.foreach { f =>
           val fName = f._1
-          val fInfo = sT.fields(fName)
+          val fInfo = sT.fieldMapping.fields(fName)
           if ( fInfo != null ) {
             //println(fName)
             var v = f._2
@@ -91,8 +91,8 @@ class TypedStructSerializer extends Serializer[TypedStruct] {
   //implicit def javaBigInteger2bigInt(x: java.math.BigInteger): BigInt = new BigInt(x)
 
   def serialize(implicit format: Formats) = {
-    case e: TypedStruct =>
-      val fields  = e.dataType.fields.map {
+    case e: StructInstance =>
+      val fields  = e.fieldMapping.fields.map {
         case (fName, info) => {
           var v = e.get(fName)
           if ( v != null && (info.dataType().getTypeCategory eq TypeCategory.MAP) ) {
@@ -101,7 +101,7 @@ class TypedStructSerializer extends Serializer[TypedStruct] {
           JField(fName, Extraction.decompose(v))
         }
       }.toList.map(_.asInstanceOf[JField])
-      JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(e.dataType.getName)) :: fields)
+      JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(e.dataTypeName)) :: fields)
   }
 }
 

src/test/java/org/apache/metadata/StructTest.java

 package org.apache.metadata;
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import junit.framework.TestCase;
-import org.apache.metadata.storage.TypedStruct;
+import org.apache.metadata.storage.StructInstance;
 import org.apache.metadata.types.*;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.Map;
-
 public class StructTest extends BaseTest {
 
     StructType structType;
@@ -28,7 +20,7 @@ public class StructTest extends BaseTest {
     @Test
     public void test1() throws MetadataException {
         Struct s = createStruct(ms);
-        TypedStruct  ts = structType.convert(s, Multiplicity.REQUIRED);
+        StructInstance ts = structType.convert(s, Multiplicity.REQUIRED);
         System.out.println(ts);
     }
 
@@ -39,7 +31,7 @@ public class StructTest extends BaseTest {
         Struct s2 = new Struct(recursiveStructType.getName());
         s2.set("a", 1);
         s2.set("s", s1);
-        TypedStruct  ts = recursiveStructType.convert(s2, Multiplicity.REQUIRED);
+        StructInstance ts = recursiveStructType.convert(s2, Multiplicity.REQUIRED);
         System.out.println(ts);
     }
 

src/test/scala/org/apache/metadata/dsl/DSLTest.scala

@@ -2,7 +2,7 @@ package org.apache.metadata.dsl
 
 import org.apache.metadata.hive.HiveMockMetadataService
 import org.apache.metadata.json.{BigIntegerSerializer, BigDecimalSerializer, TypedStructSerializer}
-import org.apache.metadata.storage.TypedStruct
+import org.apache.metadata.storage.StructInstance
 import org.apache.metadata.{Struct, BaseTest}
 import org.apache.metadata.types.{IDataType, Multiplicity, StructType}
 import org.json4s.NoTypeHints

src/test/scala/org/apache/metadata/json/SerializationTest.scala

 package org.apache.metadata.json
 
 import org.apache.metadata.Struct
-import org.apache.metadata.storage.TypedStruct
-import org.apache.metadata.storage.TypedStruct
+import org.apache.metadata.storage.StructInstance
+import org.apache.metadata.storage.StructInstance
 import org.apache.metadata.types.Multiplicity
 import org.apache.metadata.types.StructType
 import org.apache.metadata.{Struct, BaseTest}
@@ -29,7 +29,7 @@ class SerializationTest extends BaseTest {
 
   @Test def test1 {
     val s: Struct = BaseTest.createStruct(ms)
-    val ts: TypedStruct = structType.convert(s, Multiplicity.REQUIRED)
+    val ts: StructInstance = structType.convert(s, Multiplicity.REQUIRED)
 
     println("Typed Struct :")
     println(ts)
@@ -41,19 +41,19 @@ class SerializationTest extends BaseTest {
     println("Json representation :")
     println(ser)
 
-    val ts1 = read[TypedStruct](ser)
+    val ts1 = read[StructInstance](ser)
     println("Typed Struct read back:")
     println(ts1)
   }
 
   @Test def test2 {
     val s: Struct = BaseTest.createStruct(ms)
-    val ts: TypedStruct = structType.convert(s, Multiplicity.REQUIRED)
+    val ts: StructInstance = structType.convert(s, Multiplicity.REQUIRED)
 
     implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
       new BigDecimalSerializer + new BigIntegerSerializer
 
-    val ts1 = read[TypedStruct](
+    val ts1 = read[StructInstance](
       """
         {"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
         "c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"b":2.0,"a":1.0}}""")