Fixed support for Date Types across typesystem, repository, indexes

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java

@@ -55,12 +55,7 @@ import javax.inject.Inject;
 import javax.inject.Singleton;
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
@@ -930,6 +925,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                 propertyValue = typedInstance.getDouble(attributeInfo.name);
             } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
                 propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
+            } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
+                propertyValue = typedInstance.getDate(attributeInfo.name);
             }
             addProperty(instanceVertex, vertexPropertyName, propertyValue);
         }
@@ -1310,6 +1307,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
                 typedInstance.setBigDecimal(attributeInfo.name,
                         instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
+            } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
+                typedInstance.setDate(attributeInfo.name,
+                        instanceVertex.<Date>getProperty(vertexPropertyName));
             }
         }
     }

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java

@@ -45,6 +45,7 @@ import org.slf4j.LoggerFactory;
 import javax.inject.Inject;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.util.Date;
 import java.util.Map;
 
 /**
@@ -249,8 +250,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
             return Double.class;
         } else if (dataType == DataTypes.BIGDECIMAL_TYPE) {
             return BigDecimal.class;
+        } else if (dataType == DataTypes.DATE_TYPE) {
+            return Date.class;
         }
 
+
         throw new IllegalArgumentException("unknown data type " + dataType);
     }
 

repository/src/test/java/org/apache/hadoop/metadata/repository/BaseTest.java

@@ -49,6 +49,8 @@ public abstract class BaseTest {
 
     public static final String STRUCT_TYPE_1 = "t1";
     public static final String STRUCT_TYPE_2 = "t2";
+    public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
+    public static final long TEST_DATE_IN_LONG=1418265358440L;
     protected IRepository repo;
 
     public static Struct createStruct() throws MetadataException {

repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java

@@ -26,6 +26,7 @@ import com.tinkerpop.blueprints.Vertex;
 import org.apache.hadoop.metadata.RepositoryMetadataModule;
 import org.apache.hadoop.metadata.TestUtils;
 import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.hadoop.metadata.repository.BaseTest;
 import org.apache.hadoop.metadata.repository.Constants;
 import org.apache.hadoop.metadata.repository.RepositoryException;
 import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
@@ -53,11 +54,7 @@ import org.testng.annotations.Test;
 import scala.actors.threadpool.Arrays;
 
 import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
 
 /**
  * GraphBackedMetadataRepository test
@@ -154,6 +151,7 @@ public class GraphBackedMetadataRepositoryTest {
         Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
         databaseInstance.set("name", DATABASE_NAME);
         databaseInstance.set("description", "foo database");
+        databaseInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
 
         databaseInstance.set("namespace", "colo:cluster:hive:db");
         databaseInstance.set("cluster", "cluster-1");
@@ -180,6 +178,7 @@ public class GraphBackedMetadataRepositoryTest {
         String guid = getGUID();
 
         ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
+        Assert.assertEquals(table.getDate("created"), new Date(BaseTest.TEST_DATE_IN_LONG));
         System.out.println("*** table = " + table);
     }
 
@@ -426,8 +425,10 @@ public class GraphBackedMetadataRepositoryTest {
                 TypesUtil.createClassTypeDef(DATABASE_TYPE,
                         ImmutableList.of(SUPER_TYPE_NAME),
                         TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
                         TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
 
+
         StructTypeDefinition structTypeDefinition =
                 new StructTypeDefinition("serdeType",
                         new AttributeDefinition[]{
@@ -461,6 +462,7 @@ public class GraphBackedMetadataRepositoryTest {
                         TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
                         TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
                         TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
+                        TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
                         // enum
                         new AttributeDefinition("tableType", "tableType",
                                 Multiplicity.REQUIRED, false, null),
@@ -524,6 +526,7 @@ public class GraphBackedMetadataRepositoryTest {
         tableInstance.set("name", TABLE_NAME);
         tableInstance.set("description", "bar table");
         tableInstance.set("type", "managed");
+        tableInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
         tableInstance.set("tableType", 1); // enum
 
         // super type

repository/src/test/java/org/apache/hadoop/metadata/repository/memory/EnumTest.java

@@ -195,7 +195,7 @@ public class EnumTest extends BaseTest {
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
                 "\tenum3 : \tCOMMITTED\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.1, 1.1]\n" +
                 "\to : \t{b=2.0, a=1.0}\n" +
@@ -227,7 +227,7 @@ public class EnumTest extends BaseTest {
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
                 "\tenum3 : \tCOMMITTED\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.1, 1.1]\n" +
                 "\to : \t{b=2.0, a=1.0}\n" +
@@ -264,7 +264,7 @@ public class EnumTest extends BaseTest {
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
                 "\tenum3 : \tCOMMITTED\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.1, 1.1]\n" +
                 "\to : \t{b=2.0, a=1.0}\n" +
@@ -304,7 +304,7 @@ public class EnumTest extends BaseTest {
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
                 "\tenum3 : \tCOMMITTED\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
                 ".100000000000000088817841970012523233890533447265625]\n" +

repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StructTest.java

@@ -58,7 +58,7 @@ public class StructTest extends BaseTest {
                 "\ti : \t1.0\n" +
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.1, 1.1]\n" +
                 "\to : \t{b=2.0, a=1.0}\n" +
@@ -101,7 +101,7 @@ public class StructTest extends BaseTest {
                 "\ti : \t1.0\n" +
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
                 ".100000000000000088817841970012523233890533447265625]\n" +

tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala

@@ -120,7 +120,7 @@ class DSLTest {
         Assert.assertEquals(s"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}", "[b, a]")
 
         // 5. Serialize mytype instance to Json
-        Assert.assertEquals(s"${pretty(render(i))}", "{\n  \"$typeName$\":\"mytype\",\n  \"e\":1," + "\n  \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n  \"h\":1.0,\n  \"b\":true,\n  \"k\":1,\n  \"j\":1,\n  \"d\":2,\n  \"m\":[1,1],\n  \"g\":1,\n  \"a\":1,\n  \"i\":1.0,\n  \"c\":1,\n  \"l\":\"2014-12-03\",\n  \"f\":1,\n  \"o\":{\n    \"b\":2.0,\n    \"a\":1.0\n  }\n}")
+        Assert.assertEquals(s"${pretty(render(i))}", "{\n  \"$typeName$\":\"mytype\",\n  \"e\":1," + "\n  \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n  \"h\":1.0,\n  \"b\":true,\n  \"k\":1,\n  \"j\":1,\n  \"d\":2,\n  \"m\":[1,1],\n  \"g\":1,\n  \"a\":1,\n  \"i\":1.0,\n  \"c\":1,\n  \"l\":\"2014-12-03T08:00:00.000Z\",\n  \"f\":1,\n  \"o\":{\n    \"b\":2.0,\n    \"a\":1.0\n  }\n}")
     }
 
     @Test def test2 {

typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java

@@ -47,7 +47,7 @@ public class TypeSystem {
     private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() {
         @Override
         public SimpleDateFormat initialValue() {
-            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
             dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
             return dateFormat;
         }

typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/BaseTest.java

@@ -37,6 +37,8 @@ public abstract class BaseTest {
 
     public static final String STRUCT_TYPE_1 = "t1";
     public static final String STRUCT_TYPE_2 = "t2";
+    public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
+    public static final long TEST_DATE_IN_LONG=1418265358440L;
 
     public static Struct createStruct() throws MetadataException {
         StructType structType = TypeSystem.getInstance().getDataType(

typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/EnumTest.java

@@ -179,7 +179,7 @@ public class EnumTest extends BaseTest {
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
                 "\tenum3 : \tCOMMITTED\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.1, 1.1]\n" +
                 "\to : \t{b=2.0, a=1.0}\n" +
@@ -211,7 +211,7 @@ public class EnumTest extends BaseTest {
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
                 "\tenum3 : \tCOMMITTED\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.1, 1.1]\n" +
                 "\to : \t{b=2.0, a=1.0}\n" +

typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/StructTest.java

@@ -54,7 +54,7 @@ public class StructTest extends BaseTest {
                 "\ti : \t1.0\n" +
                 "\tj : \t1\n" +
                 "\tk : \t1\n" +
-                "\tl : \t2014-12-11\n" +
+                "\tl : \t" + TEST_DATE + "\n" +
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.1, 1.1]\n" +
                 "\to : \t{b=2.0, a=1.0}\n" +

typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala

@@ -44,18 +44,18 @@ class SerializationTest extends BaseTest {
         val s: Struct = BaseTest.createStruct()
         val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
 
-        Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-11\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}")
+        Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}")
 
         implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
             new BigDecimalSerializer + new BigIntegerSerializer
 
         //Json representation
         val ser = swrite(ts)
-        Assert.assertEquals(ser, "{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"2014-12-11T02:35:58.440Z\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}")
+        Assert.assertEquals(ser, "{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"" + BaseTest.TEST_DATE + "\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}")
 
         // Typed Struct read back
         val ts1 = read[StructInstance](ser)
-        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-11\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
+        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
     }
 
     @Test def test2 {
@@ -70,7 +70,7 @@ class SerializationTest extends BaseTest {
         {"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
         "c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"b":2.0,"a":1.0}}""")
         // Typed Struct read from string
-        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
+        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
     }
 
     @Test def testTrait {