Skip to content

A
atlas
Commit b9e4d865 by Suma S
Options

Merge pull request #107 from sumashivaprasad/BUG_37105

Fixed support for Date Types across typesystem, repository, indexes
parents 59b4c32d e74de9c7
Showing with 37 additions and 26 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -56,12 +56,7 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
......@@ -924,6 +919,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
propertyValue = typedInstance.getDouble(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
propertyValue = typedInstance.getDate(attributeInfo.name);
}
addProperty(instanceVertex, vertexPropertyName, propertyValue);
}
......@@ -1295,6 +1292,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
typedInstance.setBigDecimal(attributeInfo.name,
instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
typedInstance.setDate(attributeInfo.name,
instanceVertex.<Date>getProperty(vertexPropertyName));
}
}
}
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
......@@ -45,6 +45,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
/**
......@@ -249,8 +250,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
return Double.class;
} else if (dataType == DataTypes.BIGDECIMAL_TYPE) {
return BigDecimal.class;
} else if (dataType == DataTypes.DATE_TYPE) {
return Date.class;
}
throw new IllegalArgumentException("unknown data type " + dataType);
}
......
repository/src/test/java/org/apache/hadoop/metadata/repository/BaseTest.java
......@@ -49,6 +49,8 @@ public abstract class BaseTest {
public static final String STRUCT_TYPE_1 = "t1";
public static final String STRUCT_TYPE_2 = "t2";
public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
public static final long TEST_DATE_IN_LONG=1418265358440L;
protected IRepository repo;
public static Struct createStruct() throws MetadataException {
......
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -27,6 +27,7 @@ import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.TestUtils;
import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
import org.apache.hadoop.metadata.repository.BaseTest;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
......@@ -54,11 +55,7 @@ import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.*;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createStructTypeDef;
......@@ -158,6 +155,7 @@ public class GraphBackedMetadataRepositoryTest {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database");
databaseInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
databaseInstance.set("namespace", "colo:cluster:hive:db");
databaseInstance.set("cluster", "cluster-1");
......@@ -184,6 +182,7 @@ public class GraphBackedMetadataRepositoryTest {
String guid = getGUID();
ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
Assert.assertEquals(table.getDate("created"), new Date(BaseTest.TEST_DATE_IN_LONG));
System.out.println("*** table = " + table);
}
......@@ -430,8 +429,10 @@ public class GraphBackedMetadataRepositoryTest {
TypesUtil.createClassTypeDef(DATABASE_TYPE,
ImmutableList.of(SUPER_TYPE_NAME),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition =
new StructTypeDefinition("serdeType",
new AttributeDefinition[]{
......@@ -465,6 +466,7 @@ public class GraphBackedMetadataRepositoryTest {
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
// enum
new AttributeDefinition("tableType", "tableType",
Multiplicity.REQUIRED, false, null),
......@@ -528,6 +530,7 @@ public class GraphBackedMetadataRepositoryTest {
tableInstance.set("name", TABLE_NAME);
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
tableInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
tableInstance.set("tableType", 1); // enum
// super type
......
repository/src/test/java/org/apache/hadoop/metadata/repository/memory/EnumTest.java
......@@ -195,7 +195,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......@@ -227,7 +227,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......@@ -264,7 +264,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......@@ -304,7 +304,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
".100000000000000088817841970012523233890533447265625]\n" +
......
repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StructTest.java
......@@ -58,7 +58,7 @@ public class StructTest extends BaseTest {
"\ti : \t1.0\n" +
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......@@ -101,7 +101,7 @@ public class StructTest extends BaseTest {
"\ti : \t1.0\n" +
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
".100000000000000088817841970012523233890533447265625]\n" +
......
tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
......@@ -120,7 +120,7 @@ class DSLTest {
Assert.assertEquals(s"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}", "[b, a]")
// 5. Serialize mytype instance to Json
Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}")
Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03T08:00:00.000Z\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}")
}
@Test def test2 {
......
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
......@@ -47,7 +47,7 @@ public class TypeSystem {
private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() {
@Override
public SimpleDateFormat initialValue() {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
return dateFormat;
}
......
typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/BaseTest.java
......@@ -37,6 +37,8 @@ public abstract class BaseTest {
public static final String STRUCT_TYPE_1 = "t1";
public static final String STRUCT_TYPE_2 = "t2";
public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
public static final long TEST_DATE_IN_LONG=1418265358440L;
public static Struct createStruct() throws MetadataException {
StructType structType = TypeSystem.getInstance().getDataType(
......
typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/EnumTest.java
......@@ -179,7 +179,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......@@ -211,7 +211,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tenum3 : \tCOMMITTED\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......
typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/StructTest.java
......@@ -54,7 +54,7 @@ public class StructTest extends BaseTest {
"\ti : \t1.0\n" +
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tl : \t2014-12-11\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{b=2.0, a=1.0}\n" +
......
typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
......@@ -44,18 +44,18 @@ class SerializationTest extends BaseTest {
val s: Struct = BaseTest.createStruct()
val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-11\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}")
Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}")
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new BigDecimalSerializer + new BigIntegerSerializer
//Json representation
val ser = swrite(ts)
Assert.assertEquals(ser, "{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"2014-12-11T02:35:58.440Z\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}")
Assert.assertEquals(ser, "{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"" + BaseTest.TEST_DATE + "\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}")
// Typed Struct read back
val ts1 = read[StructInstance](ser)
Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-11\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
}
@Test def test2 {
......@@ -70,7 +70,7 @@ class SerializationTest extends BaseTest {
{"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
"c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"b":2.0,"a":1.0}}""")
// Typed Struct read from string
Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
}
@Test def testTrait {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment