Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
e74de9c7
Commit
e74de9c7
authored
May 26, 2015
by
Suma Shivaprasad
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fixed support for Date Types across typesystem, repository, indexes
parent
c95c71b4
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
37 additions
and
26 deletions
+37
-26
GraphBackedMetadataRepository.java
...adata/repository/graph/GraphBackedMetadataRepository.java
+6
-6
GraphBackedSearchIndexer.java
...p/metadata/repository/graph/GraphBackedSearchIndexer.java
+4
-0
BaseTest.java
.../java/org/apache/hadoop/metadata/repository/BaseTest.java
+2
-0
GraphBackedMetadataRepositoryTest.java
...a/repository/graph/GraphBackedMetadataRepositoryTest.java
+8
-5
EnumTest.java
...rg/apache/hadoop/metadata/repository/memory/EnumTest.java
+4
-4
StructTest.java
.../apache/hadoop/metadata/repository/memory/StructTest.java
+2
-2
DSLTest.scala
.../scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
+1
-1
TypeSystem.java
...g/apache/hadoop/metadata/typesystem/types/TypeSystem.java
+1
-1
BaseTest.java
...org/apache/hadoop/metadata/typesystem/types/BaseTest.java
+2
-0
EnumTest.java
...org/apache/hadoop/metadata/typesystem/types/EnumTest.java
+2
-2
StructTest.java
...g/apache/hadoop/metadata/typesystem/types/StructTest.java
+1
-1
SerializationTest.scala
...e/hadoop/metadata/typesystem/json/SerializationTest.scala
+4
-4
No files found.
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
View file @
e74de9c7
...
...
@@ -55,12 +55,7 @@ import javax.inject.Inject;
import
javax.inject.Singleton
;
import
java.math.BigDecimal
;
import
java.math.BigInteger
;
import
java.util.ArrayList
;
import
java.util.Collections
;
import
java.util.HashMap
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.*
;
import
java.util.concurrent.atomic.AtomicInteger
;
/**
...
...
@@ -930,6 +925,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
propertyValue
=
typedInstance
.
getDouble
(
attributeInfo
.
name
);
}
else
if
(
attributeInfo
.
dataType
()
==
DataTypes
.
BIGDECIMAL_TYPE
)
{
propertyValue
=
typedInstance
.
getBigDecimal
(
attributeInfo
.
name
);
}
else
if
(
attributeInfo
.
dataType
()
==
DataTypes
.
DATE_TYPE
)
{
propertyValue
=
typedInstance
.
getDate
(
attributeInfo
.
name
);
}
addProperty
(
instanceVertex
,
vertexPropertyName
,
propertyValue
);
}
...
...
@@ -1310,6 +1307,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
else
if
(
attributeInfo
.
dataType
()
==
DataTypes
.
BIGDECIMAL_TYPE
)
{
typedInstance
.
setBigDecimal
(
attributeInfo
.
name
,
instanceVertex
.<
BigDecimal
>
getProperty
(
vertexPropertyName
));
}
else
if
(
attributeInfo
.
dataType
()
==
DataTypes
.
DATE_TYPE
)
{
typedInstance
.
setDate
(
attributeInfo
.
name
,
instanceVertex
.<
Date
>
getProperty
(
vertexPropertyName
));
}
}
}
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
View file @
e74de9c7
...
...
@@ -45,6 +45,7 @@ import org.slf4j.LoggerFactory;
import
javax.inject.Inject
;
import
java.math.BigDecimal
;
import
java.math.BigInteger
;
import
java.util.Date
;
import
java.util.Map
;
/**
...
...
@@ -249,8 +250,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
return
Double
.
class
;
}
else
if
(
dataType
==
DataTypes
.
BIGDECIMAL_TYPE
)
{
return
BigDecimal
.
class
;
}
else
if
(
dataType
==
DataTypes
.
DATE_TYPE
)
{
return
Date
.
class
;
}
throw
new
IllegalArgumentException
(
"unknown data type "
+
dataType
);
}
...
...
repository/src/test/java/org/apache/hadoop/metadata/repository/BaseTest.java
View file @
e74de9c7
...
...
@@ -49,6 +49,8 @@ public abstract class BaseTest {
public
static
final
String
STRUCT_TYPE_1
=
"t1"
;
public
static
final
String
STRUCT_TYPE_2
=
"t2"
;
public
static
final
String
TEST_DATE
=
"2014-12-11T02:35:58.440Z"
;
public
static
final
long
TEST_DATE_IN_LONG
=
1418265358440L
;
protected
IRepository
repo
;
public
static
Struct
createStruct
()
throws
MetadataException
{
...
...
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
View file @
e74de9c7
...
...
@@ -26,6 +26,7 @@ import com.tinkerpop.blueprints.Vertex;
import
org.apache.hadoop.metadata.RepositoryMetadataModule
;
import
org.apache.hadoop.metadata.TestUtils
;
import
org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService
;
import
org.apache.hadoop.metadata.repository.BaseTest
;
import
org.apache.hadoop.metadata.repository.Constants
;
import
org.apache.hadoop.metadata.repository.RepositoryException
;
import
org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance
;
...
...
@@ -53,11 +54,7 @@ import org.testng.annotations.Test;
import
scala.actors.threadpool.Arrays
;
import
javax.inject.Inject
;
import
java.util.ArrayList
;
import
java.util.HashMap
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.UUID
;
import
java.util.*
;
/**
* GraphBackedMetadataRepository test
...
...
@@ -154,6 +151,7 @@ public class GraphBackedMetadataRepositoryTest {
Referenceable
databaseInstance
=
new
Referenceable
(
DATABASE_TYPE
);
databaseInstance
.
set
(
"name"
,
DATABASE_NAME
);
databaseInstance
.
set
(
"description"
,
"foo database"
);
databaseInstance
.
set
(
"created"
,
new
Date
(
BaseTest
.
TEST_DATE_IN_LONG
));
databaseInstance
.
set
(
"namespace"
,
"colo:cluster:hive:db"
);
databaseInstance
.
set
(
"cluster"
,
"cluster-1"
);
...
...
@@ -180,6 +178,7 @@ public class GraphBackedMetadataRepositoryTest {
String
guid
=
getGUID
();
ITypedReferenceableInstance
table
=
repositoryService
.
getEntityDefinition
(
guid
);
Assert
.
assertEquals
(
table
.
getDate
(
"created"
),
new
Date
(
BaseTest
.
TEST_DATE_IN_LONG
));
System
.
out
.
println
(
"*** table = "
+
table
);
}
...
...
@@ -426,8 +425,10 @@ public class GraphBackedMetadataRepositoryTest {
TypesUtil
.
createClassTypeDef
(
DATABASE_TYPE
,
ImmutableList
.
of
(
SUPER_TYPE_NAME
),
TypesUtil
.
createUniqueRequiredAttrDef
(
"name"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createOptionalAttrDef
(
"created"
,
DataTypes
.
DATE_TYPE
),
TypesUtil
.
createRequiredAttrDef
(
"description"
,
DataTypes
.
STRING_TYPE
));
StructTypeDefinition
structTypeDefinition
=
new
StructTypeDefinition
(
"serdeType"
,
new
AttributeDefinition
[]{
...
...
@@ -461,6 +462,7 @@ public class GraphBackedMetadataRepositoryTest {
TypesUtil
.
createUniqueRequiredAttrDef
(
"name"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createRequiredAttrDef
(
"description"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createRequiredAttrDef
(
"type"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createOptionalAttrDef
(
"created"
,
DataTypes
.
DATE_TYPE
),
// enum
new
AttributeDefinition
(
"tableType"
,
"tableType"
,
Multiplicity
.
REQUIRED
,
false
,
null
),
...
...
@@ -524,6 +526,7 @@ public class GraphBackedMetadataRepositoryTest {
tableInstance
.
set
(
"name"
,
TABLE_NAME
);
tableInstance
.
set
(
"description"
,
"bar table"
);
tableInstance
.
set
(
"type"
,
"managed"
);
tableInstance
.
set
(
"created"
,
new
Date
(
BaseTest
.
TEST_DATE_IN_LONG
));
tableInstance
.
set
(
"tableType"
,
1
);
// enum
// super type
...
...
repository/src/test/java/org/apache/hadoop/metadata/repository/memory/EnumTest.java
View file @
e74de9c7
...
...
@@ -195,7 +195,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tenum3 : \tCOMMITTED\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.1, 1.1]\n"
+
"\to : \t{b=2.0, a=1.0}\n"
+
...
...
@@ -227,7 +227,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tenum3 : \tCOMMITTED\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.1, 1.1]\n"
+
"\to : \t{b=2.0, a=1.0}\n"
+
...
...
@@ -264,7 +264,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tenum3 : \tCOMMITTED\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.1, 1.1]\n"
+
"\to : \t{b=2.0, a=1.0}\n"
+
...
...
@@ -304,7 +304,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tenum3 : \tCOMMITTED\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.100000000000000088817841970012523233890533447265625, 1"
+
".100000000000000088817841970012523233890533447265625]\n"
+
...
...
repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StructTest.java
View file @
e74de9c7
...
...
@@ -58,7 +58,7 @@ public class StructTest extends BaseTest {
"\ti : \t1.0\n"
+
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.1, 1.1]\n"
+
"\to : \t{b=2.0, a=1.0}\n"
+
...
...
@@ -101,7 +101,7 @@ public class StructTest extends BaseTest {
"\ti : \t1.0\n"
+
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.100000000000000088817841970012523233890533447265625, 1"
+
".100000000000000088817841970012523233890533447265625]\n"
+
...
...
tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
View file @
e74de9c7
...
...
@@ -120,7 +120,7 @@ class DSLTest {
Assert
.
assertEquals
(
s
"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}"
,
"[b, a]"
)
// 5. Serialize mytype instance to Json
Assert
.
assertEquals
(
s
"${pretty(render(i))}"
,
"{\n \"$typeName$\":\"mytype\",\n \"e\":1,"
+
"\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}"
)
Assert
.
assertEquals
(
s
"${pretty(render(i))}"
,
"{\n \"$typeName$\":\"mytype\",\n \"e\":1,"
+
"\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03
T08:00:00.000Z
\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}"
)
}
@Test
def
test2
{
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
View file @
e74de9c7
...
...
@@ -47,7 +47,7 @@ public class TypeSystem {
private
static
ThreadLocal
<
SimpleDateFormat
>
dateFormat
=
new
ThreadLocal
()
{
@Override
public
SimpleDateFormat
initialValue
()
{
SimpleDateFormat
dateFormat
=
new
SimpleDateFormat
(
"yyyy-MM-dd"
);
SimpleDateFormat
dateFormat
=
new
SimpleDateFormat
(
"yyyy-MM-dd
'T'HH:mm:ss.SSSX
"
);
dateFormat
.
setTimeZone
(
TimeZone
.
getTimeZone
(
"UTC"
));
return
dateFormat
;
}
...
...
typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/BaseTest.java
View file @
e74de9c7
...
...
@@ -37,6 +37,8 @@ public abstract class BaseTest {
public
static
final
String
STRUCT_TYPE_1
=
"t1"
;
public
static
final
String
STRUCT_TYPE_2
=
"t2"
;
public
static
final
String
TEST_DATE
=
"2014-12-11T02:35:58.440Z"
;
public
static
final
long
TEST_DATE_IN_LONG
=
1418265358440L
;
public
static
Struct
createStruct
()
throws
MetadataException
{
StructType
structType
=
TypeSystem
.
getInstance
().
getDataType
(
...
...
typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/EnumTest.java
View file @
e74de9c7
...
...
@@ -179,7 +179,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tenum3 : \tCOMMITTED\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.1, 1.1]\n"
+
"\to : \t{b=2.0, a=1.0}\n"
+
...
...
@@ -211,7 +211,7 @@ public class EnumTest extends BaseTest {
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tenum3 : \tCOMMITTED\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.1, 1.1]\n"
+
"\to : \t{b=2.0, a=1.0}\n"
+
...
...
typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/StructTest.java
View file @
e74de9c7
...
...
@@ -54,7 +54,7 @@ public class StructTest extends BaseTest {
"\ti : \t1.0\n"
+
"\tj : \t1\n"
+
"\tk : \t1\n"
+
"\tl : \t
2014-12-11
\n"
+
"\tl : \t
"
+
TEST_DATE
+
"
\n"
+
"\tm : \t[1, 1]\n"
+
"\tn : \t[1.1, 1.1]\n"
+
"\to : \t{b=2.0, a=1.0}\n"
+
...
...
typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
View file @
e74de9c7
...
...
@@ -44,18 +44,18 @@ class SerializationTest extends BaseTest {
val
s
:
Struct
=
BaseTest
.
createStruct
()
val
ts
:
ITypedStruct
=
structType
.
convert
(
s
,
Multiplicity
.
REQUIRED
)
Assert
.
assertEquals
(
ts
.
toString
,
"{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t
2014-12-11
\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}"
)
Assert
.
assertEquals
(
ts
.
toString
,
"{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t
"
+
BaseTest
.
TEST_DATE
+
"
\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}"
)
implicit
val
formats
=
org
.
json4s
.
native
.
Serialization
.
formats
(
NoTypeHints
)
+
new
TypedStructSerializer
+
new
BigDecimalSerializer
+
new
BigIntegerSerializer
//Json representation
val
ser
=
swrite
(
ts
)
Assert
.
assertEquals
(
ser
,
"{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"
2014-12-11T02:35:58.440Z
\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}"
)
Assert
.
assertEquals
(
ser
,
"{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"
"
+
BaseTest
.
TEST_DATE
+
"
\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}"
)
// Typed Struct read back
val
ts1
=
read
[
StructInstance
](
ser
)
Assert
.
assertEquals
(
ts1
.
toString
,
"{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t
2014-12-11
\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}"
)
Assert
.
assertEquals
(
ts1
.
toString
,
"{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t
"
+
BaseTest
.
TEST_DATE
+
"
\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}"
)
}
@Test
def
test2
{
...
...
@@ -70,7 +70,7 @@ class SerializationTest extends BaseTest {
{"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
"c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"b":2.0,"a":1.0}}"""
)
// Typed Struct read from string
Assert
.
assertEquals
(
ts1
.
toString
,
"{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}"
)
Assert
.
assertEquals
(
ts1
.
toString
,
"{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03
T19:38:55.053Z
\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}"
)
}
@Test
def
testTrait
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment