Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
bcfd2e7b
Commit
bcfd2e7b
authored
Jan 15, 2015
by
Venkatesh Seetharam
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ISSUE-32 Make TypeSystem a singleton. Contributed by Venkatesh Seetharam
parent
0c54f27e
Hide whitespace changes
Inline
Side-by-side
Showing
21 changed files
with
136 additions
and
88 deletions
+136
-88
pom.xml
pom.xml
+6
-0
pom.xml
repository/pom.xml
+0
-1
DefaultMetadataService.java
...ache/hadoop/metadata/services/DefaultMetadataService.java
+1
-1
GraphBackedMetadataRepository.java
...doop/metadata/services/GraphBackedMetadataRepository.java
+2
-3
GraphBackedMetadataRepositoryTest.java
.../metadata/services/GraphBackedMetadataRepositoryTest.java
+2
-3
pom.xml
typesystem/pom.xml
+5
-1
MetadataService.java
...main/java/org/apache/hadoop/metadata/MetadataService.java
+0
-10
IRepository.java
.../java/org/apache/hadoop/metadata/storage/IRepository.java
+2
-2
StructInstance.java
...va/org/apache/hadoop/metadata/storage/StructInstance.java
+1
-1
MemRepository.java
.../apache/hadoop/metadata/storage/memory/MemRepository.java
+26
-7
DataTypes.java
...main/java/org/apache/hadoop/metadata/types/DataTypes.java
+14
-12
FieldMapping.java
...n/java/org/apache/hadoop/metadata/types/FieldMapping.java
+1
-1
TypeSystem.java
...ain/java/org/apache/hadoop/metadata/types/TypeSystem.java
+60
-33
Console.scala
...c/main/scala/org/apache/hadoop/metadata/cli/Console.scala
+1
-1
DynamicTypedStruct.scala
...a/org/apache/hadoop/metadata/dsl/DynamicTypedStruct.scala
+1
-1
package.scala
...c/main/scala/org/apache/hadoop/metadata/dsl/package.scala
+1
-1
Main.scala
.../org/apache/hadoop/metadata/tools/simpleserver/Main.scala
+2
-3
BaseTest.java
...em/src/test/java/org/apache/hadoop/metadata/BaseTest.java
+6
-1
ClassTest.java
...m/src/test/java/org/apache/hadoop/metadata/ClassTest.java
+1
-2
TypesSerializationTest.scala
.../apache/hadoop/metadata/json/TypesSerializationTest.scala
+2
-1
TypesJerseyResourceIT.java
.../hadoop/metadata/web/resources/TypesJerseyResourceIT.java
+2
-3
No files found.
pom.xml
View file @
bcfd2e7b
...
@@ -440,6 +440,12 @@
...
@@ -440,6 +440,12 @@
</dependency>
</dependency>
<dependency>
<dependency>
<groupId>
com.google.inject.extensions
</groupId>
<artifactId>
guice-throwingproviders
</artifactId>
<version>
3.0
</version>
</dependency>
<dependency>
<groupId>
joda-time
</groupId>
<groupId>
joda-time
</groupId>
<artifactId>
joda-time
</artifactId>
<artifactId>
joda-time
</artifactId>
<version>
2.5
</version>
<version>
2.5
</version>
...
...
repository/pom.xml
View file @
bcfd2e7b
...
@@ -88,7 +88,6 @@
...
@@ -88,7 +88,6 @@
<dependency>
<dependency>
<groupId>
com.google.inject.extensions
</groupId>
<groupId>
com.google.inject.extensions
</groupId>
<artifactId>
guice-throwingproviders
</artifactId>
<artifactId>
guice-throwingproviders
</artifactId>
<version>
3.0
</version>
</dependency>
</dependency>
<dependency>
<dependency>
...
...
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
View file @
bcfd2e7b
...
@@ -53,7 +53,7 @@ public class DefaultMetadataService implements MetadataService {
...
@@ -53,7 +53,7 @@ public class DefaultMetadataService implements MetadataService {
@Inject
@Inject
DefaultMetadataService
(
MetadataRepository
repository
)
throws
MetadataException
{
DefaultMetadataService
(
MetadataRepository
repository
)
throws
MetadataException
{
this
.
typeSystem
=
new
TypeSystem
();
this
.
typeSystem
=
TypeSystem
.
getInstance
();
this
.
repository
=
repository
;
this
.
repository
=
repository
;
}
}
...
...
repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepository.java
View file @
bcfd2e7b
...
@@ -78,7 +78,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -78,7 +78,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
GraphBackedMetadataRepository
(
GraphService
graphService
)
throws
MetadataException
{
GraphBackedMetadataRepository
(
GraphService
graphService
)
throws
MetadataException
{
this
.
instances
=
new
ConcurrentHashMap
<>();
this
.
instances
=
new
ConcurrentHashMap
<>();
this
.
graphService
=
graphService
;
this
.
graphService
=
graphService
;
this
.
typeSystem
=
new
TypeSystem
();
this
.
typeSystem
=
TypeSystem
.
getInstance
();
}
}
/**
/**
...
@@ -186,7 +186,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -186,7 +186,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
for
(
AttributeInfo
attributeInfo
:
fields
.
values
())
{
for
(
AttributeInfo
attributeInfo
:
fields
.
values
())
{
System
.
out
.
println
(
"*** attributeInfo = "
+
attributeInfo
);
System
.
out
.
println
(
"*** attributeInfo = "
+
attributeInfo
);
final
IDataType
dataType
=
attributeInfo
.
dataType
();
final
IDataType
dataType
=
attributeInfo
.
dataType
();
String
attributeName
=
attributeInfo
.
name
;
Object
attributeValue
=
instance
.
get
(
attributeInfo
.
name
);
Object
attributeValue
=
instance
.
get
(
attributeInfo
.
name
);
switch
(
dataType
.
getTypeCategory
())
{
switch
(
dataType
.
getTypeCategory
())
{
...
@@ -220,7 +219,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -220,7 +219,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break
;
break
;
case
CLASS:
case
CLASS:
Id
id
=
(
Id
)
instance
.
get
(
attribute
N
ame
);
Id
id
=
(
Id
)
instance
.
get
(
attribute
Info
.
n
ame
);
if
(
id
!=
null
)
{
if
(
id
!=
null
)
{
Vertex
referenceVertex
=
idToVertexMap
.
get
(
id
);
Vertex
referenceVertex
=
idToVertexMap
.
get
(
id
);
addEdge
(
entityVertex
,
referenceVertex
,
"references"
);
addEdge
(
entityVertex
,
referenceVertex
,
"references"
);
...
...
repository/src/test/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryTest.java
View file @
bcfd2e7b
...
@@ -49,7 +49,7 @@ public class GraphBackedMetadataRepositoryTest extends RepositoryModuleBaseTest
...
@@ -49,7 +49,7 @@ public class GraphBackedMetadataRepositoryTest extends RepositoryModuleBaseTest
repositoryService
=
super
.
injector
.
getInstance
(
GraphBackedMetadataRepository
.
class
);
repositoryService
=
super
.
injector
.
getInstance
(
GraphBackedMetadataRepository
.
class
);
repositoryService
.
start
();
repositoryService
.
start
();
ts
=
new
TypeSystem
();
ts
=
TypeSystem
.
getInstance
();
repo
=
new
MemRepository
(
ts
);
repo
=
new
MemRepository
(
ts
);
defineDeptEmployeeTypes
(
ts
);
defineDeptEmployeeTypes
(
ts
);
...
@@ -197,4 +197,4 @@ public class GraphBackedMetadataRepositoryTest extends RepositoryModuleBaseTest
...
@@ -197,4 +197,4 @@ public class GraphBackedMetadataRepositoryTest extends RepositoryModuleBaseTest
String
name
,
ImmutableList
<
String
>
superTypes
,
AttributeDefinition
...
attrDefs
)
{
String
name
,
ImmutableList
<
String
>
superTypes
,
AttributeDefinition
...
attrDefs
)
{
return
new
HierarchicalTypeDefinition
(
ClassType
.
class
,
name
,
superTypes
,
attrDefs
);
return
new
HierarchicalTypeDefinition
(
ClassType
.
class
,
name
,
superTypes
,
attrDefs
);
}
}
}
}
\ No newline at end of file
typesystem/pom.xml
View file @
bcfd2e7b
...
@@ -38,7 +38,7 @@
...
@@ -38,7 +38,7 @@
</prerequisites>
</prerequisites>
<properties>
<properties>
<java.version>
1.
6
</java.version>
<java.version>
1.
7
</java.version>
<scala.version>
2.10.4
</scala.version>
<scala.version>
2.10.4
</scala.version>
<scala.binary.version>
2.10
</scala.binary.version>
<scala.binary.version>
2.10
</scala.binary.version>
<scala.macros.version>
2.0.1
</scala.macros.version>
<scala.macros.version>
2.0.1
</scala.macros.version>
...
@@ -183,6 +183,10 @@
...
@@ -183,6 +183,10 @@
<artifactId>
fastutil
</artifactId>
<artifactId>
fastutil
</artifactId>
<version>
${fastutil.version}
</version>
<version>
${fastutil.version}
</version>
</dependency>
</dependency>
<dependency>
<groupId>
com.google.inject
</groupId>
<artifactId>
guice
</artifactId>
</dependency>
</dependencies>
</dependencies>
<build>
<build>
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/MetadataService.java
View file @
bcfd2e7b
/**
/**
* Licensed to the Apache Software Foundation (ASF) under one
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* or more contributor license agreements. See the NOTICE file
...
@@ -51,15 +50,6 @@ public class MetadataService {
...
@@ -51,15 +50,6 @@ public class MetadataService {
return
r
;
return
r
;
}
}
public
static
TypeSystem
getCurrentTypeSystem
()
throws
MetadataException
{
MetadataService
m
=
currentSvc
.
get
();
TypeSystem
t
=
m
==
null
?
null
:
m
.
getTypeSystem
();
if
(
t
==
null
)
{
throw
new
MetadataException
(
"No TypeSystem associated with current thread"
);
}
return
t
;
}
public
MetadataService
(
IRepository
repo
,
TypeSystem
typeSystem
)
{
public
MetadataService
(
IRepository
repo
,
TypeSystem
typeSystem
)
{
this
.
typeSystem
=
typeSystem
;
this
.
typeSystem
=
typeSystem
;
this
.
repo
=
repo
;
this
.
repo
=
repo
;
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/storage/IRepository.java
View file @
bcfd2e7b
...
@@ -18,20 +18,20 @@
...
@@ -18,20 +18,20 @@
package
org
.
apache
.
hadoop
.
metadata
.
storage
;
package
org
.
apache
.
hadoop
.
metadata
.
storage
;
import
org.apache.hadoop.metadata.IReferenceableInstance
;
import
org.apache.hadoop.metadata.IReferenceableInstance
;
import
org.apache.hadoop.metadata.ITypedInstance
;
import
org.apache.hadoop.metadata.ITypedReferenceableInstance
;
import
org.apache.hadoop.metadata.ITypedReferenceableInstance
;
import
org.apache.hadoop.metadata.types.ClassType
;
import
org.apache.hadoop.metadata.types.ClassType
;
import
org.apache.hadoop.metadata.types.HierarchicalType
;
import
org.apache.hadoop.metadata.types.HierarchicalType
;
import
org.apache.hadoop.metadata.types.TraitType
;
import
org.apache.hadoop.metadata.types.TraitType
;
import
java.text.DateFormat
;
import
java.util.List
;
import
java.util.List
;
public
interface
IRepository
{
public
interface
IRepository
{
/*
DateFormat getDateFormat();
DateFormat getDateFormat();
DateFormat getTimestampFormat();
DateFormat getTimestampFormat();
boolean allowNullsInCollections();
boolean allowNullsInCollections();
*/
ITypedReferenceableInstance
create
(
IReferenceableInstance
i
)
throws
RepositoryException
;
ITypedReferenceableInstance
create
(
IReferenceableInstance
i
)
throws
RepositoryException
;
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/storage/StructInstance.java
View file @
bcfd2e7b
...
@@ -105,7 +105,7 @@ public class StructInstance implements ITypedStruct {
...
@@ -105,7 +105,7 @@ public class StructInstance implements ITypedStruct {
if
(
val
!=
null
&&
val
instanceof
Id
)
{
if
(
val
!=
null
&&
val
instanceof
Id
)
{
ClassType
clsType
=
ClassType
clsType
=
MetadataService
.
getCurrentTypeSystem
().
getDataType
(
ClassType
.
class
,
i
.
dataType
().
getName
());
TypeSystem
.
getInstance
().
getDataType
(
ClassType
.
class
,
i
.
dataType
().
getName
());
clsType
.
validateId
((
Id
)
val
);
clsType
.
validateId
((
Id
)
val
);
cVal
=
val
;
cVal
=
val
;
}
else
{
}
else
{
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/MemRepository.java
View file @
bcfd2e7b
...
@@ -21,18 +21,35 @@ package org.apache.hadoop.metadata.storage.memory;
...
@@ -21,18 +21,35 @@ package org.apache.hadoop.metadata.storage.memory;
import
org.apache.hadoop.metadata.IReferenceableInstance
;
import
org.apache.hadoop.metadata.IReferenceableInstance
;
import
org.apache.hadoop.metadata.ITypedReferenceableInstance
;
import
org.apache.hadoop.metadata.ITypedReferenceableInstance
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.storage.*
;
import
org.apache.hadoop.metadata.storage.DiscoverInstances
;
import
org.apache.hadoop.metadata.types.*
;
import
org.apache.hadoop.metadata.storage.IRepository
;
import
org.apache.hadoop.metadata.storage.Id
;
import
java.text.DateFormat
;
import
org.apache.hadoop.metadata.storage.MapIds
;
import
java.text.SimpleDateFormat
;
import
org.apache.hadoop.metadata.storage.ReferenceableInstance
;
import
java.util.*
;
import
org.apache.hadoop.metadata.storage.RepositoryException
;
import
org.apache.hadoop.metadata.types.ClassType
;
import
org.apache.hadoop.metadata.types.DataTypes
;
import
org.apache.hadoop.metadata.types.HierarchicalType
;
import
org.apache.hadoop.metadata.types.Multiplicity
;
import
org.apache.hadoop.metadata.types.ObjectGraphWalker
;
import
org.apache.hadoop.metadata.types.TraitType
;
import
org.apache.hadoop.metadata.types.TypeSystem
;
import
java.util.ArrayList
;
import
java.util.Collections
;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
import
java.util.TreeSet
;
import
java.util.concurrent.atomic.AtomicInteger
;
import
java.util.concurrent.atomic.AtomicInteger
;
public
class
MemRepository
implements
IRepository
{
public
class
MemRepository
implements
IRepository
{
/*
public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
public static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd");
public static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd");
*/
final
TypeSystem
typeSystem
;
final
TypeSystem
typeSystem
;
/*
/*
...
@@ -43,9 +60,10 @@ public class MemRepository implements IRepository {
...
@@ -43,9 +60,10 @@ public class MemRepository implements IRepository {
public
MemRepository
(
TypeSystem
typeSystem
)
{
public
MemRepository
(
TypeSystem
typeSystem
)
{
this
.
typeSystem
=
typeSystem
;
this
.
typeSystem
=
typeSystem
;
this
.
typeStores
=
new
HashMap
<
String
,
HierarchicalTypeStore
>();
this
.
typeStores
=
new
HashMap
<>();
}
}
/*
@Override
@Override
public DateFormat getDateFormat() {
public DateFormat getDateFormat() {
return dateFormat;
return dateFormat;
...
@@ -60,6 +78,7 @@ public class MemRepository implements IRepository {
...
@@ -60,6 +78,7 @@ public class MemRepository implements IRepository {
public boolean allowNullsInCollections() {
public boolean allowNullsInCollections() {
return false;
return false;
}
}
*/
@Override
@Override
public
Id
newId
(
String
typeName
)
{
public
Id
newId
(
String
typeName
)
{
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/types/DataTypes.java
View file @
bcfd2e7b
...
@@ -19,24 +19,22 @@
...
@@ -19,24 +19,22 @@
package
org
.
apache
.
hadoop
.
metadata
.
types
;
package
org
.
apache
.
hadoop
.
metadata
.
types
;
import
com.google.common.collect.ImmutableCollection
;
import
com.google.common.collect.ImmutableCollection
;
import
com.google.common.collect.ImmutableCollection.Builder
;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.ImmutableMap
;
import
com.google.common.collect.ImmutableMap
;
import
com.google.common.collect.ImmutableSet
;
import
com.google.common.collect.ImmutableSet
;
import
org.apache.hadoop.metadata.IReferenceableInstance
;
import
org.apache.hadoop.metadata.IReferenceableInstance
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataService
;
import
org.apache.hadoop.metadata.MetadataService
;
import
org.apache.hadoop.metadata.storage.Id
;
import
org.apache.hadoop.metadata.IReferenceableInstance
;
import
org.apache.hadoop.metadata.storage.IRepository
;
import
org.apache.hadoop.metadata.storage.IRepository
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataService
;
import
org.apache.hadoop.metadata.storage.Id
;
import
org.apache.hadoop.metadata.storage.Id
;
import
java.math.BigDecimal
;
import
java.math.BigDecimal
;
import
java.math.BigInteger
;
import
java.math.BigInteger
;
import
java.text.ParseException
;
import
java.text.ParseException
;
import
java.util.*
;
import
java.util.Collection
;
import
java.util.Date
;
import
java.util.Iterator
;
import
java.util.Map
;
public
class
DataTypes
{
public
class
DataTypes
{
...
@@ -388,7 +386,7 @@ public class DataTypes {
...
@@ -388,7 +386,7 @@ public class DataTypes {
return
(
Date
)
val
;
return
(
Date
)
val
;
}
else
if
(
val
instanceof
String
)
{
}
else
if
(
val
instanceof
String
)
{
try
{
try
{
return
MetadataService
.
getCurrentRepository
().
getDateFormat
().
parse
((
String
)
val
);
return
TypeSystem
.
getInstance
().
getDateFormat
().
parse
((
String
)
val
);
}
catch
(
ParseException
ne
)
{
}
catch
(
ParseException
ne
)
{
throw
new
ValueConversionException
(
this
,
val
,
ne
);
throw
new
ValueConversionException
(
this
,
val
,
ne
);
}
}
...
@@ -404,7 +402,7 @@ public class DataTypes {
...
@@ -404,7 +402,7 @@ public class DataTypes {
@Override
@Override
public
void
output
(
Date
val
,
Appendable
buf
,
String
prefix
)
throws
MetadataException
{
public
void
output
(
Date
val
,
Appendable
buf
,
String
prefix
)
throws
MetadataException
{
TypeUtils
.
outputVal
(
val
==
null
?
"<null>"
:
TypeUtils
.
outputVal
(
val
==
null
?
"<null>"
:
MetadataService
.
getCurrentRepository
().
getDateFormat
().
format
(
val
),
buf
,
prefix
);
TypeSystem
.
getInstance
().
getDateFormat
().
format
(
val
),
buf
,
prefix
);
}
}
public
Date
nullValue
()
{
public
Date
nullValue
()
{
...
@@ -488,14 +486,16 @@ public class DataTypes {
...
@@ -488,14 +486,16 @@ public class DataTypes {
ImmutableCollection
.
Builder
b
=
m
.
isUnique
?
ImmutableSet
.
builder
()
:
ImmutableList
.
builder
();
ImmutableCollection
.
Builder
b
=
m
.
isUnique
?
ImmutableSet
.
builder
()
:
ImmutableList
.
builder
();
while
(
it
.
hasNext
()
)
{
while
(
it
.
hasNext
()
)
{
b
.
add
(
elemType
.
convert
(
it
.
next
(),
b
.
add
(
elemType
.
convert
(
it
.
next
(),
r
.
allowNullsInCollections
()
?
Multiplicity
.
OPTIONAL
:
Multiplicity
.
REQUIRED
));
TypeSystem
.
getInstance
().
allowNullsInCollections
()
?
Multiplicity
.
OPTIONAL
:
Multiplicity
.
REQUIRED
));
}
}
return
b
.
build
();
return
b
.
build
();
}
}
else
{
else
{
try
{
try
{
return
ImmutableList
.
of
(
elemType
.
convert
(
val
,
return
ImmutableList
.
of
(
elemType
.
convert
(
val
,
r
.
allowNullsInCollections
()
?
Multiplicity
.
OPTIONAL
:
Multiplicity
.
REQUIRED
));
TypeSystem
.
getInstance
().
allowNullsInCollections
()
?
Multiplicity
.
OPTIONAL
:
Multiplicity
.
REQUIRED
));
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
throw
new
ValueConversionException
(
this
,
val
,
e
);
throw
new
ValueConversionException
(
this
,
val
,
e
);
}
}
...
@@ -592,9 +592,11 @@ public class DataTypes {
...
@@ -592,9 +592,11 @@ public class DataTypes {
while
(
it
.
hasNext
()
)
{
while
(
it
.
hasNext
()
)
{
Map
.
Entry
e
=
it
.
next
();
Map
.
Entry
e
=
it
.
next
();
b
.
put
(
keyType
.
convert
(
e
.
getKey
(),
b
.
put
(
keyType
.
convert
(
e
.
getKey
(),
r
.
allowNullsInCollections
()
?
Multiplicity
.
OPTIONAL
:
Multiplicity
.
REQUIRED
),
TypeSystem
.
getInstance
().
allowNullsInCollections
()
?
Multiplicity
.
OPTIONAL
:
Multiplicity
.
REQUIRED
),
valueType
.
convert
(
e
.
getValue
(),
valueType
.
convert
(
e
.
getValue
(),
r
.
allowNullsInCollections
()
?
Multiplicity
.
OPTIONAL
:
Multiplicity
.
REQUIRED
));
TypeSystem
.
getInstance
().
allowNullsInCollections
()
?
Multiplicity
.
OPTIONAL
:
Multiplicity
.
REQUIRED
));
}
}
return
b
.
build
();
return
b
.
build
();
}
}
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/types/FieldMapping.java
View file @
bcfd2e7b
...
@@ -116,7 +116,7 @@ public class FieldMapping {
...
@@ -116,7 +116,7 @@ public class FieldMapping {
outputFields
(
s
,
buf
,
fieldPrefix
);
outputFields
(
s
,
buf
,
fieldPrefix
);
TypeSystem
ts
=
MetadataService
.
getCurrentTypeSystem
();
TypeSystem
ts
=
TypeSystem
.
getInstance
();
for
(
String
sT
:
s
.
getTraits
()
)
{
for
(
String
sT
:
s
.
getTraits
()
)
{
TraitType
tt
=
ts
.
getDataType
(
TraitType
.
class
,
sT
);
TraitType
tt
=
ts
.
getDataType
(
TraitType
.
class
,
sT
);
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/types/TypeSystem.java
View file @
bcfd2e7b
...
@@ -20,22 +20,40 @@ package org.apache.hadoop.metadata.types;
...
@@ -20,22 +20,40 @@ package org.apache.hadoop.metadata.types;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.ImmutableList
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.TypesDef
;
import
org.apache.hadoop.metadata.TypesDef
;
import
javax.inject.Singleton
;
import
java.lang.reflect.Constructor
;
import
java.lang.reflect.Constructor
;
import
java.text.DateFormat
;
import
java.text.SimpleDateFormat
;
import
java.util.*
;
import
java.util.*
;
@Singleton
public
class
TypeSystem
{
public
class
TypeSystem
{
private
Map
<
String
,
IDataType
>
types
;
private
Map
<
String
,
IDataType
>
types
;
public
TypeSystem
()
throws
MetadataException
{
private
TypeSystem
()
{
types
=
new
HashMap
<
String
,
IDataType
>();
initialize
();
}
/**
* This is only used for testing prurposes.
* @nonpublic
*/
public
void
reset
()
{
initialize
();
}
private
void
initialize
()
{
types
=
new
HashMap
<>();
registerPrimitiveTypes
();
registerPrimitiveTypes
();
}
}
private
TypeSystem
(
TypeSystem
ts
)
{
private
static
final
TypeSystem
INSTANCE
=
new
TypeSystem
();
public
static
TypeSystem
getInstance
()
{
return
INSTANCE
;
}
}
public
ImmutableList
<
String
>
getTypeNames
()
{
public
ImmutableList
<
String
>
getTypeNames
()
{
...
@@ -87,27 +105,29 @@ public class TypeSystem {
...
@@ -87,27 +105,29 @@ public class TypeSystem {
boolean
errorIfExists
,
boolean
errorIfExists
,
AttributeDefinition
...
attrDefs
)
throws
MetadataException
{
AttributeDefinition
...
attrDefs
)
throws
MetadataException
{
StructTypeDefinition
structDef
=
new
StructTypeDefinition
(
name
,
attrDefs
);
StructTypeDefinition
structDef
=
new
StructTypeDefinition
(
name
,
attrDefs
);
Map
<
String
,
IDataType
>
newTypes
=
defineTypes
(
ImmutableList
.<
StructTypeDefinition
>
of
(
structDef
),
defineTypes
(
ImmutableList
.
of
(
structDef
),
ImmutableList
.<
HierarchicalTypeDefinition
<
TraitType
>>
of
(),
ImmutableList
.<
HierarchicalTypeDefinition
<
TraitType
>>
of
(),
ImmutableList
.<
HierarchicalTypeDefinition
<
ClassType
>>
of
());
ImmutableList
.<
HierarchicalTypeDefinition
<
ClassType
>>
of
());
return
getDataType
(
StructType
.
class
,
structDef
.
typeName
);
return
getDataType
(
StructType
.
class
,
structDef
.
typeName
);
}
}
public
TraitType
defineTraitType
(
HierarchicalTypeDefinition
<
TraitType
>
traitDef
public
TraitType
defineTraitType
(
HierarchicalTypeDefinition
<
TraitType
>
traitDef
)
)
throws
MetadataException
{
throws
MetadataException
{
Map
<
String
,
IDataType
>
newTypes
=
defineTypes
(
ImmutableList
.<
StructTypeDefinition
>
of
(),
ImmutableList
.<
HierarchicalTypeDefinition
<
TraitType
>>
of
(
traitDef
),
defineTypes
(
ImmutableList
.<
StructTypeDefinition
>
of
(),
ImmutableList
.
of
(
traitDef
),
ImmutableList
.<
HierarchicalTypeDefinition
<
ClassType
>>
of
());
ImmutableList
.<
HierarchicalTypeDefinition
<
ClassType
>>
of
());
return
getDataType
(
TraitType
.
class
,
traitDef
.
typeName
);
return
getDataType
(
TraitType
.
class
,
traitDef
.
typeName
);
}
}
public
ClassType
defineClassType
(
HierarchicalTypeDefinition
<
ClassType
>
classDef
public
ClassType
defineClassType
(
HierarchicalTypeDefinition
<
ClassType
>
classDef
)
)
throws
MetadataException
{
throws
MetadataException
{
Map
<
String
,
IDataType
>
newTypes
=
defineTypes
(
ImmutableList
.<
StructTypeDefinition
>
of
(),
defineTypes
(
ImmutableList
.<
StructTypeDefinition
>
of
(),
ImmutableList
.<
HierarchicalTypeDefinition
<
TraitType
>>
of
(),
ImmutableList
.<
HierarchicalTypeDefinition
<
TraitType
>>
of
(),
ImmutableList
.
<
HierarchicalTypeDefinition
<
ClassType
>>
of
(
classDef
));
ImmutableList
.
of
(
classDef
));
return
getDataType
(
ClassType
.
class
,
classDef
.
typeName
);
return
getDataType
(
ClassType
.
class
,
classDef
.
typeName
);
}
}
...
@@ -115,7 +135,7 @@ public class TypeSystem {
...
@@ -115,7 +135,7 @@ public class TypeSystem {
public
Map
<
String
,
IDataType
>
defineTraitTypes
(
HierarchicalTypeDefinition
<
TraitType
>...
traitDefs
)
public
Map
<
String
,
IDataType
>
defineTraitTypes
(
HierarchicalTypeDefinition
<
TraitType
>...
traitDefs
)
throws
MetadataException
{
throws
MetadataException
{
TransientTypeSystem
transientTypes
=
new
TransientTypeSystem
(
ImmutableList
.<
StructTypeDefinition
>
of
(),
TransientTypeSystem
transientTypes
=
new
TransientTypeSystem
(
ImmutableList
.<
StructTypeDefinition
>
of
(),
ImmutableList
.
<
HierarchicalTypeDefinition
<
TraitType
>>
copyOf
(
traitDefs
),
ImmutableList
.
copyOf
(
traitDefs
),
ImmutableList
.<
HierarchicalTypeDefinition
<
ClassType
>>
of
());
ImmutableList
.<
HierarchicalTypeDefinition
<
ClassType
>>
of
());
return
transientTypes
.
defineTypes
();
return
transientTypes
.
defineTypes
();
}
}
...
@@ -180,11 +200,11 @@ public class TypeSystem {
...
@@ -180,11 +200,11 @@ public class TypeSystem {
final
ImmutableList
<
StructTypeDefinition
>
structDefs
;
final
ImmutableList
<
StructTypeDefinition
>
structDefs
;
final
ImmutableList
<
HierarchicalTypeDefinition
<
TraitType
>>
traitDefs
;
final
ImmutableList
<
HierarchicalTypeDefinition
<
TraitType
>>
traitDefs
;
final
ImmutableList
<
HierarchicalTypeDefinition
<
ClassType
>>
classDefs
;
final
ImmutableList
<
HierarchicalTypeDefinition
<
ClassType
>>
classDefs
;
Map
<
String
,
StructTypeDefinition
>
structNameToDefMap
=
new
HashMap
<
String
,
StructTypeDefinition
>();
Map
<
String
,
StructTypeDefinition
>
structNameToDefMap
=
new
HashMap
<>();
Map
<
String
,
HierarchicalTypeDefinition
<
TraitType
>>
traitNameToDefMap
=
Map
<
String
,
HierarchicalTypeDefinition
<
TraitType
>>
traitNameToDefMap
=
new
HashMap
<
String
,
HierarchicalTypeDefinition
<
TraitType
>
>();
new
HashMap
<>();
Map
<
String
,
HierarchicalTypeDefinition
<
ClassType
>>
classNameToDefMap
=
Map
<
String
,
HierarchicalTypeDefinition
<
ClassType
>>
classNameToDefMap
=
new
HashMap
<
String
,
HierarchicalTypeDefinition
<
ClassType
>
>();
new
HashMap
<>();
Set
<
String
>
transientTypes
;
Set
<
String
>
transientTypes
;
...
@@ -197,20 +217,17 @@ public class TypeSystem {
...
@@ -197,20 +217,17 @@ public class TypeSystem {
ImmutableList
<
HierarchicalTypeDefinition
<
TraitType
>>
traitDefs
,
ImmutableList
<
HierarchicalTypeDefinition
<
TraitType
>>
traitDefs
,
ImmutableList
<
HierarchicalTypeDefinition
<
ClassType
>>
classDefs
)
{
ImmutableList
<
HierarchicalTypeDefinition
<
ClassType
>>
classDefs
)
{
super
(
TypeSystem
.
this
);
this
.
structDefs
=
structDefs
;
this
.
structDefs
=
structDefs
;
this
.
traitDefs
=
traitDefs
;
this
.
traitDefs
=
traitDefs
;
this
.
classDefs
=
classDefs
;
this
.
classDefs
=
classDefs
;
structNameToDefMap
=
new
HashMap
<
String
,
StructTypeDefinition
>();
structNameToDefMap
=
new
HashMap
<>();
traitNameToDefMap
=
traitNameToDefMap
=
new
HashMap
<>();
new
HashMap
<
String
,
HierarchicalTypeDefinition
<
TraitType
>>();
classNameToDefMap
=
new
HashMap
<>();
classNameToDefMap
=
new
HashMap
<
String
,
HierarchicalTypeDefinition
<
ClassType
>>();
recursiveRefs
=
new
ArrayList
<>();
recursiveArrayTypes
=
new
ArrayList
<>();
recursiveRefs
=
new
ArrayList
<
AttributeInfo
>();
recursiveMapTypes
=
new
ArrayList
<>();
recursiveArrayTypes
=
new
ArrayList
<
DataTypes
.
ArrayType
>();
transientTypes
=
new
LinkedHashSet
<>();
recursiveMapTypes
=
new
ArrayList
<
DataTypes
.
MapType
>();
transientTypes
=
new
LinkedHashSet
<
String
>();
}
}
private
IDataType
dataType
(
String
name
)
{
private
IDataType
dataType
(
String
name
)
{
...
@@ -264,9 +281,10 @@ public class TypeSystem {
...
@@ -264,9 +281,10 @@ public class TypeSystem {
}
}
}
}
private
<
U
extends
HierarchicalType
>
void
validateSuperTypes
(
Class
<
U
>
cls
,
HierarchicalTypeDefinition
<
U
>
def
)
private
<
U
extends
HierarchicalType
>
void
validateSuperTypes
(
Class
<
U
>
cls
,
HierarchicalTypeDefinition
<
U
>
def
)
throws
MetadataException
{
throws
MetadataException
{
Set
<
String
>
s
=
new
HashSet
<
String
>();
Set
<
String
>
s
=
new
HashSet
<>();
ImmutableList
<
String
>
superTypes
=
def
.
superTypes
;
ImmutableList
<
String
>
superTypes
=
def
.
superTypes
;
for
(
String
superTypeName
:
superTypes
)
{
for
(
String
superTypeName
:
superTypes
)
{
...
@@ -373,13 +391,13 @@ public class TypeSystem {
...
@@ -373,13 +391,13 @@ public class TypeSystem {
*/
*/
private
void
step3
()
throws
MetadataException
{
private
void
step3
()
throws
MetadataException
{
List
<
TraitType
>
traitTypes
=
new
ArrayList
<
TraitType
>();
List
<
TraitType
>
traitTypes
=
new
ArrayList
<>();
for
(
String
traitTypeName
:
traitNameToDefMap
.
keySet
())
{
for
(
String
traitTypeName
:
traitNameToDefMap
.
keySet
())
{
traitTypes
.
add
(
getDataType
(
TraitType
.
class
,
traitTypeName
));
traitTypes
.
add
(
getDataType
(
TraitType
.
class
,
traitTypeName
));
}
}
Collections
.
sort
(
traitTypes
);
Collections
.
sort
(
traitTypes
);
List
<
ClassType
>
classTypes
=
new
ArrayList
<
ClassType
>();
List
<
ClassType
>
classTypes
=
new
ArrayList
<>();
for
(
String
classTypeName
:
classNameToDefMap
.
keySet
())
{
for
(
String
classTypeName
:
classNameToDefMap
.
keySet
())
{
classTypes
.
add
(
getDataType
(
ClassType
.
class
,
classTypeName
));
classTypes
.
add
(
getDataType
(
ClassType
.
class
,
classTypeName
));
}
}
...
@@ -429,7 +447,7 @@ public class TypeSystem {
...
@@ -429,7 +447,7 @@ public class TypeSystem {
throw
me
;
throw
me
;
}
}
Map
<
String
,
IDataType
>
newTypes
=
new
HashMap
<
String
,
IDataType
>();
Map
<
String
,
IDataType
>
newTypes
=
new
HashMap
<>();
for
(
String
tName
:
transientTypes
)
{
for
(
String
tName
:
transientTypes
)
{
newTypes
.
put
(
tName
,
dataType
(
tName
));
newTypes
.
put
(
tName
,
dataType
(
tName
));
...
@@ -483,4 +501,13 @@ public class TypeSystem {
...
@@ -483,4 +501,13 @@ public class TypeSystem {
}
}
}
}
public
static
SimpleDateFormat
dateFormat
=
new
SimpleDateFormat
(
"yyyy-MM-dd"
);
public
DateFormat
getDateFormat
()
{
return
dateFormat
;
}
public
boolean
allowNullsInCollections
()
{
return
false
;
}
}
}
typesystem/src/main/scala/org/apache/hadoop/metadata/cli/Console.scala
View file @
bcfd2e7b
...
@@ -44,7 +44,7 @@ class SampleILoop extends ILoop {
...
@@ -44,7 +44,7 @@ class SampleILoop extends ILoop {
//intp = Console.in
//intp = Console.in
val
ts
:
TypeSystem
=
new
TypeSystem
val
ts
:
TypeSystem
=
TypeSystem
.
getInstance
()
val
mr
:
MemRepository
=
new
MemRepository
(
ts
)
val
mr
:
MemRepository
=
new
MemRepository
(
ts
)
val
ms
:
MetadataService
=
new
MetadataService
(
mr
,
ts
)
val
ms
:
MetadataService
=
new
MetadataService
(
mr
,
ts
)
MetadataService
.
setCurrentService
(
ms
)
MetadataService
.
setCurrentService
(
ms
)
...
...
typesystem/src/main/scala/org/apache/hadoop/metadata/dsl/DynamicTypedStruct.scala
View file @
bcfd2e7b
...
@@ -32,5 +32,5 @@ class DynamicTypedStruct(val ts : ITypedStruct) extends Dynamic {
...
@@ -32,5 +32,5 @@ class DynamicTypedStruct(val ts : ITypedStruct) extends Dynamic {
}
}
ts
.
set
(
name
,
value1
)
ts
.
set
(
name
,
value1
)
}
}
def
dataType
=
MetadataService
.
getCurrentTypeSystem
.
getDataType
(
classOf
[
StructType
],
ts
.
getTypeName
)
def
dataType
=
TypeSystem
.
getInstance
()
.
getDataType
(
classOf
[
StructType
],
ts
.
getTypeName
)
}
}
typesystem/src/main/scala/org/apache/hadoop/metadata/dsl/package.scala
View file @
bcfd2e7b
...
@@ -42,7 +42,7 @@ package object dsl {
...
@@ -42,7 +42,7 @@ package object dsl {
new
BigDecimalSerializer
+
new
BigIntegerSerializer
new
BigDecimalSerializer
+
new
BigIntegerSerializer
def
service
=
MetadataService
.
getCurrentService
def
service
=
MetadataService
.
getCurrentService
def
ts
=
MetadataService
.
getCurrentTypeSystem
def
ts
=
TypeSystem
.
getInstance
def
repo
=
MetadataService
.
getCurrentRepository
def
repo
=
MetadataService
.
getCurrentRepository
val
BOOLEAN_TYPE
=
DataTypes
.
BOOLEAN_TYPE
val
BOOLEAN_TYPE
=
DataTypes
.
BOOLEAN_TYPE
...
...
typesystem/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/Main.scala
View file @
bcfd2e7b
...
@@ -51,9 +51,9 @@ object Main extends App {
...
@@ -51,9 +51,9 @@ object Main extends App {
implicit
val
system
=
ActorSystem
(
"metadataservice"
)
implicit
val
system
=
ActorSystem
(
"metadataservice"
)
val
typSys
=
new
TypeSystem
val
typSys
=
TypeSystem
.
getInstance
()
val
memRepo
=
new
MemRepository
(
typSys
)
val
memRepo
=
new
MemRepository
(
typSys
)
val
api
=
system
.
actorOf
(
Props
(
new
RestInterface
(
typSys
,
memRepo
)),
"httpInterface"
)
val
api
=
system
.
actorOf
(
Props
(
new
RestInterface
(
typSys
,
memRepo
)),
"httpInterface"
)
IO
(
Http
)
!
Http
.
Bind
(
listener
=
api
,
interface
=
host
,
port
=
port
)
IO
(
Http
)
!
Http
.
Bind
(
listener
=
api
,
interface
=
host
,
port
=
port
)
}
}
\ No newline at end of file
typesystem/src/test/java/org/apache/hadoop/metadata/BaseTest.java
View file @
bcfd2e7b
...
@@ -41,10 +41,15 @@ public abstract class BaseTest {
...
@@ -41,10 +41,15 @@ public abstract class BaseTest {
public
static
final
String
STRUCT_TYPE_1
=
"t1"
;
public
static
final
String
STRUCT_TYPE_1
=
"t1"
;
public
static
final
String
STRUCT_TYPE_2
=
"t2"
;
public
static
final
String
STRUCT_TYPE_2
=
"t2"
;
protected
TypeSystem
getTypeSystem
()
{
return
TypeSystem
.
getInstance
();
}
@Before
@Before
public
void
setup
()
throws
MetadataException
{
public
void
setup
()
throws
MetadataException
{
TypeSystem
ts
=
new
TypeSystem
();
TypeSystem
ts
=
TypeSystem
.
getInstance
();
ts
.
reset
();
MemRepository
mr
=
new
MemRepository
(
ts
);
MemRepository
mr
=
new
MemRepository
(
ts
);
ms
=
new
MetadataService
(
mr
,
ts
);
ms
=
new
MetadataService
(
mr
,
ts
);
MetadataService
.
setCurrentService
(
ms
);
MetadataService
.
setCurrentService
(
ms
);
...
...
typesystem/src/test/java/org/apache/hadoop/metadata/ClassTest.java
View file @
bcfd2e7b
package
org
.
apache
.
hadoop
.
metadata
;
package
org
.
apache
.
hadoop
.
metadata
;
import
com.google.common.collect.ImmutableList
;
import
org.apache.hadoop.metadata.types.ClassType
;
import
org.apache.hadoop.metadata.types.ClassType
;
import
org.apache.hadoop.metadata.types.*
;
import
org.apache.hadoop.metadata.types.*
;
import
org.junit.Assert
;
import
org.junit.Assert
;
...
@@ -17,7 +16,7 @@ public class ClassTest extends BaseTest {
...
@@ -17,7 +16,7 @@ public class ClassTest extends BaseTest {
@Test
@Test
public
void
test1
()
throws
MetadataException
{
public
void
test1
()
throws
MetadataException
{
TypeSystem
ts
=
ms
.
getTypeSystem
();
TypeSystem
ts
=
getTypeSystem
();
defineDeptEmployeeTypes
(
ts
);
defineDeptEmployeeTypes
(
ts
);
Referenceable
hrDept
=
createDeptEg1
(
ts
);
Referenceable
hrDept
=
createDeptEg1
(
ts
);
...
...
typesystem/src/test/scala/org/apache/hadoop/metadata/json/TypesSerializationTest.scala
View file @
bcfd2e7b
...
@@ -132,7 +132,8 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
...
@@ -132,7 +132,8 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
val
typesDef1
=
TypesSerialization
.
fromJson
(
ser
)
val
typesDef1
=
TypesSerialization
.
fromJson
(
ser
)
val
ts1
=
new
TypeSystem
()
val
ts1
=
TypeSystem
.
getInstance
()
ts1
.
reset
()
typesDef1
.
enumTypes
.
foreach
(
ts1
.
defineEnumType
(
_
))
typesDef1
.
enumTypes
.
foreach
(
ts1
.
defineEnumType
(
_
))
...
...
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java
View file @
bcfd2e7b
...
@@ -56,7 +56,7 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
...
@@ -56,7 +56,7 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
public
void
setUp
()
throws
Exception
{
public
void
setUp
()
throws
Exception
{
super
.
setUp
();
super
.
setUp
();
typeSystem
=
new
TypeSystem
();
typeSystem
=
TypeSystem
.
getInstance
();
typeDefinitions
=
createHiveTypes
();
typeDefinitions
=
createHiveTypes
();
}
}
...
@@ -183,4 +183,4 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
...
@@ -183,4 +183,4 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
return
typeDefinitions
;
return
typeDefinitions
;
}
}
}
}
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment