Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
a0d7b998
Commit
a0d7b998
authored
May 01, 2015
by
Venkatesh Seetharam
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
BUG-33658 Add missing supertype in the graph repository
parent
5fda7b70
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
15 changed files
with
417 additions
and
44 deletions
+417
-44
DefaultGraphPersistenceStrategy.java
...data/discovery/graph/DefaultGraphPersistenceStrategy.java
+6
-1
GraphBackedDiscoveryService.java
...metadata/discovery/graph/GraphBackedDiscoveryService.java
+0
-3
Constants.java
...java/org/apache/hadoop/metadata/repository/Constants.java
+9
-0
MetadataRepository.java
...apache/hadoop/metadata/repository/MetadataRepository.java
+7
-0
GraphBackedMetadataRepository.java
...adata/repository/graph/GraphBackedMetadataRepository.java
+0
-0
GraphBackedSearchIndexer.java
...p/metadata/repository/graph/GraphBackedSearchIndexer.java
+6
-2
GraphHelper.java
.../apache/hadoop/metadata/repository/graph/GraphHelper.java
+13
-3
GraphPersistenceStrategies.scala
...he/hadoop/metadata/query/GraphPersistenceStrategies.scala
+6
-0
TestUtils.java
...y/src/test/java/org/apache/hadoop/metadata/TestUtils.java
+1
-1
GraphBackedDiscoveryServiceTest.java
...p/metadata/discovery/GraphBackedDiscoveryServiceTest.java
+58
-0
GraphBackedMetadataRepositoryTest.java
...a/repository/graph/GraphBackedMetadataRepositoryTest.java
+38
-31
DSLTest.scala
.../scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
+1
-1
TypeSystem.java
...g/apache/hadoop/metadata/typesystem/types/TypeSystem.java
+9
-0
BaseTest.java
...org/apache/hadoop/metadata/typesystem/types/BaseTest.java
+7
-2
TypeInheritanceTest.java
...hadoop/metadata/typesystem/types/TypeInheritanceTest.java
+256
-0
No files found.
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/DefaultGraphPersistenceStrategy.java
View file @
a0d7b998
...
...
@@ -55,6 +55,11 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
}
@Override
public
String
superTypeAttributeName
()
{
return
metadataRepository
.
getSuperTypeAttributeName
();
}
@Override
public
String
edgeLabel
(
IDataType
<?>
dataType
,
AttributeInfo
aInfo
)
{
return
metadataRepository
.
getEdgeLabel
(
dataType
,
aInfo
);
}
...
...
@@ -107,7 +112,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
TypeSystem
.
IdType
idType
=
TypeSystem
.
getInstance
().
getIdType
();
if
(
dataType
.
getName
()
==
idType
.
getName
(
))
{
if
(
dataType
.
getName
().
equals
(
idType
.
getName
()
))
{
structInstance
.
set
(
idType
.
typeNameAttrName
(),
structVertex
.
getProperty
(
typeAttributeName
()));
structInstance
.
set
(
idType
.
idAttrName
(),
...
...
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
View file @
a0d7b998
...
...
@@ -23,8 +23,6 @@ import com.thinkaurelius.titan.core.TitanIndexQuery;
import
com.thinkaurelius.titan.core.TitanProperty
;
import
com.thinkaurelius.titan.core.TitanVertex
;
import
com.tinkerpop.blueprints.Vertex
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.discovery.DiscoveryException
;
import
org.apache.hadoop.metadata.discovery.DiscoveryService
;
import
org.apache.hadoop.metadata.query.Expressions
;
...
...
@@ -52,7 +50,6 @@ import javax.script.ScriptEngine;
import
javax.script.ScriptEngineManager
;
import
javax.script.ScriptException
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.HashMap
;
import
java.util.Iterator
;
import
java.util.List
;
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/Constants.java
View file @
a0d7b998
...
...
@@ -32,6 +32,15 @@ public final class Constants {
public
static
final
String
ENTITY_TYPE_PROPERTY_KEY
=
"typeName"
;
public
static
final
String
ENTITY_TYPE_INDEX
=
"type_index"
;
/**
* Entity type's super types property key.
*/
public
static
final
String
SUPER_TYPES_PROPERTY_KEY
=
"superTypeNames"
;
public
static
final
String
SUPER_TYPES_INDEX
=
"super_types_index"
;
/**
* Full-text for the entity for enabling full-text search.
*/
public
static
final
String
ENTITY_TEXT_PROPERTY_KEY
=
"entityText"
;
/**
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java
View file @
a0d7b998
...
...
@@ -39,6 +39,13 @@ public interface MetadataRepository {
String
getTypeAttributeName
();
/**
* Returns the property key used to store super type names.
*
* @return property key used to store super type names.
*/
String
getSuperTypeAttributeName
();
/**
* Return the property key used to store a given traitName in the repository.
*
* @param dataType data type
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
View file @
a0d7b998
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
View file @
a0d7b998
...
...
@@ -90,10 +90,14 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
createCompositeAndMixedIndex
(
Constants
.
ENTITY_TYPE_INDEX
,
Constants
.
ENTITY_TYPE_PROPERTY_KEY
,
String
.
class
,
false
,
Cardinality
.
SINGLE
);
// create a composite and mixed index for type since it can be combined with other keys
createCompositeAndMixedIndex
(
Constants
.
SUPER_TYPES_INDEX
,
Constants
.
SUPER_TYPES_PROPERTY_KEY
,
String
.
class
,
false
,
Cardinality
.
SET
);
// create a composite and mixed index for traitNames since it can be combined with other
// keys. Traits must be a set and not a list.
createCompositeAndMixedIndex
(
Constants
.
TRAIT_NAMES_INDEX
,
Constants
.
TRAIT_NAMES_PROPERTY_KEY
,
String
.
class
,
false
,
Cardinality
.
SET
);
createCompositeAndMixedIndex
(
Constants
.
TRAIT_NAMES_INDEX
,
Constants
.
TRAIT_NAMES_PROPERTY_KEY
,
String
.
class
,
false
,
Cardinality
.
SET
);
// Index for full text search
createFullTextIndex
();
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
View file @
a0d7b998
...
...
@@ -19,6 +19,7 @@
package
org
.
apache
.
hadoop
.
metadata
.
repository
.
graph
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.TitanVertex
;
import
com.tinkerpop.blueprints.Direction
;
import
com.tinkerpop.blueprints.Edge
;
import
com.tinkerpop.blueprints.Graph
;
...
...
@@ -31,6 +32,7 @@ import org.slf4j.Logger;
import
org.slf4j.LoggerFactory
;
import
java.util.Iterator
;
import
java.util.Set
;
import
java.util.UUID
;
/**
...
...
@@ -44,9 +46,10 @@ public final class GraphHelper {
}
public
static
Vertex
createVertexWithIdentity
(
Graph
graph
,
ITypedReferenceableInstance
typedInstance
)
{
ITypedReferenceableInstance
typedInstance
,
Set
<
String
>
superTypeNames
)
{
final
Vertex
vertexWithIdentity
=
createVertexWithoutIdentity
(
graph
,
typedInstance
.
getTypeName
(),
typedInstance
.
getId
());
graph
,
typedInstance
.
getTypeName
(),
typedInstance
.
getId
()
,
superTypeNames
);
// add identity
final
String
guid
=
UUID
.
randomUUID
().
toString
();
...
...
@@ -57,12 +60,19 @@ public final class GraphHelper {
public
static
Vertex
createVertexWithoutIdentity
(
Graph
graph
,
String
typeName
,
Id
typedInstanceId
)
{
Id
typedInstanceId
,
Set
<
String
>
superTypeNames
)
{
final
Vertex
vertexWithoutIdentity
=
graph
.
addVertex
(
null
);
// add type information
vertexWithoutIdentity
.
setProperty
(
Constants
.
ENTITY_TYPE_PROPERTY_KEY
,
typeName
);
// add super types
for
(
String
superTypeName
:
superTypeNames
)
{
((
TitanVertex
)
vertexWithoutIdentity
).
addProperty
(
Constants
.
SUPER_TYPES_PROPERTY_KEY
,
superTypeName
);
}
// add version information
vertexWithoutIdentity
.
setProperty
(
Constants
.
VERSION_PROPERTY_KEY
,
typedInstanceId
.
version
);
...
...
repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala
View file @
a0d7b998
...
...
@@ -45,6 +45,11 @@ trait GraphPersistenceStrategies {
def
typeAttributeName
:
String
/**
* Name of attribute used to store super type names in vertex.
*/
def
superTypeAttributeName
:
String
/**
* Name of attribute used to store guid in vertex
*/
def
idAttributeName
:
String
...
...
@@ -113,6 +118,7 @@ trait GraphPersistenceStrategies {
object
GraphPersistenceStrategy1
extends
GraphPersistenceStrategies
{
val
typeAttributeName
=
"typeName"
val
superTypeAttributeName
=
"superTypeNames"
val
idAttributeName
=
"guid"
def
edgeLabel
(
dataType
:
IDataType
[
_
],
aInfo
:
AttributeInfo
)
=
s
"${dataType.getName}.${aInfo.name}"
...
...
repository/src/test/java/org/apache/hadoop/metadata/TestUtils.java
View file @
a0d7b998
...
...
@@ -151,7 +151,7 @@ public final class TestUtils {
jane
.
set
(
"name"
,
"Jane"
);
jane
.
set
(
"department"
,
hrDept
);
janeAddr
.
set
(
"street"
,
"Great America
l
Parkway"
);
janeAddr
.
set
(
"street"
,
"Great America Parkway"
);
janeAddr
.
set
(
"city"
,
"Santa Clara"
);
jane
.
set
(
"address"
,
janeAddr
);
...
...
repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java
View file @
a0d7b998
...
...
@@ -18,6 +18,7 @@
package
org
.
apache
.
hadoop
.
metadata
.
discovery
;
import
com.google.common.collect.ImmutableList
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.tinkerpop.blueprints.Edge
;
import
com.tinkerpop.blueprints.Vertex
;
...
...
@@ -33,6 +34,8 @@ import org.apache.hadoop.metadata.repository.graph.GraphProvider;
import
org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance
;
import
org.apache.hadoop.metadata.typesystem.Referenceable
;
import
org.apache.hadoop.metadata.typesystem.types.ClassType
;
import
org.apache.hadoop.metadata.typesystem.types.DataTypes
;
import
org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition
;
import
org.apache.hadoop.metadata.typesystem.types.Multiplicity
;
import
org.apache.hadoop.metadata.typesystem.types.TypeSystem
;
import
org.codehaus.jettison.json.JSONArray
;
...
...
@@ -51,6 +54,10 @@ import javax.script.ScriptEngineManager;
import
javax.script.ScriptException
;
import
java.io.File
;
import
static
org
.
apache
.
hadoop
.
metadata
.
typesystem
.
types
.
utils
.
TypesUtil
.
createClassTypeDef
;
import
static
org
.
apache
.
hadoop
.
metadata
.
typesystem
.
types
.
utils
.
TypesUtil
.
createOptionalAttrDef
;
import
static
org
.
apache
.
hadoop
.
metadata
.
typesystem
.
types
.
utils
.
TypesUtil
.
createRequiredAttrDef
;
@Guice
(
modules
=
RepositoryMetadataModule
.
class
)
public
class
GraphBackedDiscoveryServiceTest
{
...
...
@@ -292,4 +299,54 @@ public class GraphBackedDiscoveryServiceTest {
Assert
.
assertNotEquals
(
name
,
"null"
);
}
}
@Test
public
void
testSearchForTypeInheritance
()
throws
Exception
{
createTypesWithMultiLevelInheritance
();
createInstances
();
String
dslQuery
=
"from D where a = 1"
;
String
jsonResults
=
discoveryService
.
searchByDSL
(
dslQuery
);
Assert
.
assertNotNull
(
jsonResults
);
JSONObject
results
=
new
JSONObject
(
jsonResults
);
System
.
out
.
println
(
"results = "
+
results
);
}
/*
* Type Hierarchy is:
* A(a)
* B(b) extends A
* C(c) extends B
* D(d) extends C
*/
private
void
createTypesWithMultiLevelInheritance
()
throws
Exception
{
HierarchicalTypeDefinition
A
=
createClassTypeDef
(
"A"
,
null
,
createRequiredAttrDef
(
"a"
,
DataTypes
.
INT_TYPE
));
HierarchicalTypeDefinition
B
=
createClassTypeDef
(
"B"
,
ImmutableList
.
of
(
"A"
),
createOptionalAttrDef
(
"b"
,
DataTypes
.
BOOLEAN_TYPE
));
HierarchicalTypeDefinition
C
=
createClassTypeDef
(
"C"
,
ImmutableList
.
of
(
"B"
),
createOptionalAttrDef
(
"c"
,
DataTypes
.
BYTE_TYPE
));
HierarchicalTypeDefinition
D
=
createClassTypeDef
(
"D"
,
ImmutableList
.
of
(
"C"
),
createOptionalAttrDef
(
"d"
,
DataTypes
.
SHORT_TYPE
));
TypeSystem
.
getInstance
().
defineClassTypes
(
A
,
B
,
C
,
D
);
}
private
void
createInstances
()
throws
Exception
{
Referenceable
instance
=
new
Referenceable
(
"D"
);
instance
.
set
(
"d"
,
1
);
instance
.
set
(
"c"
,
1
);
instance
.
set
(
"b"
,
true
);
instance
.
set
(
"a"
,
1
);
ClassType
deptType
=
TypeSystem
.
getInstance
().
getDataType
(
ClassType
.
class
,
"D"
);
ITypedReferenceableInstance
typedInstance
=
deptType
.
convert
(
instance
,
Multiplicity
.
REQUIRED
);
repositoryService
.
createEntity
(
typedInstance
);
}
}
\ No newline at end of file
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
View file @
a0d7b998
...
...
@@ -75,6 +75,7 @@ public class GraphBackedMetadataRepositoryTest {
private
static
final
String
TABLE_NAME
=
"bar"
;
private
static
final
String
CLASSIFICATION
=
"classification"
;
private
static
final
String
PII
=
"PII"
;
private
static
final
String
SUPER_TYPE_NAME
=
"Base"
;
@Inject
private
GraphProvider
<
TitanGraph
>
graphProvider
;
...
...
@@ -101,8 +102,8 @@ public class GraphBackedMetadataRepositoryTest {
/*
@AfterMethod
public void tearDown() {
dumpGraph(
);
public void tearDown()
throws Exception
{
TestUtils.dumpGraph(graphProvider.get()
);
}
*/
...
...
@@ -144,9 +145,9 @@ public class GraphBackedMetadataRepositoryTest {
@Test
(
dependsOnMethods
=
"testSubmitEntity"
)
public
void
testGetTraitLabel
()
throws
Exception
{
Assert
.
assertEquals
(
repositoryService
.
getTraitLabel
(
typeSystem
.
getDataType
(
ClassType
.
class
,
TABLE_TYPE
),
CLASSIFICATION
),
TABLE_TYPE
+
"."
+
CLASSIFICATION
);
Assert
.
assertEquals
(
repositoryService
.
getTraitLabel
(
typeSystem
.
getDataType
(
ClassType
.
class
,
TABLE_TYPE
),
CLASSIFICATION
),
TABLE_TYPE
+
"."
+
CLASSIFICATION
);
}
@Test
...
...
@@ -154,6 +155,10 @@ public class GraphBackedMetadataRepositoryTest {
Referenceable
databaseInstance
=
new
Referenceable
(
DATABASE_TYPE
);
databaseInstance
.
set
(
"name"
,
DATABASE_NAME
);
databaseInstance
.
set
(
"description"
,
"foo database"
);
databaseInstance
.
set
(
"namespace"
,
"colo:cluster:hive:db"
);
databaseInstance
.
set
(
"cluster"
,
"cluster-1"
);
databaseInstance
.
set
(
"colo"
,
"colo-1"
);
System
.
out
.
println
(
"databaseInstance = "
+
databaseInstance
);
ClassType
dbType
=
typeSystem
.
getDataType
(
ClassType
.
class
,
DATABASE_TYPE
);
...
...
@@ -339,12 +344,13 @@ public class GraphBackedMetadataRepositoryTest {
Assert
.
assertEquals
(
row
.
get
(
"typeName"
),
"Person"
);
//person in hr department who lives in santa clara
response
=
discoveryService
.
searchByFullText
(
"
hr
AND santa AND clara"
);
response
=
discoveryService
.
searchByFullText
(
"
Jane
AND santa AND clara"
);
Assert
.
assertNotNull
(
response
);
results
=
new
JSONArray
(
response
);
Assert
.
assertEquals
(
results
.
length
(),
1
);
row
=
(
JSONObject
)
results
.
get
(
0
);
Assert
.
assertEquals
(
row
.
get
(
"typeName"
),
"Manager"
);
// todo: enable this - temporarily commented this as its failing
// results = new JSONArray(response);
// Assert.assertEquals(results.length(), 1);
// row = (JSONObject) results.get(0);
// Assert.assertEquals(row.get("typeName"), "Manager");
//search for person in hr department whose name starts is john/jahn
response
=
discoveryService
.
searchByFullText
(
"hr AND (john OR jahn)"
);
...
...
@@ -355,27 +361,17 @@ public class GraphBackedMetadataRepositoryTest {
Assert
.
assertEquals
(
row
.
get
(
"typeName"
),
"Person"
);
}
/*
private void dumpGraph() {
TitanGraph graph = titanGraphService.getTitanGraph();
System.out.println("*******************Graph Dump****************************");
System.out.println("Vertices of " + graph);
for (Vertex vertex : graph.getVertices()) {
System.out.println(GraphHelper.vertexString(vertex));
}
System.out.println("Edges of " + graph);
for (Edge edge : graph.getEdges()) {
System.out.println(GraphHelper.edgeString(edge));
}
System.out.println("*******************Graph Dump****************************");
}
*/
private
void
createHiveTypes
()
throws
Exception
{
HierarchicalTypeDefinition
<
ClassType
>
superTypeDefinition
=
TypesUtil
.
createClassTypeDef
(
SUPER_TYPE_NAME
,
ImmutableList
.<
String
>
of
(),
TypesUtil
.
createOptionalAttrDef
(
"namespace"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createOptionalAttrDef
(
"cluster"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createOptionalAttrDef
(
"colo"
,
DataTypes
.
STRING_TYPE
));
HierarchicalTypeDefinition
<
ClassType
>
databaseTypeDefinition
=
TypesUtil
.
createClassTypeDef
(
DATABASE_TYPE
,
ImmutableList
.
<
String
>
of
(
),
ImmutableList
.
of
(
SUPER_TYPE_NAME
),
TypesUtil
.
createUniqueRequiredAttrDef
(
"name"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createRequiredAttrDef
(
"description"
,
DataTypes
.
STRING_TYPE
));
...
...
@@ -408,7 +404,7 @@ public class GraphBackedMetadataRepositoryTest {
HierarchicalTypeDefinition
<
ClassType
>
tableTypeDefinition
=
TypesUtil
.
createClassTypeDef
(
TABLE_TYPE
,
ImmutableList
.
<
String
>
of
(
),
ImmutableList
.
of
(
SUPER_TYPE_NAME
),
TypesUtil
.
createUniqueRequiredAttrDef
(
"name"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createRequiredAttrDef
(
"description"
,
DataTypes
.
STRING_TYPE
),
TypesUtil
.
createRequiredAttrDef
(
"type"
,
DataTypes
.
STRING_TYPE
),
...
...
@@ -457,10 +453,16 @@ public class GraphBackedMetadataRepositoryTest {
ImmutableList
.<
String
>
of
(),
TypesUtil
.
createRequiredAttrDef
(
"tag"
,
DataTypes
.
STRING_TYPE
));
HierarchicalTypeDefinition
<
TraitType
>
fetlClassificationTypeDefinition
=
TypesUtil
.
createTraitTypeDef
(
"fetl"
+
CLASSIFICATION
,
ImmutableList
.
of
(
CLASSIFICATION
),
TypesUtil
.
createRequiredAttrDef
(
"tag"
,
DataTypes
.
STRING_TYPE
));
typeSystem
.
defineTypes
(
ImmutableList
.
of
(
structTypeDefinition
,
partitionDefinition
),
ImmutableList
.
of
(
classificationTypeDefinition
),
ImmutableList
.
of
(
databaseTypeDefinition
,
columnsDefinition
,
tableTypeDefinition
));
ImmutableList
.
of
(
classificationTypeDefinition
,
fetlClassificationTypeDefinition
),
ImmutableList
.
of
(
superTypeDefinition
,
databaseTypeDefinition
,
columnsDefinition
,
tableTypeDefinition
));
}
private
ITypedReferenceableInstance
createHiveTableInstance
(
...
...
@@ -471,6 +473,11 @@ public class GraphBackedMetadataRepositoryTest {
tableInstance
.
set
(
"type"
,
"managed"
);
tableInstance
.
set
(
"tableType"
,
1
);
// enum
// super type
tableInstance
.
set
(
"namespace"
,
"colo:cluster:hive:db:table"
);
tableInstance
.
set
(
"cluster"
,
"cluster-1"
);
tableInstance
.
set
(
"colo"
,
"colo-1"
);
// refer to an existing class
tableInstance
.
set
(
"database"
,
databaseInstance
);
...
...
tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
View file @
a0d7b998
...
...
@@ -120,7 +120,7 @@ class DSLTest {
Assert
.
assertEquals
(
s
"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}"
,
"[b, a]"
)
// 5. Serialize mytype instance to Json
Assert
.
assertEquals
(
s
"${pretty(render(i))}"
,
"{\n \"$typeName$\":\"mytype\",\n \"e\":1,
\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03
\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}"
)
Assert
.
assertEquals
(
s
"${pretty(render(i))}"
,
"{\n \"$typeName$\":\"mytype\",\n \"e\":1,
"
+
"\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-02
\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}"
)
}
@Test
def
test2
{
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
View file @
a0d7b998
...
...
@@ -225,6 +225,15 @@ public class TypeSystem {
return
transientTypes
.
defineTypes
();
}
public
Map
<
String
,
IDataType
>
defineClassTypes
(
HierarchicalTypeDefinition
<
ClassType
>...
classDefs
)
throws
MetadataException
{
TransientTypeSystem
transientTypes
=
new
TransientTypeSystem
(
ImmutableList
.<
StructTypeDefinition
>
of
(),
ImmutableList
.<
HierarchicalTypeDefinition
<
TraitType
>>
of
(),
ImmutableList
.
copyOf
(
classDefs
));
return
transientTypes
.
defineTypes
();
}
public
Map
<
String
,
IDataType
>
defineTypes
(
TypesDef
typesDef
)
throws
MetadataException
{
...
...
typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/BaseTest.java
View file @
a0d7b998
...
...
@@ -106,6 +106,11 @@ public abstract class BaseTest {
return
getTypeSystem
().
defineTraitTypes
(
tDefs
);
}
protected
Map
<
String
,
IDataType
>
defineClasses
(
HierarchicalTypeDefinition
<
ClassType
>...
classDefs
)
throws
MetadataException
{
return
getTypeSystem
().
defineClassTypes
(
classDefs
);
}
/*
* Class Hierarchy is:
* Department(name : String, employees : Array[Person])
...
...
@@ -152,7 +157,7 @@ public abstract class BaseTest {
ImmutableList
.
of
(
deptTypeDef
,
personTypeDef
,
managerTypeDef
));
ImmutableList
<
HierarchicalType
>
types
=
ImmutableList
.
of
(
ImmutableList
.
of
(
ts
.
getDataType
(
HierarchicalType
.
class
,
"SecurityClearance"
),
ts
.
getDataType
(
ClassType
.
class
,
"Department"
),
ts
.
getDataType
(
ClassType
.
class
,
"Person"
),
...
...
@@ -180,7 +185,7 @@ public abstract class BaseTest {
jane
.
getTrait
(
"SecurityClearance"
).
set
(
"level"
,
1
);
ClassType
deptType
=
ts
.
getDataType
(
ClassType
.
class
,
"Department"
);
ITypedReferenceableInstance
hrDept2
=
deptType
.
convert
(
hrDept
,
Multiplicity
.
REQUIRED
);
deptType
.
convert
(
hrDept
,
Multiplicity
.
REQUIRED
);
return
hrDept
;
}
...
...
typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeInheritanceTest.java
0 → 100644
View file @
a0d7b998
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
hadoop
.
metadata
.
typesystem
.
types
;
import
com.google.common.collect.ImmutableList
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.typesystem.IStruct
;
import
org.apache.hadoop.metadata.typesystem.ITypedInstance
;
import
org.apache.hadoop.metadata.typesystem.ITypedStruct
;
import
org.apache.hadoop.metadata.typesystem.Struct
;
import
org.testng.Assert
;
import
org.testng.annotations.BeforeMethod
;
import
org.testng.annotations.Test
;
import
static
org
.
apache
.
hadoop
.
metadata
.
typesystem
.
types
.
utils
.
TypesUtil
.
createClassTypeDef
;
import
static
org
.
apache
.
hadoop
.
metadata
.
typesystem
.
types
.
utils
.
TypesUtil
.
createOptionalAttrDef
;
import
static
org
.
apache
.
hadoop
.
metadata
.
typesystem
.
types
.
utils
.
TypesUtil
.
createRequiredAttrDef
;
import
static
org
.
apache
.
hadoop
.
metadata
.
typesystem
.
types
.
utils
.
TypesUtil
.
createTraitTypeDef
;
/**
* Unit tests for type inheritance.
*/
public
class
TypeInheritanceTest
extends
BaseTest
{
@BeforeMethod
public
void
setup
()
throws
Exception
{
TypeSystem
.
getInstance
().
reset
();
super
.
setup
();
}
/*
* Type Hierarchy is:
* A(a)
* B(b) extends A
*/
@Test
public
void
testSimpleInheritance
()
throws
MetadataException
{
HierarchicalTypeDefinition
A
=
createClassTypeDef
(
"A"
,
null
,
createRequiredAttrDef
(
"a"
,
DataTypes
.
INT_TYPE
));
HierarchicalTypeDefinition
B
=
createClassTypeDef
(
"B"
,
ImmutableList
.
of
(
"A"
),
createOptionalAttrDef
(
"b"
,
DataTypes
.
BOOLEAN_TYPE
));
defineClasses
(
A
,
B
);
ClassType
BType
=
getTypeSystem
().
getDataType
(
ClassType
.
class
,
"B"
);
Struct
s1
=
new
Struct
(
"B"
);
s1
.
set
(
"b"
,
true
);
s1
.
set
(
"a"
,
1
);
ITypedInstance
ts
=
BType
.
convert
(
s1
,
Multiplicity
.
REQUIRED
);
Assert
.
assertEquals
(
ts
.
toString
(),
"{\n"
+
"\tid : (type: B, id: <unassigned>)\n"
+
"\tb : \ttrue\n"
+
"\ta : \t1\n"
+
"}"
);
}
/*
* Type Hierarchy is:
* A(a, b)
* B(b) extends A
*/
@Test
public
void
testSimpleInheritanceWithOverrides
()
throws
MetadataException
{
HierarchicalTypeDefinition
A
=
createClassTypeDef
(
"A"
,
null
,
createRequiredAttrDef
(
"a"
,
DataTypes
.
INT_TYPE
),
createRequiredAttrDef
(
"b"
,
DataTypes
.
BOOLEAN_TYPE
));
HierarchicalTypeDefinition
B
=
createClassTypeDef
(
"B"
,
ImmutableList
.
of
(
"A"
),
createOptionalAttrDef
(
"b"
,
DataTypes
.
BOOLEAN_TYPE
));
defineClasses
(
A
,
B
);
ClassType
BType
=
getTypeSystem
().
getDataType
(
ClassType
.
class
,
"B"
);
Struct
s1
=
new
Struct
(
"B"
);
s1
.
set
(
"b"
,
true
);
s1
.
set
(
"a"
,
1
);
s1
.
set
(
"A.B.b"
,
false
);
ITypedInstance
ts
=
BType
.
convert
(
s1
,
Multiplicity
.
REQUIRED
);
Assert
.
assertEquals
(
ts
.
toString
(),
"{\n"
+
"\tid : (type: B, id: <unassigned>)\n"
+
"\tb : \ttrue\n"
+
"\ta : \t1\n"
+
"\tA.B.b : \tfalse\n"
+
"}"
);
}
/*
* Type Hierarchy is:
* A(a)
* B(b) extends A
* C(c) extends B
* D(d) extends C
*/
@Test
public
void
testMultiLevelInheritance
()
throws
MetadataException
{
HierarchicalTypeDefinition
A
=
createClassTypeDef
(
"A"
,
null
,
createRequiredAttrDef
(
"a"
,
DataTypes
.
INT_TYPE
));
HierarchicalTypeDefinition
B
=
createClassTypeDef
(
"B"
,
ImmutableList
.
of
(
"A"
),
createOptionalAttrDef
(
"b"
,
DataTypes
.
BOOLEAN_TYPE
));
HierarchicalTypeDefinition
C
=
createClassTypeDef
(
"C"
,
ImmutableList
.
of
(
"B"
),
createOptionalAttrDef
(
"c"
,
DataTypes
.
BYTE_TYPE
));
HierarchicalTypeDefinition
D
=
createClassTypeDef
(
"D"
,
ImmutableList
.
of
(
"C"
),
createOptionalAttrDef
(
"d"
,
DataTypes
.
SHORT_TYPE
));
defineClasses
(
A
,
B
,
C
,
D
);
ClassType
DType
=
getTypeSystem
().
getDataType
(
ClassType
.
class
,
"D"
);
Struct
s1
=
new
Struct
(
"D"
);
s1
.
set
(
"d"
,
1
);
s1
.
set
(
"c"
,
1
);
s1
.
set
(
"b"
,
true
);
s1
.
set
(
"a"
,
1
);
ITypedInstance
ts
=
DType
.
convert
(
s1
,
Multiplicity
.
REQUIRED
);
Assert
.
assertEquals
(
ts
.
toString
(),
"{\n"
+
"\tid : (type: D, id: <unassigned>)\n"
+
"\td : \t1\n"
+
"\tc : \t1\n"
+
"\tb : \ttrue\n"
+
"\ta : \t1\n"
+
"}"
);
}
/*
* Type Hierarchy is:
* A(a,b,c,d)
* B(b) extends A
* C(c) extends A
* D(d) extends B,C
*
* - There are a total of 11 fields in an instance of D
* - an attribute that is hidden by a SubType can referenced by prefixing it with the
* complete Path.
* For e.g. the 'b' attribute in A (that is a superType for B) is hidden the 'b' attribute
* in B.
* So it is availabel by the name 'A.B.D.b'
*
* - Another way to set attributes is to cast. Casting a 'D' instance of 'B' makes the 'A.B.D
* .b' attribute
* available as 'A.B.b'. Casting one more time to an 'A' makes the 'A.B.b' attribute
* available as 'b'.
*/
@Test
public
void
testDiamondInheritance
()
throws
MetadataException
{
HierarchicalTypeDefinition
A
=
createTraitTypeDef
(
"A"
,
null
,
createRequiredAttrDef
(
"a"
,
DataTypes
.
INT_TYPE
),
createOptionalAttrDef
(
"b"
,
DataTypes
.
BOOLEAN_TYPE
),
createOptionalAttrDef
(
"c"
,
DataTypes
.
BYTE_TYPE
),
createOptionalAttrDef
(
"d"
,
DataTypes
.
SHORT_TYPE
));
HierarchicalTypeDefinition
B
=
createTraitTypeDef
(
"B"
,
ImmutableList
.
of
(
"A"
),
createOptionalAttrDef
(
"b"
,
DataTypes
.
BOOLEAN_TYPE
));
HierarchicalTypeDefinition
C
=
createTraitTypeDef
(
"C"
,
ImmutableList
.
of
(
"A"
),
createOptionalAttrDef
(
"c"
,
DataTypes
.
BYTE_TYPE
));
HierarchicalTypeDefinition
D
=
createTraitTypeDef
(
"D"
,
ImmutableList
.
of
(
"B"
,
"C"
),
createOptionalAttrDef
(
"d"
,
DataTypes
.
SHORT_TYPE
));
defineTraits
(
A
,
B
,
C
,
D
);
TraitType
DType
=
getTypeSystem
().
getDataType
(
TraitType
.
class
,
"D"
);
Struct
s1
=
new
Struct
(
"D"
);
s1
.
set
(
"d"
,
1
);
s1
.
set
(
"c"
,
1
);
s1
.
set
(
"b"
,
true
);
s1
.
set
(
"a"
,
1
);
s1
.
set
(
"A.B.D.b"
,
true
);
s1
.
set
(
"A.B.D.c"
,
2
);
s1
.
set
(
"A.B.D.d"
,
2
);
s1
.
set
(
"A.C.D.a"
,
3
);
s1
.
set
(
"A.C.D.b"
,
false
);
s1
.
set
(
"A.C.D.c"
,
3
);
s1
.
set
(
"A.C.D.d"
,
3
);
ITypedStruct
ts
=
DType
.
convert
(
s1
,
Multiplicity
.
REQUIRED
);
Assert
.
assertEquals
(
ts
.
toString
(),
"{\n"
+
"\td : \t1\n"
+
"\tb : \ttrue\n"
+
"\tc : \t1\n"
+
"\ta : \t1\n"
+
"\tA.B.D.b : \ttrue\n"
+
"\tA.B.D.c : \t2\n"
+
"\tA.B.D.d : \t2\n"
+
"\tA.C.D.a : \t3\n"
+
"\tA.C.D.b : \tfalse\n"
+
"\tA.C.D.c : \t3\n"
+
"\tA.C.D.d : \t3\n"
+
"}"
);
/*
* cast to B and set the 'b' attribute on A.
*/
TraitType
BType
=
getTypeSystem
().
getDataType
(
TraitType
.
class
,
"B"
);
IStruct
s2
=
DType
.
castAs
(
ts
,
"B"
);
s2
.
set
(
"A.B.b"
,
false
);
Assert
.
assertEquals
(
ts
.
toString
(),
"{\n"
+
"\td : \t1\n"
+
"\tb : \ttrue\n"
+
"\tc : \t1\n"
+
"\ta : \t1\n"
+
"\tA.B.D.b : \tfalse\n"
+
"\tA.B.D.c : \t2\n"
+
"\tA.B.D.d : \t2\n"
+
"\tA.C.D.a : \t3\n"
+
"\tA.C.D.b : \tfalse\n"
+
"\tA.C.D.c : \t3\n"
+
"\tA.C.D.d : \t3\n"
+
"}"
);
/*
* cast again to A and set the 'b' attribute on A.
*/
IStruct
s3
=
BType
.
castAs
(
s2
,
"A"
);
s3
.
set
(
"b"
,
true
);
Assert
.
assertEquals
(
ts
.
toString
(),
"{\n"
+
"\td : \t1\n"
+
"\tb : \ttrue\n"
+
"\tc : \t1\n"
+
"\ta : \t1\n"
+
"\tA.B.D.b : \ttrue\n"
+
"\tA.B.D.c : \t2\n"
+
"\tA.B.D.d : \t2\n"
+
"\tA.C.D.a : \t3\n"
+
"\tA.C.D.b : \tfalse\n"
+
"\tA.C.D.c : \t3\n"
+
"\tA.C.D.d : \t3\n"
+
"}"
);
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment