Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
02cf8c48
Commit
02cf8c48
authored
8 years ago
by
Jeff Hagelberg
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-1510: Consolidate/batch calls to GraphBackedTypeStore.findVertex()
parent
09089e09
master
No related merge requests found
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
524 additions
and
54 deletions
+524
-54
release-log.txt
release-log.txt
+1
-0
GraphHelper.java
...n/java/org/apache/atlas/repository/graph/GraphHelper.java
+1
-1
GraphBackedTypeStore.java
...ache/atlas/repository/typestore/GraphBackedTypeStore.java
+105
-51
TypePersistenceVisitor.java
...he/atlas/repository/typestore/TypePersistenceVisitor.java
+117
-0
TypeVertexFinder.java
...g/apache/atlas/repository/typestore/TypeVertexFinder.java
+104
-0
TypeVertexInfo.java
...org/apache/atlas/repository/typestore/TypeVertexInfo.java
+95
-0
TypeVisitor.java
...va/org/apache/atlas/repository/typestore/TypeVisitor.java
+97
-0
GraphBackedTypeStoreTest.java
.../atlas/repository/typestore/GraphBackedTypeStoreTest.java
+4
-2
No files found.
release-log.txt
View file @
02cf8c48
...
...
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelb)
ATLAS-1388 Cache entities that are created/updated (jnhagelb)
ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb)
ATLAS-1517: updated hive_model to include schema related attributes (sarath.kum4r@gmail.com via mneethiraj)
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
View file @
02cf8c48
...
...
@@ -538,7 +538,7 @@ public final class GraphHelper {
*
* @return propertyValue to AtlasVertex map with the result.
*/
p
rivate
Map
<
String
,
AtlasVertex
>
getVerticesForPropertyValues
(
String
property
,
List
<
String
>
values
)
p
ublic
Map
<
String
,
AtlasVertex
>
getVerticesForPropertyValues
(
String
property
,
List
<
String
>
values
)
throws
RepositoryException
{
if
(
values
.
isEmpty
())
{
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
100755 → 100644
View file @
02cf8c48
...
...
@@ -21,14 +21,17 @@ package org.apache.atlas.repository.typestore;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
setProperty
;
import
java.util.ArrayList
;
import
java.util.HashMap
;
import
java.util.HashSet
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.GraphTransaction
;
import
org.apache.atlas.repository.Constants
;
import
org.apache.atlas.repository.RepositoryException
;
import
org.apache.atlas.repository.graph.AtlasGraphProvider
;
import
org.apache.atlas.repository.graph.GraphHelper
;
import
org.apache.atlas.repository.graphdb.AtlasEdge
;
...
...
@@ -57,8 +60,10 @@ import org.codehaus.jettison.json.JSONException;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.google.common.base.Function
;
import
com.google.common.collect.ImmutableList
;
import
com.google.common.collect.ImmutableSet
;
import
com.google.common.collect.Lists
;
import
com.google.inject.Inject
;
import
com.google.inject.Singleton
;
...
...
@@ -82,25 +87,37 @@ public class GraphBackedTypeStore implements ITypeStore {
@Override
@GraphTransaction
public
void
store
(
TypeSystem
typeSystem
,
ImmutableList
<
String
>
typeNames
)
throws
AtlasException
{
//Pre-create the vertices that are needed for the types. This allows us to execute
//one query to determine all of the vertices that already exist.
Map
<
String
,
AtlasVertex
>
typeVertices
=
getOrCreateTypeVertices
(
typeSystem
,
typeNames
);
//Complete the storage process by adding properties and edges to the vertices
//that were created.
TypePersistenceVisitor
visitor
=
new
TypePersistenceVisitor
(
this
,
typeVertices
,
typeSystem
);
processTypes
(
typeNames
,
typeSystem
,
visitor
);
}
private
void
processTypes
(
ImmutableList
<
String
>
typeNames
,
TypeSystem
typeSystem
,
TypeVisitor
visitor
)
throws
AtlasException
{
for
(
String
typeName
:
typeNames
)
{
IDataType
dataType
=
typeSystem
.
getDataType
(
IDataType
.
class
,
typeName
);
LOG
.
debug
(
"Processing {}.{}.{} in type store"
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
dataType
.
getDescription
());
switch
(
dataType
.
getTypeCategory
())
{
case
ENUM:
storeInGraph
((
EnumType
)
dataType
);
visitor
.
visitEnumeration
((
EnumType
)
dataType
);
break
;
case
STRUCT:
StructType
structType
=
(
StructType
)
dataType
;
storeInGraph
(
typeSystem
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
dataType
.
getDescription
(),
ImmutableList
.
copyOf
(
structType
.
infoToNameMap
.
keySet
()),
ImmutableSet
.<
String
>
of
());
processType
(
typeSystem
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
dataType
.
getDescription
(),
ImmutableList
.
copyOf
(
structType
.
infoToNameMap
.
keySet
()),
ImmutableSet
.<
String
>
of
()
,
visitor
);
break
;
case
TRAIT:
case
CLASS:
HierarchicalType
type
=
(
HierarchicalType
)
dataType
;
storeInGraph
(
typeSystem
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
type
.
getDescription
(),
type
.
immediateAttrs
,
type
.
superTypes
);
processType
(
typeSystem
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
type
.
getDescription
(),
type
.
immediateAttrs
,
type
.
superTypes
,
visitor
);
break
;
default
:
//Ignore primitive/collection types as they are covered under references
...
...
@@ -109,63 +126,70 @@ public class GraphBackedTypeStore implements ITypeStore {
}
}
private
void
storeInGraph
(
EnumType
dataType
)
{
AtlasVertex
AtlasVertex
=
createVertex
(
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
dataType
.
getDescription
());
List
<
String
>
values
=
new
ArrayList
<>(
dataType
.
values
().
size
());
for
(
EnumValue
enumValue
:
dataType
.
values
())
{
String
key
=
getPropertyKey
(
dataType
.
getName
(),
enumValue
.
value
);
setProperty
(
AtlasVertex
,
key
,
enumValue
.
ordinal
);
values
.
add
(
enumValue
.
value
);
private
Map
<
String
,
AtlasVertex
>
getOrCreateTypeVertices
(
TypeSystem
typeSystem
,
ImmutableList
<
String
>
typeNames
)
throws
AtlasException
{
//examine the types to determine what type vertices are needed
TypeVertexFinder
vertexFinder
=
new
TypeVertexFinder
(
typeSystem
);
processTypes
(
typeNames
,
typeSystem
,
vertexFinder
);
List
<
TypeVertexInfo
>
typeVerticesNeeded
=
vertexFinder
.
getVerticesToCreate
();
//find or create the type vertices
List
<
AtlasVertex
>
vertices
=
createVertices
(
typeVerticesNeeded
);
//Create a type name->AtlasVertex map with the result
Map
<
String
,
AtlasVertex
>
result
=
new
HashMap
<
String
,
AtlasVertex
>(
typeVerticesNeeded
.
size
());
for
(
int
i
=
0
;
i
<
typeVerticesNeeded
.
size
();
i
++)
{
TypeVertexInfo
createdVertexInfo
=
typeVerticesNeeded
.
get
(
i
);
AtlasVertex
createdVertex
=
vertices
.
get
(
i
);
result
.
put
(
createdVertexInfo
.
getTypeName
(),
createdVertex
);
}
setProperty
(
AtlasVertex
,
getPropertyKey
(
dataType
.
getName
()),
values
);
return
result
;
}
private
String
getPropertyKey
(
String
name
)
{
static
String
getPropertyKey
(
String
name
)
{
return
PROPERTY_PREFIX
+
name
;
}
private
String
getPropertyKey
(
String
parent
,
String
child
)
{
static
String
getPropertyKey
(
String
parent
,
String
child
)
{
return
PROPERTY_PREFIX
+
parent
+
"."
+
child
;
}
String
getEdgeLabel
(
String
parent
,
String
child
)
{
static
String
getEdgeLabel
(
String
parent
,
String
child
)
{
return
PROPERTY_PREFIX
+
"edge."
+
parent
+
"."
+
child
;
}
private
void
storeInGraph
(
TypeSystem
typeSystem
,
DataTypes
.
TypeCategory
category
,
String
typeName
,
String
typeDescription
,
ImmutableList
<
AttributeInfo
>
attributes
,
ImmutableSet
<
String
>
superTypes
)
throws
AtlasException
{
AtlasVertex
vertex
=
createVertex
(
category
,
typeName
,
typeDescription
);
private
void
processType
(
TypeSystem
typeSystem
,
DataTypes
.
TypeCategory
category
,
String
typeName
,
String
typeDescription
,
ImmutableList
<
AttributeInfo
>
attributes
,
ImmutableSet
<
String
>
superTypes
,
TypeVisitor
visitor
)
throws
AtlasException
{
visitor
.
visitDataType
(
category
,
typeName
,
typeDescription
);
List
<
String
>
attrNames
=
new
ArrayList
<>();
if
(
attributes
!=
null
)
{
for
(
AttributeInfo
attribute
:
attributes
)
{
String
propertyKey
=
getPropertyKey
(
typeName
,
attribute
.
name
);
try
{
setProperty
(
vertex
,
propertyKey
,
attribute
.
toJson
());
}
catch
(
JSONException
e
)
{
throw
new
StorageException
(
typeName
,
e
);
}
visitor
.
visitAttribute
(
typeName
,
attribute
);
attrNames
.
add
(
attribute
.
name
);
addReferencesForAttribute
(
typeSystem
,
vertex
,
attribute
);
processsAttribute
(
typeSystem
,
typeName
,
attribute
,
visitor
);
}
}
setProperty
(
vertex
,
getPropertyKey
(
typeName
)
,
attrNames
);
visitor
.
visitAttributeNames
(
typeName
,
attrNames
);
//Add edges for hierarchy
if
(
superTypes
!=
null
)
{
for
(
String
superTypeName
:
superTypes
)
{
HierarchicalType
superType
=
typeSystem
.
getDataType
(
HierarchicalType
.
class
,
superTypeName
);
AtlasVertex
superVertex
=
createVertex
(
superType
.
getTypeCategory
(),
superTypeName
,
superType
.
getDescription
());
graphHelper
.
getOrCreateEdge
(
vertex
,
superVertex
,
SUPERTYPE_EDGE_LABEL
);
visitor
.
visitSuperType
(
typeName
,
superTypeName
);
}
}
}
private
void
addReferencesForAttribute
(
TypeSystem
typeSystem
,
AtlasVertex
vertex
,
AttributeInfo
attribute
)
private
void
processsAttribute
(
TypeSystem
typeSystem
,
String
typeName
,
AttributeInfo
attribute
,
TypeVisitor
visitor
)
throws
AtlasException
{
ImmutableList
<
String
>
coreTypes
=
typeSystem
.
getCoreTypes
();
List
<
IDataType
>
attrDataTypes
=
new
ArrayList
<>();
IDataType
attrDataType
=
attribute
.
dataType
();
String
vertexTypeName
=
GraphHelper
.
getSingleValuedProperty
(
vertex
,
Constants
.
TYPENAME_PROPERTY_KEY
,
String
.
class
);
switch
(
attrDataType
.
getTypeCategory
())
{
case
ARRAY:
...
...
@@ -200,11 +224,10 @@ public class GraphBackedTypeStore implements ITypeStore {
"Attribute cannot reference instances of type : "
+
attrDataType
.
getTypeCategory
());
}
for
(
IDataType
attrType
:
attrDataTypes
)
{
if
(!
coreTypes
.
contains
(
attrType
.
getName
()))
{
AtlasVertex
attrVertex
=
createVertex
(
attrType
.
getTypeCategory
(),
attrType
.
getName
(),
attrType
.
getDescription
());
String
label
=
getEdgeLabel
(
vertexTypeName
,
attribute
.
name
);
graphHelper
.
getOrCreateEdge
(
vertex
,
attrVertex
,
label
);
visitor
.
visitAttributeDataType
(
typeName
,
attribute
,
attrType
);
}
}
}
...
...
@@ -328,23 +351,54 @@ public class GraphBackedTypeStore implements ITypeStore {
return
vertex
;
}
private
AtlasVertex
createVertex
(
DataTypes
.
TypeCategory
category
,
String
typeName
,
String
typeDescription
)
{
AtlasVertex
vertex
=
findVertex
(
category
,
typeName
);
if
(
vertex
==
null
)
{
LOG
.
debug
(
"Adding vertex {}{}"
,
PROPERTY_PREFIX
,
typeName
);
vertex
=
graph
.
addVertex
();
setProperty
(
vertex
,
Constants
.
VERTEX_TYPE_PROPERTY_KEY
,
VERTEX_TYPE
);
// Mark as type AtlasVertex
setProperty
(
vertex
,
Constants
.
TYPE_CATEGORY_PROPERTY_KEY
,
category
);
setProperty
(
vertex
,
Constants
.
TYPENAME_PROPERTY_KEY
,
typeName
);
}
if
(
typeDescription
!=
null
)
{
String
oldDescription
=
getPropertyKey
(
Constants
.
TYPEDESCRIPTION_PROPERTY_KEY
);
if
(!
typeDescription
.
equals
(
oldDescription
))
{
setProperty
(
vertex
,
Constants
.
TYPEDESCRIPTION_PROPERTY_KEY
,
typeDescription
);
//package-private for testing
Map
<
String
,
AtlasVertex
>
findVertices
(
List
<
String
>
typeNames
)
throws
RepositoryException
{
LOG
.
debug
(
"Finding vertices for {}"
,
typeNames
.
toString
());
Map
<
String
,
AtlasVertex
>
foundVertices
=
graphHelper
.
getVerticesForPropertyValues
(
Constants
.
TYPENAME_PROPERTY_KEY
,
typeNames
);
return
foundVertices
;
}
/**
* Finds or creates type vertices with the information specified.
*
* @param infoList
* @return list with the vertices corresponding to the types in the list.
* @throws AtlasException
*/
private
List
<
AtlasVertex
>
createVertices
(
List
<
TypeVertexInfo
>
infoList
)
throws
AtlasException
{
List
<
AtlasVertex
>
result
=
new
ArrayList
<>(
infoList
.
size
());
List
<
String
>
typeNames
=
Lists
.
transform
(
infoList
,
new
Function
<
TypeVertexInfo
,
String
>()
{
@Override
public
String
apply
(
TypeVertexInfo
input
)
{
return
input
.
getTypeName
();
}
});
Map
<
String
,
AtlasVertex
>
vertices
=
findVertices
(
typeNames
);
for
(
TypeVertexInfo
info
:
infoList
)
{
AtlasVertex
vertex
=
vertices
.
get
(
info
.
getTypeName
());
if
(!
GraphHelper
.
elementExists
(
vertex
))
{
LOG
.
debug
(
"Adding vertex {}{}"
,
PROPERTY_PREFIX
,
info
.
getTypeName
());
vertex
=
graph
.
addVertex
();
setProperty
(
vertex
,
Constants
.
VERTEX_TYPE_PROPERTY_KEY
,
VERTEX_TYPE
);
// Mark as type AtlasVertex
setProperty
(
vertex
,
Constants
.
TYPE_CATEGORY_PROPERTY_KEY
,
info
.
getCategory
());
setProperty
(
vertex
,
Constants
.
TYPENAME_PROPERTY_KEY
,
info
.
getTypeName
());
}
}
else
{
LOG
.
debug
(
" type description is null "
);
String
newDescription
=
info
.
getTypeDescription
();
if
(
newDescription
!=
null
)
{
String
oldDescription
=
getPropertyKey
(
Constants
.
TYPEDESCRIPTION_PROPERTY_KEY
);
if
(!
newDescription
.
equals
(
oldDescription
))
{
setProperty
(
vertex
,
Constants
.
TYPEDESCRIPTION_PROPERTY_KEY
,
newDescription
);
}
}
else
{
LOG
.
debug
(
" type description is null "
);
}
result
.
add
(
vertex
);
}
return
vertex
;
return
result
;
}
}
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java
0 → 100644
View file @
02cf8c48
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
repository
.
typestore
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
setProperty
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.Map
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.repository.Constants
;
import
org.apache.atlas.repository.RepositoryException
;
import
org.apache.atlas.repository.graph.GraphHelper
;
import
org.apache.atlas.repository.graphdb.AtlasVertex
;
import
org.apache.atlas.typesystem.types.AttributeInfo
;
import
org.apache.atlas.typesystem.types.DataTypes.TypeCategory
;
import
org.apache.atlas.typesystem.types.EnumType
;
import
org.apache.atlas.typesystem.types.EnumValue
;
import
org.apache.atlas.typesystem.types.HierarchicalType
;
import
org.apache.atlas.typesystem.types.IDataType
;
import
org.apache.atlas.typesystem.types.TypeSystem
;
import
org.codehaus.jettison.json.JSONException
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
/**
* TypeVisitor implementation that completes the type storage process by
* adding the required properties and edges to the type vertices
* that were created.
*/
public
class
TypePersistenceVisitor
implements
TypeVisitor
{
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
TypePersistenceVisitor
.
class
);
private
static
final
GraphHelper
graphHelper
=
GraphHelper
.
getInstance
();
private
final
GraphBackedTypeStore
typeStore_
;
private
final
Map
<
String
,
AtlasVertex
>
typeVertices
;
private
final
TypeSystem
typeSystem
;
/**
* @param graphBackedTypeStore
*/
public
TypePersistenceVisitor
(
GraphBackedTypeStore
graphBackedTypeStore
,
Map
<
String
,
AtlasVertex
>
typeVertices
,
TypeSystem
typeSystem
)
{
typeStore_
=
graphBackedTypeStore
;
this
.
typeVertices
=
typeVertices
;
this
.
typeSystem
=
typeSystem
;
}
@Override
public
void
visitEnumeration
(
EnumType
dataType
)
throws
AtlasException
{
AtlasVertex
vertex
=
typeVertices
.
get
(
dataType
.
getName
());
List
<
String
>
values
=
new
ArrayList
<>(
dataType
.
values
().
size
());
for
(
EnumValue
enumValue
:
dataType
.
values
())
{
String
key
=
GraphBackedTypeStore
.
getPropertyKey
(
dataType
.
getName
(),
enumValue
.
value
);
setProperty
(
vertex
,
key
,
enumValue
.
ordinal
);
values
.
add
(
enumValue
.
value
);
}
setProperty
(
vertex
,
GraphBackedTypeStore
.
getPropertyKey
(
dataType
.
getName
()),
values
);
}
@Override
public
void
visitAttributeDataType
(
String
typeName
,
AttributeInfo
attribute
,
IDataType
attrType
)
throws
AtlasException
{
AtlasVertex
vertex
=
typeVertices
.
get
(
typeName
);
String
vertexTypeName
=
GraphHelper
.
getSingleValuedProperty
(
vertex
,
Constants
.
TYPENAME_PROPERTY_KEY
,
String
.
class
);
AtlasVertex
attrVertex
=
typeVertices
.
get
(
attrType
.
getName
());
String
label
=
GraphBackedTypeStore
.
getEdgeLabel
(
vertexTypeName
,
attribute
.
name
);
graphHelper
.
getOrCreateEdge
(
vertex
,
attrVertex
,
label
);
}
@Override
public
void
visitSuperType
(
String
typeName
,
String
superTypeName
)
throws
AtlasException
{
AtlasVertex
vertex
=
typeVertices
.
get
(
typeName
);
HierarchicalType
superType
=
typeSystem
.
getDataType
(
HierarchicalType
.
class
,
superTypeName
);
AtlasVertex
superVertex
=
typeVertices
.
get
(
superTypeName
);
graphHelper
.
getOrCreateEdge
(
vertex
,
superVertex
,
GraphBackedTypeStore
.
SUPERTYPE_EDGE_LABEL
);
}
@Override
public
void
visitAttributeNames
(
String
typeName
,
List
<
String
>
attrNames
)
throws
AtlasException
{
AtlasVertex
vertex
=
typeVertices
.
get
(
typeName
);
setProperty
(
vertex
,
GraphBackedTypeStore
.
getPropertyKey
(
typeName
),
attrNames
);
}
@Override
public
void
visitAttribute
(
String
typeName
,
AttributeInfo
attribute
)
throws
AtlasException
{
AtlasVertex
vertex
=
typeVertices
.
get
(
typeName
);
String
propertyKey
=
GraphBackedTypeStore
.
getPropertyKey
(
typeName
,
attribute
.
name
);
try
{
setProperty
(
vertex
,
propertyKey
,
attribute
.
toJson
());
}
catch
(
JSONException
e
)
{
throw
new
StorageException
(
typeName
,
e
);
}
}
@Override
public
void
visitDataType
(
TypeCategory
category
,
String
typeName
,
String
typeDescription
)
{
//nothing to do
}
}
\ No newline at end of file
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java
0 → 100644
View file @
02cf8c48
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
repository
.
typestore
;
import
java.util.ArrayList
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.Set
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.typesystem.types.AttributeInfo
;
import
org.apache.atlas.typesystem.types.DataTypes.TypeCategory
;
import
org.apache.atlas.typesystem.types.EnumType
;
import
org.apache.atlas.typesystem.types.HierarchicalType
;
import
org.apache.atlas.typesystem.types.IDataType
;
import
org.apache.atlas.typesystem.types.TypeSystem
;
/**
* TypeVisitor implementation that builds up a list of type vertices
* that need to be created for the types that are being stored.
*
*/
public
class
TypeVertexFinder
implements
TypeVisitor
{
private
final
List
<
TypeVertexInfo
>
toCreate
=
new
ArrayList
<
TypeVertexInfo
>();
private
final
Set
<
String
>
typesIncluded
=
new
HashSet
<
String
>();
private
final
TypeSystem
typeSystem
;
public
TypeVertexFinder
(
TypeSystem
ts
)
{
typeSystem
=
ts
;
}
@Override
public
void
visitEnumeration
(
EnumType
dataType
)
{
visitDataType
(
dataType
);
}
private
void
addTypeIfNeeded
(
TypeVertexInfo
info
)
{
if
(!
typesIncluded
.
contains
(
info
.
getTypeName
()))
{
toCreate
.
add
(
info
);
typesIncluded
.
add
(
info
.
getTypeName
());
}
}
@Override
public
void
visitAttributeDataType
(
String
typeName
,
AttributeInfo
sourceAttr
,
IDataType
attrType
)
throws
AtlasException
{
visitDataType
(
attrType
);
}
@Override
public
void
visitSuperType
(
String
typeName
,
String
superTypeName
)
throws
AtlasException
{
HierarchicalType
superType
=
typeSystem
.
getDataType
(
HierarchicalType
.
class
,
superTypeName
);
visitDataType
(
superType
);
}
@Override
public
void
visitAttributeNames
(
String
typeName
,
List
<
String
>
attrNames
)
throws
AtlasException
{
//nothing to do
}
@Override
public
void
visitAttribute
(
String
typeName
,
AttributeInfo
attribute
)
throws
StorageException
,
AtlasException
{
//nothing to do
}
private
void
visitDataType
(
IDataType
dataType
)
{
TypeVertexInfo
info
=
null
;
info
=
new
TypeVertexInfo
(
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
dataType
.
getDescription
());
addTypeIfNeeded
(
info
);
}
public
List
<
TypeVertexInfo
>
getVerticesToCreate
()
{
return
toCreate
;
}
@Override
public
void
visitDataType
(
TypeCategory
category
,
String
typeName
,
String
typeDescription
)
{
TypeVertexInfo
info
=
new
TypeVertexInfo
(
category
,
typeName
,
typeDescription
);
addTypeIfNeeded
(
info
);
}
}
\ No newline at end of file
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java
0 → 100644
View file @
02cf8c48
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
repository
.
typestore
;
import
java.util.Objects
;
import
org.apache.atlas.typesystem.types.DataTypes
;
import
org.apache.atlas.typesystem.types.DataTypes.TypeCategory
;
/**
* Records the information needed to create a particular type vertex.
*/
public
class
TypeVertexInfo
{
private
DataTypes
.
TypeCategory
category
;
private
String
typeName
;
private
String
typeDescription
;
public
TypeVertexInfo
(
TypeCategory
category
,
String
typeName
,
String
typeDescription
)
{
super
();
this
.
category
=
category
;
this
.
typeName
=
typeName
;
this
.
typeDescription
=
typeDescription
;
}
public
DataTypes
.
TypeCategory
getCategory
()
{
return
category
;
}
public
void
setCategory
(
DataTypes
.
TypeCategory
category
)
{
this
.
category
=
category
;
}
public
String
getTypeName
()
{
return
typeName
;
}
public
void
setTypeName
(
String
typeName
)
{
this
.
typeName
=
typeName
;
}
public
String
getTypeDescription
()
{
return
typeDescription
;
}
public
void
setTypeDescription
(
String
typeDescription
)
{
this
.
typeDescription
=
typeDescription
;
}
@Override
public
int
hashCode
()
{
return
Objects
.
hash
(
category
,
typeName
);
}
@Override
public
boolean
equals
(
Object
obj
)
{
if
(
this
==
obj
)
{
return
true
;
}
if
(
getClass
()
!=
obj
.
getClass
())
{
return
false
;
}
TypeVertexInfo
other
=
(
TypeVertexInfo
)
obj
;
if
(!
Objects
.
equals
(
category
,
other
.
category
))
{
return
false
;
}
if
(!
Objects
.
equals
(
typeName
,
other
.
typeName
))
{
return
false
;
}
return
true
;
}
}
\ No newline at end of file
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java
0 → 100644
View file @
02cf8c48
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
repository
.
typestore
;
import
java.util.List
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.repository.RepositoryException
;
import
org.apache.atlas.typesystem.types.AttributeInfo
;
import
org.apache.atlas.typesystem.types.DataTypes.TypeCategory
;
import
org.apache.atlas.typesystem.types.EnumType
;
import
org.apache.atlas.typesystem.types.IDataType
;
/**
* Callback mechanism used when storing types. As {@link GraphBackedTypeStore} traverses
* through the types being persisted, these methods are called with the information that
* it finds.
*/
public
interface
TypeVisitor
{
/**
* Called when an enumeration type is found
* @param type
* @throws AtlasException
*/
void
visitEnumeration
(
EnumType
type
)
throws
AtlasException
;
/**
* Called with a data type that is associated with a given attribute. There can
* be more than one. For example, map types have both a key and a value type.
* This is called once for each type. This is called once for each datatype
* associated with the given attribute.
*
* @param typeName The name of the type being processed.
* @param sourceAttr The attribute in that type that we are processing.
* @param attrType A dataType associated with that attribute.
* @throws AtlasException
*/
void
visitAttributeDataType
(
String
typeName
,
AttributeInfo
sourceAttr
,
IDataType
attrType
)
throws
AtlasException
;
/**
* Called when a super type is found. It is called once for each superType.
*
* @param typeName The type being processed.
* @param superType The name of the super type that was found.
* @throws RepositoryException
* @throws AtlasException
*/
void
visitSuperType
(
String
typeName
,
String
superType
)
throws
RepositoryException
,
AtlasException
;
/**
* Called with the list of immediate attribute names that were found for the given type. It
* is called once per type.
*
* @param typeName The name of the type that is being processed.
* @param attrNames The names of all of the immediate attributes in the type.
* @throws AtlasException
*/
void
visitAttributeNames
(
String
typeName
,
List
<
String
>
attrNames
)
throws
AtlasException
;
/**
* Called once for each immediate attribute in a type.
* @param typeName The name of the type that is being procesed
* @param attribute The immediate attribute that was found
*
* @throws StorageException
* @throws AtlasException
*/
void
visitAttribute
(
String
typeName
,
AttributeInfo
attribute
)
throws
StorageException
,
AtlasException
;
/**
* Called once for each struct, class, and trait type that was found. It is
* called when we start processing that type.
*
* @param category The category of the type
* @param typeName The name of the type
* @param typeDescription The description of the type.
*/
void
visitDataType
(
TypeCategory
category
,
String
typeName
,
String
typeDescription
);
}
\ No newline at end of file
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
View file @
02cf8c48
...
...
@@ -23,6 +23,7 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAt
import
static
org
.
apache
.
atlas
.
typesystem
.
types
.
utils
.
TypesUtil
.
createRequiredAttrDef
;
import
static
org
.
apache
.
atlas
.
typesystem
.
types
.
utils
.
TypesUtil
.
createStructTypeDef
;
import
java.util.Collections
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.Map
;
...
...
@@ -32,6 +33,7 @@ import javax.inject.Inject;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.RepositoryMetadataModule
;
import
org.apache.atlas.TestUtils
;
import
org.apache.atlas.repository.RepositoryException
;
import
org.apache.atlas.repository.graph.AtlasGraphProvider
;
import
org.apache.atlas.repository.graph.GraphHelper
;
import
org.apache.atlas.repository.graphdb.AtlasEdge
;
...
...
@@ -200,11 +202,11 @@ public class GraphBackedTypeStoreTest {
verifyEdges
();
}
private
void
verifyEdges
()
{
private
void
verifyEdges
()
throws
RepositoryException
{
// ATLAS-474: verify that type update did not write duplicate edges to the type store.
if
(
typeStore
instanceof
GraphBackedTypeStore
)
{
GraphBackedTypeStore
gbTypeStore
=
(
GraphBackedTypeStore
)
typeStore
;
AtlasVertex
typeVertex
=
gbTypeStore
.
findVert
ex
(
TypeCategory
.
CLASS
,
"Department"
);
AtlasVertex
typeVertex
=
gbTypeStore
.
findVert
ices
(
Collections
.
singletonList
(
"Department"
)).
get
(
"Department"
);
int
edgeCount
=
countOutgoingEdges
(
typeVertex
,
gbTypeStore
.
getEdgeLabel
(
"Department"
,
"employees"
));
Assert
.
assertEquals
(
edgeCount
,
1
,
"Should only be 1 edge for employees attribute on Department type AtlasVertex"
);
}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment