Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
cbca591d
Commit
cbca591d
authored
May 14, 2018
by
Sarath Subramanian
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-2672: Change primitive map type storage in vertex
parent
b7641e74
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
287 additions
and
140 deletions
+287
-140
AtlasJanusGraphDatabase.java
...las/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+2
-6
TestUtilsV2.java
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+57
-1
GraphBackedSearchIndexer.java
...ache/atlas/repository/graph/GraphBackedSearchIndexer.java
+55
-13
GraphHelper.java
...n/java/org/apache/atlas/repository/graph/GraphHelper.java
+8
-25
DeleteHandlerV1.java
...ache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+1
-1
EntityGraphMapper.java
...he/atlas/repository/store/graph/v1/EntityGraphMapper.java
+39
-86
EntityGraphRetriever.java
...atlas/repository/store/graph/v1/EntityGraphRetriever.java
+4
-4
HiveStocksTest.java
...org/apache/atlas/repository/migration/HiveStocksTest.java
+3
-3
AtlasComplexAttributesTest.java
...repository/store/graph/v1/AtlasComplexAttributesTest.java
+117
-1
AtlasEntityTestBase.java
.../atlas/repository/store/graph/v1/AtlasEntityTestBase.java
+1
-0
No files found.
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
View file @
cbca591d
...
@@ -70,7 +70,6 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
...
@@ -70,7 +70,6 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
private
static
volatile
JanusGraph
graphInstance
;
private
static
volatile
JanusGraph
graphInstance
;
public
AtlasJanusGraphDatabase
()
{
public
AtlasJanusGraphDatabase
()
{
//update registry
//update registry
GraphSONMapper
.
build
().
addRegistry
(
JanusGraphIoRegistry
.
getInstance
()).
create
();
GraphSONMapper
.
build
().
addRegistry
(
JanusGraphIoRegistry
.
getInstance
()).
create
();
}
}
...
@@ -79,14 +78,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
...
@@ -79,14 +78,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
startLocalSolr
();
startLocalSolr
();
Configuration
configProperties
=
ApplicationProperties
.
get
();
Configuration
configProperties
=
ApplicationProperties
.
get
();
Configuration
janusConfig
=
ApplicationProperties
.
getSubsetConfiguration
(
configProperties
,
GRAPH_PREFIX
);
Configuration
janusConfig
=
ApplicationProperties
.
getSubsetConfiguration
(
configProperties
,
GRAPH_PREFIX
);
//add serializers for non-standard property value types that Atlas uses
//add serializers for non-standard property value types that Atlas uses
janusConfig
.
addProperty
(
"attributes.custom.attribute1.attribute-class"
,
TypeCategory
.
class
.
getName
());
janusConfig
.
addProperty
(
"attributes.custom.attribute1.attribute-class"
,
TypeCategory
.
class
.
getName
());
janusConfig
.
addProperty
(
"attributes.custom.attribute1.serializer-class"
,
janusConfig
.
addProperty
(
"attributes.custom.attribute1.serializer-class"
,
TypeCategorySerializer
.
class
.
getName
());
TypeCategorySerializer
.
class
.
getName
());
//not ideal, but avoids making large changes to Atlas
//not ideal, but avoids making large changes to Atlas
janusConfig
.
addProperty
(
"attributes.custom.attribute2.attribute-class"
,
ArrayList
.
class
.
getName
());
janusConfig
.
addProperty
(
"attributes.custom.attribute2.attribute-class"
,
ArrayList
.
class
.
getName
());
...
...
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
View file @
cbca591d
...
@@ -18,6 +18,7 @@
...
@@ -18,6 +18,7 @@
package
org
.
apache
.
atlas
;
package
org
.
apache
.
atlas
;
import
com.sun.org.apache.xpath.internal.operations.Bool
;
import
org.apache.atlas.model.instance.AtlasEntity
;
import
org.apache.atlas.model.instance.AtlasEntity
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo
;
...
@@ -67,9 +68,10 @@ public final class TestUtilsV2 {
...
@@ -67,9 +68,10 @@ public final class TestUtilsV2 {
public
static
final
long
TEST_DATE_IN_LONG
=
1418265358440L
;
public
static
final
long
TEST_DATE_IN_LONG
=
1418265358440L
;
public
static
final
String
TEST_USER
=
"testUser"
;
public
static
final
String
TEST_USER
=
"testUser"
;
public
static
final
String
STRUCT_TYPE
=
"struct_type"
;
public
static
final
String
STRUCT_TYPE
=
"struct_type"
;
public
static
final
String
ENTITY_TYPE
=
"entity_type"
;
public
static
final
String
ENTITY_TYPE
=
"entity_type"
;
public
static
final
String
ENTITY_TYPE_MAP
=
"map_entity_type"
;
private
static
AtomicInteger
seq
=
new
AtomicInteger
();
private
static
AtomicInteger
seq
=
new
AtomicInteger
();
...
@@ -977,6 +979,21 @@ public final class TestUtilsV2 {
...
@@ -977,6 +979,21 @@ public final class TestUtilsV2 {
return
ret
;
return
ret
;
}
}
public
static
AtlasTypesDef
defineTypeWithMapAttributes
()
{
AtlasEntityDef
entityType
=
createClassTypeDef
(
ENTITY_TYPE_MAP
,
"entity_type_map_description"
,
Collections
.
emptySet
(),
createUniqueRequiredAttrDef
(
"mapAttr1"
,
"map<string,string>"
),
createUniqueRequiredAttrDef
(
"mapAttr2"
,
"map<string,int>"
),
createUniqueRequiredAttrDef
(
"mapAttr3"
,
"map<string,boolean>"
),
createOptionalAttrDef
(
"mapAttr4"
,
"map<string,float>"
),
createOptionalAttrDef
(
"mapAttr5"
,
"map<string,date>"
));
AtlasTypesDef
ret
=
AtlasTypeUtil
.
getTypesDef
(
Collections
.
emptyList
(),
Collections
.
emptyList
(),
Collections
.
emptyList
(),
Arrays
.
asList
(
entityType
));
populateSystemAttributes
(
ret
);
return
ret
;
}
public
static
AtlasEntityWithExtInfo
createNestedCollectionAttrEntity
()
{
public
static
AtlasEntityWithExtInfo
createNestedCollectionAttrEntity
()
{
AtlasEntity
entity
=
new
AtlasEntity
(
ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR
);
AtlasEntity
entity
=
new
AtlasEntity
(
ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR
);
...
@@ -1059,6 +1076,45 @@ public final class TestUtilsV2 {
...
@@ -1059,6 +1076,45 @@ public final class TestUtilsV2 {
return
ret
;
return
ret
;
}
}
public
static
AtlasEntityWithExtInfo
createMapAttrEntity
()
{
AtlasEntity
entity
=
new
AtlasEntity
(
ENTITY_TYPE_MAP
);
Map
<
String
,
String
>
map1
=
new
HashMap
<>();
map1
.
put
(
"map1Key1"
,
"value1"
);
map1
.
put
(
"map1Key2"
,
"value2"
);
map1
.
put
(
"map1Key3"
,
"value3"
);
Map
<
String
,
Integer
>
map2
=
new
HashMap
<>();
map2
.
put
(
"map2Key1"
,
100
);
map2
.
put
(
"map2Key2"
,
200
);
map2
.
put
(
"map2Key3"
,
300
);
Map
<
String
,
Boolean
>
map3
=
new
HashMap
<>();
map3
.
put
(
"map3Key1"
,
false
);
map3
.
put
(
"map3Key2"
,
true
);
map3
.
put
(
"map3Key3"
,
false
);
Map
<
String
,
Float
>
map4
=
new
HashMap
<>();
map4
.
put
(
"map4Key1"
,
1.0f
);
map4
.
put
(
"map4Key2"
,
2.0f
);
map4
.
put
(
"map4Key3"
,
3.0f
);
Map
<
String
,
Date
>
map5
=
new
HashMap
<>();
map5
.
put
(
"map5Key1"
,
new
Date
());
map5
.
put
(
"map5Key2"
,
new
Date
());
map5
.
put
(
"map5Key3"
,
new
Date
());
entity
.
setAttribute
(
"mapAttr1"
,
map1
);
entity
.
setAttribute
(
"mapAttr2"
,
map2
);
entity
.
setAttribute
(
"mapAttr3"
,
map3
);
entity
.
setAttribute
(
"mapAttr4"
,
map4
);
entity
.
setAttribute
(
"mapAttr5"
,
map5
);
AtlasEntityWithExtInfo
ret
=
new
AtlasEntityWithExtInfo
(
entity
);
return
ret
;
}
public
static
final
String
randomString
()
{
public
static
final
String
randomString
()
{
return
RandomStringUtils
.
randomAlphanumeric
(
10
);
return
RandomStringUtils
.
randomAlphanumeric
(
10
);
}
}
...
...
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
View file @
cbca591d
...
@@ -42,9 +42,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
...
@@ -42,9 +42,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import
org.apache.atlas.repository.graphdb.AtlasGraphIndex
;
import
org.apache.atlas.repository.graphdb.AtlasGraphIndex
;
import
org.apache.atlas.repository.graphdb.AtlasGraphManagement
;
import
org.apache.atlas.repository.graphdb.AtlasGraphManagement
;
import
org.apache.atlas.repository.graphdb.AtlasPropertyKey
;
import
org.apache.atlas.repository.graphdb.AtlasPropertyKey
;
import
org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1
;
import
org.apache.atlas.type.AtlasArrayType
;
import
org.apache.atlas.type.AtlasClassificationType
;
import
org.apache.atlas.type.AtlasClassificationType
;
import
org.apache.atlas.type.AtlasEntityType
;
import
org.apache.atlas.type.AtlasEntityType
;
import
org.apache.atlas.type.AtlasEnumType
;
import
org.apache.atlas.type.AtlasEnumType
;
import
org.apache.atlas.type.AtlasMapType
;
import
org.apache.atlas.type.AtlasRelationshipType
;
import
org.apache.atlas.type.AtlasRelationshipType
;
import
org.apache.atlas.type.AtlasStructType
;
import
org.apache.atlas.type.AtlasStructType
;
import
org.apache.atlas.type.AtlasType
;
import
org.apache.atlas.type.AtlasType
;
...
@@ -62,6 +65,7 @@ import java.math.BigInteger;
...
@@ -62,6 +65,7 @@ import java.math.BigInteger;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Arrays
;
import
java.util.Collections
;
import
java.util.Collections
;
import
java.util.HashMap
;
import
java.util.HashSet
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Set
;
import
java.util.Set
;
...
@@ -71,6 +75,9 @@ import static org.apache.atlas.repository.Constants.*;
...
@@ -71,6 +75,9 @@ import static org.apache.atlas.repository.Constants.*;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
LIST
;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
LIST
;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
SET
;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
SET
;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
SINGLE
;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
SINGLE
;
import
static
org
.
apache
.
atlas
.
repository
.
store
.
graph
.
v1
.
AtlasGraphUtilsV1
.
isReference
;
import
static
org
.
apache
.
atlas
.
type
.
AtlasTypeUtil
.
isArrayType
;
import
static
org
.
apache
.
atlas
.
type
.
AtlasTypeUtil
.
isMapType
;
/**
/**
...
@@ -267,7 +274,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
...
@@ -267,7 +274,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex
(
management
,
TYPENAME_PROPERTY_KEY
,
String
.
class
,
true
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
TYPENAME_PROPERTY_KEY
,
String
.
class
,
true
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
VERTEX_TYPE_PROPERTY_KEY
,
String
.
class
,
false
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
VERTEX_TYPE_PROPERTY_KEY
,
String
.
class
,
false
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
CLASSIFICATION_ENTITY_GUID
,
String
.
class
,
false
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
CLASSIFICATION_ENTITY_GUID
,
String
.
class
,
false
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
VERTEX_ID_IN_IMPORT_KEY
,
Long
.
class
,
false
,
SINGLE
,
true
,
false
);
createVertexIndex
(
management
,
VERTEX_ID_IN_IMPORT_KEY
,
Long
.
class
,
false
,
SINGLE
,
true
,
false
);
// create vertex-centric index
// create vertex-centric index
...
@@ -316,19 +322,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
...
@@ -316,19 +322,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
boolean
isIndexable
=
attributeDef
.
getIsIndexable
();
boolean
isIndexable
=
attributeDef
.
getIsIndexable
();
String
attribTypeName
=
attributeDef
.
getTypeName
();
String
attribTypeName
=
attributeDef
.
getTypeName
();
boolean
isBuiltInType
=
AtlasTypeUtil
.
isBuiltInType
(
attribTypeName
);
boolean
isBuiltInType
=
AtlasTypeUtil
.
isBuiltInType
(
attribTypeName
);
boolean
isArrayType
=
AtlasTypeUtil
.
isArrayType
(
attribTypeName
);
boolean
isArrayType
=
isArrayType
(
attribTypeName
);
boolean
isMapType
=
AtlasTypeUtil
.
isMapType
(
attribTypeName
);
boolean
isMapType
=
isMapType
(
attribTypeName
);
try
{
try
{
AtlasType
atlasType
=
typeRegistry
.
getType
(
typeName
);
AtlasType
atlasType
=
typeRegistry
.
getType
(
typeName
);
AtlasType
attributeType
=
typeRegistry
.
getType
(
attribTypeName
);
AtlasType
attributeType
=
typeRegistry
.
getType
(
attribTypeName
);
if
(
is
MapType
||
is
ClassificationType
(
attributeType
))
{
if
(
isClassificationType
(
attributeType
))
{
LOG
.
warn
(
"Ignoring non-indexable attribute {}"
,
attribTypeName
);
LOG
.
warn
(
"Ignoring non-indexable attribute {}"
,
attribTypeName
);
}
if
(
isArrayType
)
{
}
if
(
isArrayType
)
{
createLabelIfNeeded
(
management
,
propertyName
,
attribTypeName
);
createLabelIfNeeded
(
management
,
propertyName
,
attribTypeName
);
}
if
(
isEntityType
(
attributeType
))
{
AtlasArrayType
arrayType
=
(
AtlasArrayType
)
attributeType
;
boolean
isReference
=
isReference
(
arrayType
.
getElementType
());
if
(!
isReference
)
{
createPropertyKey
(
management
,
propertyName
,
ArrayList
.
class
,
SINGLE
);
}
}
if
(
isMapType
)
{
createLabelIfNeeded
(
management
,
propertyName
,
attribTypeName
);
AtlasMapType
mapType
=
(
AtlasMapType
)
attributeType
;
boolean
isReference
=
isReference
(
mapType
.
getValueType
());
if
(!
isReference
)
{
createPropertyKey
(
management
,
propertyName
,
HashMap
.
class
,
SINGLE
);
}
}
if
(
isEntityType
(
attributeType
))
{
createEdgeLabel
(
management
,
propertyName
);
createEdgeLabel
(
management
,
propertyName
);
}
else
if
(
isBuiltInType
)
{
}
else
if
(
isBuiltInType
)
{
if
(
isRelationshipType
(
atlasType
))
{
if
(
isRelationshipType
(
atlasType
))
{
createEdgeIndex
(
management
,
propertyName
,
getPrimitiveClass
(
attribTypeName
),
cardinality
,
false
);
createEdgeIndex
(
management
,
propertyName
,
getPrimitiveClass
(
attribTypeName
),
cardinality
,
false
);
...
@@ -381,7 +410,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
...
@@ -381,7 +410,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private
Class
getPrimitiveClass
(
String
attribTypeName
)
{
private
Class
getPrimitiveClass
(
String
attribTypeName
)
{
switch
(
attribTypeName
.
toLowerCase
())
{
String
attributeTypeName
=
attribTypeName
.
toLowerCase
();
switch
(
attributeTypeName
)
{
case
ATLAS_TYPE_BOOLEAN:
case
ATLAS_TYPE_BOOLEAN:
return
Boolean
.
class
;
return
Boolean
.
class
;
case
ATLAS_TYPE_BYTE:
case
ATLAS_TYPE_BYTE:
...
@@ -437,6 +468,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
...
@@ -437,6 +468,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
}
}
private
AtlasPropertyKey
createPropertyKey
(
AtlasGraphManagement
management
,
String
propertyName
,
Class
propertyClass
,
AtlasCardinality
cardinality
)
{
AtlasPropertyKey
propertyKey
=
management
.
getPropertyKey
(
propertyName
);
if
(
propertyKey
==
null
)
{
propertyKey
=
management
.
makePropertyKey
(
propertyName
,
propertyClass
,
cardinality
);
}
return
propertyKey
;
}
private
AtlasPropertyKey
createVertexIndex
(
AtlasGraphManagement
management
,
String
propertyName
,
Class
propertyClass
,
private
AtlasPropertyKey
createVertexIndex
(
AtlasGraphManagement
management
,
String
propertyName
,
Class
propertyClass
,
boolean
isUnique
,
AtlasCardinality
cardinality
,
boolean
createCompositeIndex
,
boolean
isUnique
,
AtlasCardinality
cardinality
,
boolean
createCompositeIndex
,
boolean
createCompositeIndexWithTypeAndSuperTypes
)
{
boolean
createCompositeIndexWithTypeAndSuperTypes
)
{
...
@@ -689,17 +730,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
...
@@ -689,17 +730,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private
void
cleanupIndexForAttribute
(
AtlasGraphManagement
management
,
String
typeName
,
AtlasAttributeDef
attributeDef
)
{
private
void
cleanupIndexForAttribute
(
AtlasGraphManagement
management
,
String
typeName
,
AtlasAttributeDef
attributeDef
)
{
final
String
propertyName
=
GraphHelper
.
encodePropertyKey
(
typeName
+
"."
+
attributeDef
.
getName
());
final
String
propertyName
=
GraphHelper
.
encodePropertyKey
(
typeName
+
"."
+
attributeDef
.
getName
());
String
attribTypeName
=
attributeDef
.
getTypeName
();
String
attribTypeName
=
attributeDef
.
getTypeName
();
boolean
isBuiltInType
=
AtlasTypeUtil
.
isBuiltInType
(
attribTypeName
);
boolean
isBuiltInType
=
AtlasTypeUtil
.
isBuiltInType
(
attribTypeName
);
boolean
isArrayType
=
AtlasTypeUtil
.
isArrayType
(
attribTypeName
);
boolean
isArrayType
=
isArrayType
(
attribTypeName
);
boolean
isMapType
=
AtlasTypeUtil
.
isMapType
(
attribTypeName
);
boolean
isMapType
=
isMapType
(
attribTypeName
);
try
{
try
{
AtlasType
atlasType
=
typeRegistry
.
getType
(
attribTypeName
);
AtlasType
atlasType
=
typeRegistry
.
getType
(
attribTypeName
);
if
(
is
MapType
||
isArrayType
||
is
ClassificationType
(
atlasType
)
||
isEntityType
(
atlasType
))
{
if
(
isClassificationType
(
atlasType
)
||
isEntityType
(
atlasType
))
{
LOG
.
warn
(
"Ignoring non-indexable attribute {}"
,
attribTypeName
);
LOG
.
warn
(
"Ignoring non-indexable attribute {}"
,
attribTypeName
);
}
else
if
(
isBuiltInType
||
isEnumType
(
atlasType
))
{
}
else
if
(
isBuiltInType
||
isEnumType
(
atlasType
)
||
isArrayType
||
isMapType
)
{
cleanupIndex
(
management
,
propertyName
);
cleanupIndex
(
management
,
propertyName
);
}
else
if
(
isStructType
(
atlasType
))
{
}
else
if
(
isStructType
(
atlasType
))
{
AtlasStructDef
structDef
=
typeRegistry
.
getStructDefByName
(
attribTypeName
);
AtlasStructDef
structDef
=
typeRegistry
.
getStructDefByName
(
attribTypeName
);
...
@@ -714,6 +755,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
...
@@ -714,6 +755,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if
(
LOG
.
isDebugEnabled
())
{
if
(
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"Invalidating property key = {}"
,
propertyKey
);
LOG
.
debug
(
"Invalidating property key = {}"
,
propertyKey
);
}
}
management
.
deletePropertyKey
(
propertyKey
);
management
.
deletePropertyKey
(
propertyKey
);
}
}
...
...
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
View file @
cbca591d
...
@@ -1567,23 +1567,13 @@ public final class GraphHelper {
...
@@ -1567,23 +1567,13 @@ public final class GraphHelper {
return
typeName
!=
null
&&
typeName
.
startsWith
(
Constants
.
INTERNAL_PROPERTY_KEY_PREFIX
);
return
typeName
!=
null
&&
typeName
.
startsWith
(
Constants
.
INTERNAL_PROPERTY_KEY_PREFIX
);
}
}
public
static
Object
getMapValueProperty
(
AtlasType
elementType
,
AtlasVertex
instanceVertex
,
String
propertyName
)
{
public
static
List
<
Object
>
getArrayElementsProperty
(
AtlasType
elementType
,
AtlasVertex
instanceVertex
,
AtlasAttribute
attribute
)
{
String
vertexPropertyName
=
GraphHelper
.
encodePropertyKey
(
propertyName
);
String
propertyName
=
attribute
.
getVertexPropertyName
();
if
(!
AtlasGraphUtilsV1
.
isReference
(
elementType
))
{
return
instanceVertex
.
getProperty
(
vertexPropertyName
,
Object
.
class
);
}
return
null
;
}
public
static
List
<
Object
>
getArrayElementsProperty
(
AtlasType
elementType
,
AtlasVertex
instanceVertex
,
String
propertyName
,
AtlasAttribute
attribute
)
{
String
encodedPropertyName
=
GraphHelper
.
encodePropertyKey
(
propertyName
);
if
(
isReference
(
elementType
))
{
if
(
isReference
(
elementType
))
{
return
(
List
)
getCollectionElementsUsingRelationship
(
instanceVertex
,
attribute
);
return
(
List
)
getCollectionElementsUsingRelationship
(
instanceVertex
,
attribute
);
}
else
{
}
else
{
return
(
List
)
instanceVertex
.
getListProperty
(
encodedP
ropertyName
);
return
(
List
)
instanceVertex
.
getListProperty
(
p
ropertyName
);
}
}
}
}
...
@@ -1593,7 +1583,7 @@ public final class GraphHelper {
...
@@ -1593,7 +1583,7 @@ public final class GraphHelper {
if
(
isReference
(
mapValueType
))
{
if
(
isReference
(
mapValueType
))
{
return
getReferenceMap
(
instanceVertex
,
attribute
);
return
getReferenceMap
(
instanceVertex
,
attribute
);
}
else
{
}
else
{
return
getPrimitiveMap
(
instanceVertex
,
propertyName
,
mapValueType
);
return
(
Map
)
instanceVertex
.
getProperty
(
propertyName
,
Map
.
class
);
}
}
}
}
...
@@ -1622,18 +1612,11 @@ public final class GraphHelper {
...
@@ -1622,18 +1612,11 @@ public final class GraphHelper {
}
}
// map elements for primitive types
// map elements for primitive types
public
static
Map
<
String
,
Object
>
getPrimitiveMap
(
AtlasVertex
instanceVertex
,
String
propertyName
,
AtlasType
mapValueType
)
{
public
static
Map
<
String
,
Object
>
getPrimitiveMap
(
AtlasVertex
instanceVertex
,
String
propertyName
)
{
String
encodedPropertyName
=
encodePropertyKey
(
propertyName
);
Map
<
String
,
Object
>
ret
=
instanceVertex
.
getProperty
(
encodePropertyKey
(
propertyName
),
Map
.
class
);
List
<
String
>
currentKeys
=
getListProperty
(
instanceVertex
,
encodedPropertyName
);
Map
<
String
,
Object
>
ret
=
new
HashMap
<>();
if
(
CollectionUtils
.
isNotEmpty
(
currentKeys
))
{
if
(
ret
==
null
)
{
for
(
String
key
:
currentKeys
)
{
ret
=
new
HashMap
<>();
String
propertyNameForKey
=
getQualifiedNameForMapKey
(
encodedPropertyName
,
encodePropertyKey
(
key
));
Object
propertyValueForKey
=
getMapValueProperty
(
mapValueType
,
instanceVertex
,
propertyNameForKey
);
ret
.
put
(
key
,
propertyValueForKey
);
}
}
}
return
ret
;
return
ret
;
...
...
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
View file @
cbca591d
...
@@ -202,7 +202,7 @@ public abstract class DeleteHandlerV1 {
...
@@ -202,7 +202,7 @@ public abstract class DeleteHandlerV1 {
vertexInfoMap
.
put
(
guid
,
new
GraphHelper
.
VertexInfo
(
entity
,
vertex
));
vertexInfoMap
.
put
(
guid
,
new
GraphHelper
.
VertexInfo
(
entity
,
vertex
));
for
(
AtlasStructType
.
AtlasAttribute
attributeInfo
:
entityType
.
getAllAttributes
().
values
())
{
for
(
AtlasStructType
.
AtlasAttribute
attributeInfo
:
entityType
.
getAllAttributes
().
values
())
{
if
(!
attributeInfo
.
isOwnedRef
())
{
if
(!
attributeInfo
.
isOwnedRef
())
{
continue
;
continue
;
}
}
...
...
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
View file @
cbca591d
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
View file @
cbca591d
...
@@ -619,7 +619,7 @@ public final class EntityGraphRetriever {
...
@@ -619,7 +619,7 @@ public final class EntityGraphRetriever {
ret
=
mapVertexToObjectId
(
entityVertex
,
edgeLabel
,
null
,
entityExtInfo
,
isOwnedAttribute
,
edgeDirection
);
ret
=
mapVertexToObjectId
(
entityVertex
,
edgeLabel
,
null
,
entityExtInfo
,
isOwnedAttribute
,
edgeDirection
);
break
;
break
;
case
ARRAY:
case
ARRAY:
ret
=
mapVertexToArray
(
entityVertex
,
vertexPropertyName
,
entityExtInfo
,
isOwnedAttribute
,
attribute
);
ret
=
mapVertexToArray
(
entityVertex
,
entityExtInfo
,
isOwnedAttribute
,
attribute
);
break
;
break
;
case
MAP:
case
MAP:
ret
=
mapVertexToMap
(
entityVertex
,
vertexPropertyName
,
entityExtInfo
,
isOwnedAttribute
,
attribute
);
ret
=
mapVertexToMap
(
entityVertex
,
vertexPropertyName
,
entityExtInfo
,
isOwnedAttribute
,
attribute
);
...
@@ -660,7 +660,7 @@ public final class EntityGraphRetriever {
...
@@ -660,7 +660,7 @@ public final class EntityGraphRetriever {
}
}
}
}
}
else
{
}
else
{
ret
=
getPrimitiveMap
(
entityVertex
,
propertyName
,
mapValueType
);
ret
=
getPrimitiveMap
(
entityVertex
,
propertyName
);
}
}
if
(
MapUtils
.
isEmpty
(
ret
))
{
if
(
MapUtils
.
isEmpty
(
ret
))
{
...
@@ -670,12 +670,12 @@ public final class EntityGraphRetriever {
...
@@ -670,12 +670,12 @@ public final class EntityGraphRetriever {
return
ret
;
return
ret
;
}
}
private
List
<
Object
>
mapVertexToArray
(
AtlasVertex
entityVertex
,
String
propertyName
,
AtlasEntityExtInfo
entityExtInfo
,
private
List
<
Object
>
mapVertexToArray
(
AtlasVertex
entityVertex
,
AtlasEntityExtInfo
entityExtInfo
,
boolean
isOwnedAttribute
,
AtlasAttribute
attribute
)
throws
AtlasBaseException
{
boolean
isOwnedAttribute
,
AtlasAttribute
attribute
)
throws
AtlasBaseException
{
AtlasArrayType
arrayType
=
(
AtlasArrayType
)
attribute
.
getAttributeType
();
AtlasArrayType
arrayType
=
(
AtlasArrayType
)
attribute
.
getAttributeType
();
AtlasType
arrayElementType
=
arrayType
.
getElementType
();
AtlasType
arrayElementType
=
arrayType
.
getElementType
();
List
<
Object
>
arrayElements
=
getArrayElementsProperty
(
arrayElementType
,
entityVertex
,
propertyName
,
attribute
);
List
<
Object
>
arrayElements
=
getArrayElementsProperty
(
arrayElementType
,
entityVertex
,
attribute
);
if
(
CollectionUtils
.
isEmpty
(
arrayElements
))
{
if
(
CollectionUtils
.
isEmpty
(
arrayElements
))
{
return
null
;
return
null
;
...
...
repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
View file @
cbca591d
...
@@ -37,9 +37,9 @@ public class HiveStocksTest extends MigrationBaseAsserts {
...
@@ -37,9 +37,9 @@ public class HiveStocksTest extends MigrationBaseAsserts {
@Test
@Test
public
void
migrateStocks
()
throws
AtlasBaseException
,
IOException
{
public
void
migrateStocks
()
throws
AtlasBaseException
,
IOException
{
final
int
EXPECTED_TOTAL_COUNT
=
188
;
final
int
EXPECTED_TOTAL_COUNT
=
188
;
final
int
EXPECTED_DB_COUNT
=
1
;
final
int
EXPECTED_DB_COUNT
=
1
;
final
int
EXPECTED_TABLE_COUNT
=
1
;
final
int
EXPECTED_TABLE_COUNT
=
1
;
final
int
EXPECTED_COLUMN_COUNT
=
7
;
final
int
EXPECTED_COLUMN_COUNT
=
7
;
runFileImporter
(
"stocks_db"
);
runFileImporter
(
"stocks_db"
);
...
...
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java
View file @
cbca591d
...
@@ -30,18 +30,21 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
...
@@ -30,18 +30,21 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import
org.apache.atlas.repository.graphdb.AtlasEdge
;
import
org.apache.atlas.repository.graphdb.AtlasEdge
;
import
org.apache.atlas.repository.graphdb.AtlasEdgeDirection
;
import
org.apache.atlas.repository.graphdb.AtlasEdgeDirection
;
import
org.apache.atlas.repository.graphdb.AtlasVertex
;
import
org.apache.atlas.repository.graphdb.AtlasVertex
;
import
org.apache.commons.lang.time.DateUtils
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.Guice
;
import
org.testng.annotations.Guice
;
import
org.testng.annotations.Test
;
import
org.testng.annotations.Test
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Arrays
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.HashMap
;
import
java.util.Iterator
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE_MAP
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
NAME
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
NAME
;
...
@@ -54,18 +57,21 @@ import static org.testng.AssertJUnit.assertEquals;
...
@@ -54,18 +57,21 @@ import static org.testng.AssertJUnit.assertEquals;
public
class
AtlasComplexAttributesTest
extends
AtlasEntityTestBase
{
public
class
AtlasComplexAttributesTest
extends
AtlasEntityTestBase
{
private
AtlasEntityWithExtInfo
complexCollectionAttrEntity
;
private
AtlasEntityWithExtInfo
complexCollectionAttrEntity
;
private
AtlasEntityWithExtInfo
complexCollectionAttrEntityForDelete
;
private
AtlasEntityWithExtInfo
complexCollectionAttrEntityForDelete
;
private
AtlasEntityWithExtInfo
mapAttributesEntity
;
@BeforeClass
@BeforeClass
public
void
setUp
()
throws
Exception
{
public
void
setUp
()
throws
Exception
{
super
.
setUp
();
super
.
setUp
();
// create typeDefs
// create typeDefs
AtlasTypesDef
[]
testTypesDefs
=
new
AtlasTypesDef
[]
{
TestUtilsV2
.
defineTypeWithComplexCollectionAttributes
()
};
AtlasTypesDef
[]
testTypesDefs
=
new
AtlasTypesDef
[]
{
TestUtilsV2
.
defineTypeWithComplexCollectionAttributes
(),
TestUtilsV2
.
defineTypeWithMapAttributes
()
};
createTypesDef
(
testTypesDefs
);
createTypesDef
(
testTypesDefs
);
// create entity
// create entity
complexCollectionAttrEntity
=
TestUtilsV2
.
createComplexCollectionAttrEntity
();
complexCollectionAttrEntity
=
TestUtilsV2
.
createComplexCollectionAttrEntity
();
complexCollectionAttrEntityForDelete
=
TestUtilsV2
.
createComplexCollectionAttrEntity
();
complexCollectionAttrEntityForDelete
=
TestUtilsV2
.
createComplexCollectionAttrEntity
();
mapAttributesEntity
=
TestUtilsV2
.
createMapAttrEntity
();
}
}
@Test
@Test
...
@@ -78,6 +84,116 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
...
@@ -78,6 +84,116 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
validateEntity
(
complexCollectionAttrEntity
,
getEntityFromStore
(
entityCreated
));
validateEntity
(
complexCollectionAttrEntity
,
getEntityFromStore
(
entityCreated
));
}
}
@Test
public
void
testPrimitiveMapAttributes
()
throws
Exception
{
init
();
EntityMutationResponse
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
mapAttributesEntity
),
false
);
AtlasEntityHeader
entityCreated
=
response
.
getFirstCreatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
AtlasEntity
entityFromStore
=
getEntityFromStore
(
entityCreated
);
validateEntity
(
mapAttributesEntity
,
entityFromStore
);
// Modify map of primitives
AtlasEntity
attrEntity
=
getEntityFromStore
(
mapAttributesEntity
.
getEntity
().
getGuid
());
Map
<
String
,
String
>
map1
=
new
HashMap
<
String
,
String
>()
{{
put
(
"map1Key11"
,
"value11"
);
put
(
"map1Key22"
,
"value22"
);
put
(
"map1Key33"
,
"value33"
);
}};
Map
<
String
,
Integer
>
map2
=
new
HashMap
<
String
,
Integer
>()
{{
put
(
"map2Key11"
,
1100
);
put
(
"map2Key22"
,
2200
);
put
(
"map2Key33"
,
3300
);
}};
Map
<
String
,
Boolean
>
map3
=
new
HashMap
<
String
,
Boolean
>()
{{
put
(
"map3Key11"
,
true
);
put
(
"map3Key22"
,
false
);
put
(
"map3Key33"
,
true
);
}};
Map
<
String
,
Float
>
map4
=
new
HashMap
<
String
,
Float
>()
{{
put
(
"map4Key11"
,
11.0f
);
put
(
"map4Key22"
,
22.0f
);
put
(
"map4Key33"
,
33.0f
);
}};
Map
<
String
,
Date
>
map5
=
new
HashMap
<
String
,
Date
>()
{{
put
(
"map5Key11"
,
DateUtils
.
addHours
(
new
Date
(),
1
));
put
(
"map5Key22"
,
DateUtils
.
addHours
(
new
Date
(),
2
));
put
(
"map5Key33"
,
DateUtils
.
addHours
(
new
Date
(),
3
));
}};
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
AtlasEntitiesWithExtInfo
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
AtlasEntityHeader
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
AtlasEntity
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
// Add new entry to map of primitives
map1
.
put
(
"map1Key44"
,
"value44"
);
map2
.
put
(
"map2Key44"
,
4400
);
map3
.
put
(
"map3Key44"
,
false
);
map4
.
put
(
"map4Key44"
,
44.0f
);
map5
.
put
(
"map5Key44"
,
DateUtils
.
addHours
(
new
Date
(),
4
));
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
// Remove an entry from map of primitives
map1
.
remove
(
"map1Key11"
);
map2
.
remove
(
"map2Key11"
);
map3
.
remove
(
"map3Key11"
);
map4
.
remove
(
"map4Key11"
);
map5
.
remove
(
"map5Key11"
);
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
// Edit existing entry to map of primitives
map1
.
put
(
"map1Key44"
,
"value44-edit"
);
map2
.
put
(
"map2Key44"
,
5555
);
map3
.
put
(
"map3Key44"
,
true
);
map4
.
put
(
"map4Key44"
,
55.5f
);
map5
.
put
(
"map5Key44"
,
DateUtils
.
addHours
(
new
Date
(),
5
));
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
// clear primitive map entries
map1
.
clear
();
map2
.
clear
();
map3
.
clear
();
map4
.
clear
();
map5
.
clear
();
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
}
private
void
updateEntityMapAttributes
(
AtlasEntity
attrEntity
,
Map
<
String
,
String
>
map1
,
Map
<
String
,
Integer
>
map2
,
Map
<
String
,
Boolean
>
map3
,
Map
<
String
,
Float
>
map4
,
Map
<
String
,
Date
>
map5
)
{
attrEntity
.
setAttribute
(
"mapAttr1"
,
map1
);
attrEntity
.
setAttribute
(
"mapAttr2"
,
map2
);
attrEntity
.
setAttribute
(
"mapAttr3"
,
map3
);
attrEntity
.
setAttribute
(
"mapAttr4"
,
map4
);
attrEntity
.
setAttribute
(
"mapAttr5"
,
map5
);
}
@Test
(
dependsOnMethods
=
"testCreateComplexAttributeEntity"
)
@Test
(
dependsOnMethods
=
"testCreateComplexAttributeEntity"
)
public
void
testStructArray
()
throws
Exception
{
public
void
testStructArray
()
throws
Exception
{
init
();
init
();
...
...
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
View file @
cbca591d
...
@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
...
@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import
org.apache.atlas.model.typedef.AtlasTypesDef
;
import
org.apache.atlas.model.typedef.AtlasTypesDef
;
import
org.apache.atlas.repository.graph.AtlasGraphProvider
;
import
org.apache.atlas.repository.graph.AtlasGraphProvider
;
import
org.apache.atlas.repository.graph.GraphBackedSearchIndexer
;
import
org.apache.atlas.repository.graph.GraphBackedSearchIndexer
;
import
org.apache.atlas.repository.impexp.ExportService
;
import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer
;
import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer
;
import
org.apache.atlas.repository.store.graph.AtlasEntityStore
;
import
org.apache.atlas.repository.store.graph.AtlasEntityStore
;
import
org.apache.atlas.runner.LocalSolrRunner
;
import
org.apache.atlas.runner.LocalSolrRunner
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment