Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
cbca591d
Commit
cbca591d
authored
6 years ago
by
Sarath Subramanian
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-2672: Change primitive map type storage in vertex
parent
b7641e74
Show whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
263 additions
and
116 deletions
+263
-116
AtlasJanusGraphDatabase.java
...las/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+1
-5
TestUtilsV2.java
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+56
-0
GraphBackedSearchIndexer.java
...ache/atlas/repository/graph/GraphBackedSearchIndexer.java
+53
-11
GraphHelper.java
...n/java/org/apache/atlas/repository/graph/GraphHelper.java
+8
-25
DeleteHandlerV1.java
...ache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+1
-1
EntityGraphMapper.java
...he/atlas/repository/store/graph/v1/EntityGraphMapper.java
+22
-69
EntityGraphRetriever.java
...atlas/repository/store/graph/v1/EntityGraphRetriever.java
+4
-4
HiveStocksTest.java
...org/apache/atlas/repository/migration/HiveStocksTest.java
+0
-0
AtlasComplexAttributesTest.java
...repository/store/graph/v1/AtlasComplexAttributesTest.java
+117
-1
AtlasEntityTestBase.java
.../atlas/repository/store/graph/v1/AtlasEntityTestBase.java
+1
-0
No files found.
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
View file @
cbca591d
...
...
@@ -70,7 +70,6 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
private
static
volatile
JanusGraph
graphInstance
;
public
AtlasJanusGraphDatabase
()
{
//update registry
GraphSONMapper
.
build
().
addRegistry
(
JanusGraphIoRegistry
.
getInstance
()).
create
();
}
...
...
@@ -79,14 +78,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
startLocalSolr
();
Configuration
configProperties
=
ApplicationProperties
.
get
();
Configuration
janusConfig
=
ApplicationProperties
.
getSubsetConfiguration
(
configProperties
,
GRAPH_PREFIX
);
//add serializers for non-standard property value types that Atlas uses
janusConfig
.
addProperty
(
"attributes.custom.attribute1.attribute-class"
,
TypeCategory
.
class
.
getName
());
janusConfig
.
addProperty
(
"attributes.custom.attribute1.serializer-class"
,
TypeCategorySerializer
.
class
.
getName
());
janusConfig
.
addProperty
(
"attributes.custom.attribute1.serializer-class"
,
TypeCategorySerializer
.
class
.
getName
());
//not ideal, but avoids making large changes to Atlas
janusConfig
.
addProperty
(
"attributes.custom.attribute2.attribute-class"
,
ArrayList
.
class
.
getName
());
...
...
This diff is collapsed.
Click to expand it.
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
View file @
cbca591d
...
...
@@ -18,6 +18,7 @@
package
org
.
apache
.
atlas
;
import
com.sun.org.apache.xpath.internal.operations.Bool
;
import
org.apache.atlas.model.instance.AtlasEntity
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo
;
...
...
@@ -70,6 +71,7 @@ public final class TestUtilsV2 {
public
static
final
String
TEST_USER
=
"testUser"
;
public
static
final
String
STRUCT_TYPE
=
"struct_type"
;
public
static
final
String
ENTITY_TYPE
=
"entity_type"
;
public
static
final
String
ENTITY_TYPE_MAP
=
"map_entity_type"
;
private
static
AtomicInteger
seq
=
new
AtomicInteger
();
...
...
@@ -977,6 +979,21 @@ public final class TestUtilsV2 {
return
ret
;
}
public
static
AtlasTypesDef
defineTypeWithMapAttributes
()
{
AtlasEntityDef
entityType
=
createClassTypeDef
(
ENTITY_TYPE_MAP
,
"entity_type_map_description"
,
Collections
.
emptySet
(),
createUniqueRequiredAttrDef
(
"mapAttr1"
,
"map<string,string>"
),
createUniqueRequiredAttrDef
(
"mapAttr2"
,
"map<string,int>"
),
createUniqueRequiredAttrDef
(
"mapAttr3"
,
"map<string,boolean>"
),
createOptionalAttrDef
(
"mapAttr4"
,
"map<string,float>"
),
createOptionalAttrDef
(
"mapAttr5"
,
"map<string,date>"
));
AtlasTypesDef
ret
=
AtlasTypeUtil
.
getTypesDef
(
Collections
.
emptyList
(),
Collections
.
emptyList
(),
Collections
.
emptyList
(),
Arrays
.
asList
(
entityType
));
populateSystemAttributes
(
ret
);
return
ret
;
}
public
static
AtlasEntityWithExtInfo
createNestedCollectionAttrEntity
()
{
AtlasEntity
entity
=
new
AtlasEntity
(
ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR
);
...
...
@@ -1059,6 +1076,45 @@ public final class TestUtilsV2 {
return
ret
;
}
public
static
AtlasEntityWithExtInfo
createMapAttrEntity
()
{
AtlasEntity
entity
=
new
AtlasEntity
(
ENTITY_TYPE_MAP
);
Map
<
String
,
String
>
map1
=
new
HashMap
<>();
map1
.
put
(
"map1Key1"
,
"value1"
);
map1
.
put
(
"map1Key2"
,
"value2"
);
map1
.
put
(
"map1Key3"
,
"value3"
);
Map
<
String
,
Integer
>
map2
=
new
HashMap
<>();
map2
.
put
(
"map2Key1"
,
100
);
map2
.
put
(
"map2Key2"
,
200
);
map2
.
put
(
"map2Key3"
,
300
);
Map
<
String
,
Boolean
>
map3
=
new
HashMap
<>();
map3
.
put
(
"map3Key1"
,
false
);
map3
.
put
(
"map3Key2"
,
true
);
map3
.
put
(
"map3Key3"
,
false
);
Map
<
String
,
Float
>
map4
=
new
HashMap
<>();
map4
.
put
(
"map4Key1"
,
1.0f
);
map4
.
put
(
"map4Key2"
,
2.0f
);
map4
.
put
(
"map4Key3"
,
3.0f
);
Map
<
String
,
Date
>
map5
=
new
HashMap
<>();
map5
.
put
(
"map5Key1"
,
new
Date
());
map5
.
put
(
"map5Key2"
,
new
Date
());
map5
.
put
(
"map5Key3"
,
new
Date
());
entity
.
setAttribute
(
"mapAttr1"
,
map1
);
entity
.
setAttribute
(
"mapAttr2"
,
map2
);
entity
.
setAttribute
(
"mapAttr3"
,
map3
);
entity
.
setAttribute
(
"mapAttr4"
,
map4
);
entity
.
setAttribute
(
"mapAttr5"
,
map5
);
AtlasEntityWithExtInfo
ret
=
new
AtlasEntityWithExtInfo
(
entity
);
return
ret
;
}
public
static
final
String
randomString
()
{
return
RandomStringUtils
.
randomAlphanumeric
(
10
);
}
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
View file @
cbca591d
...
...
@@ -42,9 +42,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import
org.apache.atlas.repository.graphdb.AtlasGraphIndex
;
import
org.apache.atlas.repository.graphdb.AtlasGraphManagement
;
import
org.apache.atlas.repository.graphdb.AtlasPropertyKey
;
import
org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1
;
import
org.apache.atlas.type.AtlasArrayType
;
import
org.apache.atlas.type.AtlasClassificationType
;
import
org.apache.atlas.type.AtlasEntityType
;
import
org.apache.atlas.type.AtlasEnumType
;
import
org.apache.atlas.type.AtlasMapType
;
import
org.apache.atlas.type.AtlasRelationshipType
;
import
org.apache.atlas.type.AtlasStructType
;
import
org.apache.atlas.type.AtlasType
;
...
...
@@ -62,6 +65,7 @@ import java.math.BigInteger;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Collections
;
import
java.util.HashMap
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.Set
;
...
...
@@ -71,6 +75,9 @@ import static org.apache.atlas.repository.Constants.*;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
LIST
;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
SET
;
import
static
org
.
apache
.
atlas
.
repository
.
graphdb
.
AtlasCardinality
.
SINGLE
;
import
static
org
.
apache
.
atlas
.
repository
.
store
.
graph
.
v1
.
AtlasGraphUtilsV1
.
isReference
;
import
static
org
.
apache
.
atlas
.
type
.
AtlasTypeUtil
.
isArrayType
;
import
static
org
.
apache
.
atlas
.
type
.
AtlasTypeUtil
.
isMapType
;
/**
...
...
@@ -267,7 +274,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex
(
management
,
TYPENAME_PROPERTY_KEY
,
String
.
class
,
true
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
VERTEX_TYPE_PROPERTY_KEY
,
String
.
class
,
false
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
CLASSIFICATION_ENTITY_GUID
,
String
.
class
,
false
,
SINGLE
,
true
,
true
);
createVertexIndex
(
management
,
VERTEX_ID_IN_IMPORT_KEY
,
Long
.
class
,
false
,
SINGLE
,
true
,
false
);
// create vertex-centric index
...
...
@@ -316,19 +322,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
boolean
isIndexable
=
attributeDef
.
getIsIndexable
();
String
attribTypeName
=
attributeDef
.
getTypeName
();
boolean
isBuiltInType
=
AtlasTypeUtil
.
isBuiltInType
(
attribTypeName
);
boolean
isArrayType
=
AtlasTypeUtil
.
isArrayType
(
attribTypeName
);
boolean
isMapType
=
AtlasTypeUtil
.
isMapType
(
attribTypeName
);
boolean
isArrayType
=
isArrayType
(
attribTypeName
);
boolean
isMapType
=
isMapType
(
attribTypeName
);
try
{
AtlasType
atlasType
=
typeRegistry
.
getType
(
typeName
);
AtlasType
attributeType
=
typeRegistry
.
getType
(
attribTypeName
);
if
(
is
MapType
||
is
ClassificationType
(
attributeType
))
{
if
(
isClassificationType
(
attributeType
))
{
LOG
.
warn
(
"Ignoring non-indexable attribute {}"
,
attribTypeName
);
}
if
(
isArrayType
)
{
}
if
(
isArrayType
)
{
createLabelIfNeeded
(
management
,
propertyName
,
attribTypeName
);
}
if
(
isEntityType
(
attributeType
))
{
AtlasArrayType
arrayType
=
(
AtlasArrayType
)
attributeType
;
boolean
isReference
=
isReference
(
arrayType
.
getElementType
());
if
(!
isReference
)
{
createPropertyKey
(
management
,
propertyName
,
ArrayList
.
class
,
SINGLE
);
}
}
if
(
isMapType
)
{
createLabelIfNeeded
(
management
,
propertyName
,
attribTypeName
);
AtlasMapType
mapType
=
(
AtlasMapType
)
attributeType
;
boolean
isReference
=
isReference
(
mapType
.
getValueType
());
if
(!
isReference
)
{
createPropertyKey
(
management
,
propertyName
,
HashMap
.
class
,
SINGLE
);
}
}
if
(
isEntityType
(
attributeType
))
{
createEdgeLabel
(
management
,
propertyName
);
}
else
if
(
isBuiltInType
)
{
if
(
isRelationshipType
(
atlasType
))
{
createEdgeIndex
(
management
,
propertyName
,
getPrimitiveClass
(
attribTypeName
),
cardinality
,
false
);
...
...
@@ -381,7 +410,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
private
Class
getPrimitiveClass
(
String
attribTypeName
)
{
switch
(
attribTypeName
.
toLowerCase
())
{
String
attributeTypeName
=
attribTypeName
.
toLowerCase
();
switch
(
attributeTypeName
)
{
case
ATLAS_TYPE_BOOLEAN:
return
Boolean
.
class
;
case
ATLAS_TYPE_BYTE:
...
...
@@ -437,6 +468,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private
AtlasPropertyKey
createPropertyKey
(
AtlasGraphManagement
management
,
String
propertyName
,
Class
propertyClass
,
AtlasCardinality
cardinality
)
{
AtlasPropertyKey
propertyKey
=
management
.
getPropertyKey
(
propertyName
);
if
(
propertyKey
==
null
)
{
propertyKey
=
management
.
makePropertyKey
(
propertyName
,
propertyClass
,
cardinality
);
}
return
propertyKey
;
}
private
AtlasPropertyKey
createVertexIndex
(
AtlasGraphManagement
management
,
String
propertyName
,
Class
propertyClass
,
boolean
isUnique
,
AtlasCardinality
cardinality
,
boolean
createCompositeIndex
,
boolean
createCompositeIndexWithTypeAndSuperTypes
)
{
...
...
@@ -691,15 +732,15 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
final
String
propertyName
=
GraphHelper
.
encodePropertyKey
(
typeName
+
"."
+
attributeDef
.
getName
());
String
attribTypeName
=
attributeDef
.
getTypeName
();
boolean
isBuiltInType
=
AtlasTypeUtil
.
isBuiltInType
(
attribTypeName
);
boolean
isArrayType
=
AtlasTypeUtil
.
isArrayType
(
attribTypeName
);
boolean
isMapType
=
AtlasTypeUtil
.
isMapType
(
attribTypeName
);
boolean
isArrayType
=
isArrayType
(
attribTypeName
);
boolean
isMapType
=
isMapType
(
attribTypeName
);
try
{
AtlasType
atlasType
=
typeRegistry
.
getType
(
attribTypeName
);
if
(
is
MapType
||
isArrayType
||
is
ClassificationType
(
atlasType
)
||
isEntityType
(
atlasType
))
{
if
(
isClassificationType
(
atlasType
)
||
isEntityType
(
atlasType
))
{
LOG
.
warn
(
"Ignoring non-indexable attribute {}"
,
attribTypeName
);
}
else
if
(
isBuiltInType
||
isEnumType
(
atlasType
))
{
}
else
if
(
isBuiltInType
||
isEnumType
(
atlasType
)
||
isArrayType
||
isMapType
)
{
cleanupIndex
(
management
,
propertyName
);
}
else
if
(
isStructType
(
atlasType
))
{
AtlasStructDef
structDef
=
typeRegistry
.
getStructDefByName
(
attribTypeName
);
...
...
@@ -714,6 +755,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if
(
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"Invalidating property key = {}"
,
propertyKey
);
}
management
.
deletePropertyKey
(
propertyKey
);
}
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
View file @
cbca591d
...
...
@@ -1567,23 +1567,13 @@ public final class GraphHelper {
return
typeName
!=
null
&&
typeName
.
startsWith
(
Constants
.
INTERNAL_PROPERTY_KEY_PREFIX
);
}
public
static
Object
getMapValueProperty
(
AtlasType
elementType
,
AtlasVertex
instanceVertex
,
String
propertyName
)
{
String
vertexPropertyName
=
GraphHelper
.
encodePropertyKey
(
propertyName
);
if
(!
AtlasGraphUtilsV1
.
isReference
(
elementType
))
{
return
instanceVertex
.
getProperty
(
vertexPropertyName
,
Object
.
class
);
}
return
null
;
}
public
static
List
<
Object
>
getArrayElementsProperty
(
AtlasType
elementType
,
AtlasVertex
instanceVertex
,
String
propertyName
,
AtlasAttribute
attribute
)
{
String
encodedPropertyName
=
GraphHelper
.
encodePropertyKey
(
propertyName
);
public
static
List
<
Object
>
getArrayElementsProperty
(
AtlasType
elementType
,
AtlasVertex
instanceVertex
,
AtlasAttribute
attribute
)
{
String
propertyName
=
attribute
.
getVertexPropertyName
();
if
(
isReference
(
elementType
))
{
return
(
List
)
getCollectionElementsUsingRelationship
(
instanceVertex
,
attribute
);
}
else
{
return
(
List
)
instanceVertex
.
getListProperty
(
encodedP
ropertyName
);
return
(
List
)
instanceVertex
.
getListProperty
(
p
ropertyName
);
}
}
...
...
@@ -1593,7 +1583,7 @@ public final class GraphHelper {
if
(
isReference
(
mapValueType
))
{
return
getReferenceMap
(
instanceVertex
,
attribute
);
}
else
{
return
getPrimitiveMap
(
instanceVertex
,
propertyName
,
mapValueType
);
return
(
Map
)
instanceVertex
.
getProperty
(
propertyName
,
Map
.
class
);
}
}
...
...
@@ -1622,18 +1612,11 @@ public final class GraphHelper {
}
// map elements for primitive types
public
static
Map
<
String
,
Object
>
getPrimitiveMap
(
AtlasVertex
instanceVertex
,
String
propertyName
,
AtlasType
mapValueType
)
{
String
encodedPropertyName
=
encodePropertyKey
(
propertyName
);
List
<
String
>
currentKeys
=
getListProperty
(
instanceVertex
,
encodedPropertyName
);
Map
<
String
,
Object
>
ret
=
new
HashMap
<>();
if
(
CollectionUtils
.
isNotEmpty
(
currentKeys
))
{
for
(
String
key
:
currentKeys
)
{
String
propertyNameForKey
=
getQualifiedNameForMapKey
(
encodedPropertyName
,
encodePropertyKey
(
key
));
Object
propertyValueForKey
=
getMapValueProperty
(
mapValueType
,
instanceVertex
,
propertyNameForKey
);
public
static
Map
<
String
,
Object
>
getPrimitiveMap
(
AtlasVertex
instanceVertex
,
String
propertyName
)
{
Map
<
String
,
Object
>
ret
=
instanceVertex
.
getProperty
(
encodePropertyKey
(
propertyName
),
Map
.
class
);
ret
.
put
(
key
,
propertyValueForKey
);
}
if
(
ret
==
null
)
{
ret
=
new
HashMap
<>();
}
return
ret
;
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
View file @
cbca591d
...
...
@@ -202,7 +202,7 @@ public abstract class DeleteHandlerV1 {
vertexInfoMap
.
put
(
guid
,
new
GraphHelper
.
VertexInfo
(
entity
,
vertex
));
for
(
AtlasStructType
.
AtlasAttribute
attributeInfo
:
entityType
.
getAllAttributes
().
values
())
{
if
(!
attributeInfo
.
isOwnedRef
())
{
if
(!
attributeInfo
.
isOwnedRef
())
{
continue
;
}
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
View file @
cbca591d
...
...
@@ -19,7 +19,6 @@ package org.apache.atlas.repository.store.graph.v1;
import
org.apache.atlas.AtlasErrorCode
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.RequestContextV1
;
import
org.apache.atlas.exception.AtlasBaseException
;
import
org.apache.atlas.model.TimeBoundary
;
...
...
@@ -82,9 +81,7 @@ import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getCollectionElementsUsingRelationship
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getClassificationEdge
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getClassificationVertex
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getListProperty
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getMapElementsProperty
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getQualifiedNameForMapKey
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getStatus
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getTraitLabel
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getTraitNames
;
...
...
@@ -92,7 +89,6 @@ import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
getTypeNames
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
isPropagationEnabled
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
isRelationshipEdge
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
setListProperty
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
string
;
import
static
org
.
apache
.
atlas
.
repository
.
graph
.
GraphHelper
.
updateModificationMetadata
;
import
static
org
.
apache
.
atlas
.
repository
.
store
.
graph
.
v1
.
AtlasGraphUtilsV1
.
getIdFromVertex
;
...
...
@@ -811,51 +807,45 @@ public class EntityGraphMapper {
boolean
isReference
=
isReference
(
mapType
.
getValueType
());
if
(
MapUtils
.
isNotEmpty
(
newVal
))
{
AtlasAttribute
inverseRefAttribute
=
attribute
.
getInverseRefAttribute
();
String
propertyName
=
ctx
.
getVertexProperty
();
if
(
isReference
)
{
for
(
Map
.
Entry
<
Object
,
Object
>
entry
:
newVal
.
entrySet
())
{
String
key
=
entry
.
getKey
().
toString
();
String
propertyName
=
(
isReference
)
?
ctx
.
getVertexProperty
()
:
getQualifiedNameForMapKey
(
ctx
.
getVertexProperty
(),
GraphHelper
.
encodePropertyKey
(
key
));
AtlasEdge
existingEdge
=
getEdgeIfExists
(
mapType
,
currentMap
,
key
);
AttributeMutationContext
mapCtx
=
new
AttributeMutationContext
(
ctx
.
getOp
(),
ctx
.
getReferringVertex
(),
attribute
,
entry
.
getValue
(),
propertyName
,
mapType
.
getValueType
(),
existingEdge
);
//Add/Update/Remove property value
// Add/Update/Remove property value
Object
newEntry
=
mapCollectionElementsToVertex
(
mapCtx
,
context
);
if
(
isReferenc
e
)
{
if
(
newEntry
instanceof
AtlasEdg
e
)
{
AtlasEdge
edge
=
(
AtlasEdge
)
newEntry
;
edge
.
setProperty
(
ATTRIBUTE_KEY_PROPERTY_KEY
,
key
);
}
else
{
ctx
.
getReferringVertex
().
setProperty
(
propertyName
,
newEntry
);
}
newMap
.
put
(
key
,
newEntr
y
);
edge
.
setProperty
(
ATTRIBUTE_KEY_PROPERTY_KEY
,
ke
y
);
// If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute,
// update the inverse reference value.
if
(
isReference
&&
newEntry
instanceof
AtlasEdge
&&
inverseRefAttribute
!=
null
)
{
AtlasEdge
newEdge
=
(
AtlasEdge
)
newEntry
;
AtlasAttribute
inverseRefAttribute
=
attribute
.
getInverseRefAttribute
();
addInverseReference
(
context
,
inverseRefAttribute
,
newEdge
,
getRelationshipAttributes
(
ctx
.
getValue
()));
if
(
inverseRefAttribute
!=
null
)
{
addInverseReference
(
context
,
inverseRefAttribute
,
edge
,
getRelationshipAttributes
(
ctx
.
getValue
()));
}
}
}
Map
<
String
,
Object
>
finalMap
=
removeUnusedMapEntries
(
attribute
,
ctx
.
getReferringVertex
(),
ctx
.
getVertexProperty
(),
currentMap
,
newMap
);
for
(
Object
newEntry
:
newMap
.
values
())
{
updateInConsistentOwnedMapVertices
(
ctx
,
mapType
,
newEntry
);
}
// for dereference on way out for primitive map types
if
(!
isReference
)
{
Set
<
String
>
newKeys
=
new
LinkedHashSet
<>(
newMap
.
keySet
());
newMap
.
put
(
key
,
newEntry
);
}
}
newKeys
.
addAll
(
finalMap
.
keySet
());
Map
<
String
,
Object
>
finalMap
=
removeUnusedMapEntries
(
attribute
,
ctx
.
getReferringVertex
(),
currentMap
,
newMap
);
newMap
.
putAll
(
finalMap
);
}
else
{
// primitive type map
ctx
.
getReferringVertex
().
setProperty
(
propertyName
,
new
HashMap
<>(
newVal
));
setListProperty
(
ctx
.
getReferringVertex
(),
ctx
.
getVertexProperty
(),
new
ArrayList
<>(
newKeys
));
newVal
.
forEach
((
key
,
value
)
->
newMap
.
put
(
key
.
toString
(),
value
));
}
}
if
(
LOG
.
isDebugEnabled
())
{
...
...
@@ -1084,42 +1074,13 @@ public class EntityGraphMapper {
return
ret
;
}
public
static
Object
getMapValueProperty
(
AtlasType
elementType
,
AtlasVertex
vertex
,
String
vertexPropertyName
)
{
if
(
isReference
(
elementType
))
{
return
vertex
.
getProperty
(
vertexPropertyName
,
AtlasEdge
.
class
);
}
else
if
(
elementType
instanceof
AtlasArrayType
)
{
return
vertex
.
getProperty
(
vertexPropertyName
,
List
.
class
);
}
else
if
(
elementType
instanceof
AtlasMapType
)
{
return
vertex
.
getProperty
(
vertexPropertyName
,
Map
.
class
);
}
else
{
return
vertex
.
getProperty
(
vertexPropertyName
,
String
.
class
).
toString
();
}
}
private
static
void
setMapValueProperty
(
AtlasType
elementType
,
AtlasVertex
vertex
,
String
vertexPropertyName
,
Object
value
)
{
if
(
isReference
(
elementType
))
{
vertex
.
setPropertyFromElementId
(
vertexPropertyName
,
(
AtlasEdge
)
value
);
}
else
{
vertex
.
setProperty
(
vertexPropertyName
,
value
);
}
}
//Remove unused entries from map
private
Map
<
String
,
Object
>
removeUnusedMapEntries
(
AtlasAttribute
attribute
,
AtlasVertex
vertex
,
String
propertyName
,
Map
<
String
,
Object
>
currentMap
,
Map
<
String
,
Object
>
newMap
)
throws
AtlasBaseException
{
AtlasMapType
mapType
=
(
AtlasMapType
)
attribute
.
getAttributeType
();
boolean
isReference
=
isReference
(
mapType
.
getValueType
());
//Remove unused entries for reference map
private
Map
<
String
,
Object
>
removeUnusedMapEntries
(
AtlasAttribute
attribute
,
AtlasVertex
vertex
,
Map
<
String
,
Object
>
currentMap
,
Map
<
String
,
Object
>
newMap
)
throws
AtlasBaseException
{
Map
<
String
,
Object
>
additionalMap
=
new
HashMap
<>();
AtlasMapType
mapType
=
(
AtlasMapType
)
attribute
.
getAttributeType
();
for
(
String
currentKey
:
currentMap
.
keySet
())
{
boolean
shouldDeleteKey
=
!
newMap
.
containsKey
(
currentKey
);
if
(
isReference
)
{
//Delete the edge reference if its not part of new edges created/updated
AtlasEdge
currentEdge
=
(
AtlasEdge
)
currentMap
.
get
(
currentKey
);
...
...
@@ -1128,15 +1089,7 @@ public class EntityGraphMapper {
if
(!
deleted
)
{
additionalMap
.
put
(
currentKey
,
currentEdge
);
shouldDeleteKey
=
false
;
}
}
}
if
(!
isReference
&&
shouldDeleteKey
)
{
String
propertyNameForKey
=
getQualifiedNameForMapKey
(
propertyName
,
GraphHelper
.
encodePropertyKey
(
currentKey
));
GraphHelper
.
setProperty
(
vertex
,
propertyNameForKey
,
null
);
}
}
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
View file @
cbca591d
...
...
@@ -619,7 +619,7 @@ public final class EntityGraphRetriever {
ret
=
mapVertexToObjectId
(
entityVertex
,
edgeLabel
,
null
,
entityExtInfo
,
isOwnedAttribute
,
edgeDirection
);
break
;
case
ARRAY:
ret
=
mapVertexToArray
(
entityVertex
,
vertexPropertyName
,
entityExtInfo
,
isOwnedAttribute
,
attribute
);
ret
=
mapVertexToArray
(
entityVertex
,
entityExtInfo
,
isOwnedAttribute
,
attribute
);
break
;
case
MAP:
ret
=
mapVertexToMap
(
entityVertex
,
vertexPropertyName
,
entityExtInfo
,
isOwnedAttribute
,
attribute
);
...
...
@@ -660,7 +660,7 @@ public final class EntityGraphRetriever {
}
}
}
else
{
ret
=
getPrimitiveMap
(
entityVertex
,
propertyName
,
mapValueType
);
ret
=
getPrimitiveMap
(
entityVertex
,
propertyName
);
}
if
(
MapUtils
.
isEmpty
(
ret
))
{
...
...
@@ -670,12 +670,12 @@ public final class EntityGraphRetriever {
return
ret
;
}
private
List
<
Object
>
mapVertexToArray
(
AtlasVertex
entityVertex
,
String
propertyName
,
AtlasEntityExtInfo
entityExtInfo
,
private
List
<
Object
>
mapVertexToArray
(
AtlasVertex
entityVertex
,
AtlasEntityExtInfo
entityExtInfo
,
boolean
isOwnedAttribute
,
AtlasAttribute
attribute
)
throws
AtlasBaseException
{
AtlasArrayType
arrayType
=
(
AtlasArrayType
)
attribute
.
getAttributeType
();
AtlasType
arrayElementType
=
arrayType
.
getElementType
();
List
<
Object
>
arrayElements
=
getArrayElementsProperty
(
arrayElementType
,
entityVertex
,
propertyName
,
attribute
);
List
<
Object
>
arrayElements
=
getArrayElementsProperty
(
arrayElementType
,
entityVertex
,
attribute
);
if
(
CollectionUtils
.
isEmpty
(
arrayElements
))
{
return
null
;
...
...
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
View file @
cbca591d
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java
View file @
cbca591d
...
...
@@ -30,18 +30,21 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import
org.apache.atlas.repository.graphdb.AtlasEdge
;
import
org.apache.atlas.repository.graphdb.AtlasEdgeDirection
;
import
org.apache.atlas.repository.graphdb.AtlasVertex
;
import
org.apache.commons.lang.time.DateUtils
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.Guice
;
import
org.testng.annotations.Test
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.Map
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE_MAP
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE
;
import
static
org
.
apache
.
atlas
.
TestUtilsV2
.
NAME
;
...
...
@@ -54,18 +57,21 @@ import static org.testng.AssertJUnit.assertEquals;
public
class
AtlasComplexAttributesTest
extends
AtlasEntityTestBase
{
private
AtlasEntityWithExtInfo
complexCollectionAttrEntity
;
private
AtlasEntityWithExtInfo
complexCollectionAttrEntityForDelete
;
private
AtlasEntityWithExtInfo
mapAttributesEntity
;
@BeforeClass
public
void
setUp
()
throws
Exception
{
super
.
setUp
();
// create typeDefs
AtlasTypesDef
[]
testTypesDefs
=
new
AtlasTypesDef
[]
{
TestUtilsV2
.
defineTypeWithComplexCollectionAttributes
()
};
AtlasTypesDef
[]
testTypesDefs
=
new
AtlasTypesDef
[]
{
TestUtilsV2
.
defineTypeWithComplexCollectionAttributes
(),
TestUtilsV2
.
defineTypeWithMapAttributes
()
};
createTypesDef
(
testTypesDefs
);
// create entity
complexCollectionAttrEntity
=
TestUtilsV2
.
createComplexCollectionAttrEntity
();
complexCollectionAttrEntityForDelete
=
TestUtilsV2
.
createComplexCollectionAttrEntity
();
mapAttributesEntity
=
TestUtilsV2
.
createMapAttrEntity
();
}
@Test
...
...
@@ -78,6 +84,116 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
validateEntity
(
complexCollectionAttrEntity
,
getEntityFromStore
(
entityCreated
));
}
@Test
public
void
testPrimitiveMapAttributes
()
throws
Exception
{
init
();
EntityMutationResponse
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
mapAttributesEntity
),
false
);
AtlasEntityHeader
entityCreated
=
response
.
getFirstCreatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
AtlasEntity
entityFromStore
=
getEntityFromStore
(
entityCreated
);
validateEntity
(
mapAttributesEntity
,
entityFromStore
);
// Modify map of primitives
AtlasEntity
attrEntity
=
getEntityFromStore
(
mapAttributesEntity
.
getEntity
().
getGuid
());
Map
<
String
,
String
>
map1
=
new
HashMap
<
String
,
String
>()
{{
put
(
"map1Key11"
,
"value11"
);
put
(
"map1Key22"
,
"value22"
);
put
(
"map1Key33"
,
"value33"
);
}};
Map
<
String
,
Integer
>
map2
=
new
HashMap
<
String
,
Integer
>()
{{
put
(
"map2Key11"
,
1100
);
put
(
"map2Key22"
,
2200
);
put
(
"map2Key33"
,
3300
);
}};
Map
<
String
,
Boolean
>
map3
=
new
HashMap
<
String
,
Boolean
>()
{{
put
(
"map3Key11"
,
true
);
put
(
"map3Key22"
,
false
);
put
(
"map3Key33"
,
true
);
}};
Map
<
String
,
Float
>
map4
=
new
HashMap
<
String
,
Float
>()
{{
put
(
"map4Key11"
,
11.0f
);
put
(
"map4Key22"
,
22.0f
);
put
(
"map4Key33"
,
33.0f
);
}};
Map
<
String
,
Date
>
map5
=
new
HashMap
<
String
,
Date
>()
{{
put
(
"map5Key11"
,
DateUtils
.
addHours
(
new
Date
(),
1
));
put
(
"map5Key22"
,
DateUtils
.
addHours
(
new
Date
(),
2
));
put
(
"map5Key33"
,
DateUtils
.
addHours
(
new
Date
(),
3
));
}};
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
AtlasEntitiesWithExtInfo
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
AtlasEntityHeader
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
AtlasEntity
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
// Add new entry to map of primitives
map1
.
put
(
"map1Key44"
,
"value44"
);
map2
.
put
(
"map2Key44"
,
4400
);
map3
.
put
(
"map3Key44"
,
false
);
map4
.
put
(
"map4Key44"
,
44.0f
);
map5
.
put
(
"map5Key44"
,
DateUtils
.
addHours
(
new
Date
(),
4
));
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
// Remove an entry from map of primitives
map1
.
remove
(
"map1Key11"
);
map2
.
remove
(
"map2Key11"
);
map3
.
remove
(
"map3Key11"
);
map4
.
remove
(
"map4Key11"
);
map5
.
remove
(
"map5Key11"
);
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
// Edit existing entry to map of primitives
map1
.
put
(
"map1Key44"
,
"value44-edit"
);
map2
.
put
(
"map2Key44"
,
5555
);
map3
.
put
(
"map3Key44"
,
true
);
map4
.
put
(
"map4Key44"
,
55.5f
);
map5
.
put
(
"map5Key44"
,
DateUtils
.
addHours
(
new
Date
(),
5
));
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
// clear primitive map entries
map1
.
clear
();
map2
.
clear
();
map3
.
clear
();
map4
.
clear
();
map5
.
clear
();
updateEntityMapAttributes
(
attrEntity
,
map1
,
map2
,
map3
,
map4
,
map5
);
attrEntitiesInfo
=
new
AtlasEntitiesWithExtInfo
(
attrEntity
);
response
=
entityStore
.
createOrUpdate
(
new
AtlasEntityStream
(
attrEntitiesInfo
),
false
);
updatedAttrEntity
=
response
.
getFirstUpdatedEntityByTypeName
(
ENTITY_TYPE_MAP
);
updatedFromStore
=
getEntityFromStore
(
updatedAttrEntity
);
validateEntity
(
attrEntitiesInfo
,
updatedFromStore
);
}
private
void
updateEntityMapAttributes
(
AtlasEntity
attrEntity
,
Map
<
String
,
String
>
map1
,
Map
<
String
,
Integer
>
map2
,
Map
<
String
,
Boolean
>
map3
,
Map
<
String
,
Float
>
map4
,
Map
<
String
,
Date
>
map5
)
{
attrEntity
.
setAttribute
(
"mapAttr1"
,
map1
);
attrEntity
.
setAttribute
(
"mapAttr2"
,
map2
);
attrEntity
.
setAttribute
(
"mapAttr3"
,
map3
);
attrEntity
.
setAttribute
(
"mapAttr4"
,
map4
);
attrEntity
.
setAttribute
(
"mapAttr5"
,
map5
);
}
@Test
(
dependsOnMethods
=
"testCreateComplexAttributeEntity"
)
public
void
testStructArray
()
throws
Exception
{
init
();
...
...
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
View file @
cbca591d
...
...
@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import
org.apache.atlas.model.typedef.AtlasTypesDef
;
import
org.apache.atlas.repository.graph.AtlasGraphProvider
;
import
org.apache.atlas.repository.graph.GraphBackedSearchIndexer
;
import
org.apache.atlas.repository.impexp.ExportService
;
import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer
;
import
org.apache.atlas.repository.store.graph.AtlasEntityStore
;
import
org.apache.atlas.runner.LocalSolrRunner
;
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment