Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
801d40c3
Commit
801d40c3
authored
4 years ago
by
Ashutosh Mestry
Committed by
Nikhil Bonte
4 years ago
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-4006, ATLAS-4007: Support business metadata in export and import operations.
Signed-off-by:
Nikhil P Bonte
<
nbonte@apache.org
>
parent
e6fff748
master
No related merge requests found
Show whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
223 additions
and
2 deletions
+223
-2
AtlasServerService.java
...rg/apache/atlas/repository/impexp/AtlasServerService.java
+1
-1
ExportService.java
...ava/org/apache/atlas/repository/impexp/ExportService.java
+7
-0
ExportTypeProcessor.java
...g/apache/atlas/repository/impexp/ExportTypeProcessor.java
+9
-0
ImportTypeDefProcessor.java
...pache/atlas/repository/impexp/ImportTypeDefProcessor.java
+6
-0
TypeAttributeDifference.java
...ache/atlas/repository/impexp/TypeAttributeDifference.java
+12
-0
TypeAttributeDifferenceTest.java
.../atlas/repository/impexp/TypeAttributeDifferenceTest.java
+23
-1
typesDef-bm.json
repository/src/test/resources/json/typesDef-bm.json
+165
-0
No files found.
repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
View file @
801d40c3
...
...
@@ -105,7 +105,7 @@ public class AtlasServerService {
AtlasObjectId
objectId
=
getObjectId
(
server
);
for
(
String
guid
:
entityGuids
)
{
AtlasEntity
.
AtlasEntityWithExtInfo
entityWithExtInfo
=
entityStore
.
getById
(
guid
,
false
,
tru
e
);
AtlasEntity
.
AtlasEntityWithExtInfo
entityWithExtInfo
=
entityStore
.
getById
(
guid
,
false
,
fals
e
);
updateAttribute
(
entityWithExtInfo
,
attributeName
,
objectId
);
}
}
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
View file @
801d40c3
...
...
@@ -25,6 +25,7 @@ import org.apache.atlas.model.impexp.AtlasExportResult;
import
org.apache.atlas.model.instance.AtlasEntity
;
import
org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo
;
import
org.apache.atlas.model.instance.AtlasObjectId
;
import
org.apache.atlas.model.typedef.AtlasBusinessMetadataDef
;
import
org.apache.atlas.model.typedef.AtlasClassificationDef
;
import
org.apache.atlas.model.typedef.AtlasEntityDef
;
import
org.apache.atlas.model.typedef.AtlasEnumDef
;
...
...
@@ -163,6 +164,12 @@ public class ExportService {
typesDef
.
getRelationshipDefs
().
add
(
relationshipDef
);
}
for
(
String
bm
:
context
.
businessMetadataTypes
)
{
AtlasBusinessMetadataDef
bmDef
=
typeRegistry
.
getBusinessMetadataDefByName
(
bm
);
typesDef
.
getBusinessMetadataDefs
().
add
(
bmDef
);
}
}
private
AtlasExportResult
.
OperationStatus
[]
processItems
(
AtlasExportRequest
request
,
ExportContext
context
)
{
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
View file @
801d40c3
...
...
@@ -121,6 +121,7 @@ class ExportTypeProcessor {
addAttributeTypes
(
entityType
,
context
);
addRelationshipTypes
(
entityType
,
context
);
addBusinessMetadataType
(
entityType
,
context
);
if
(
CollectionUtils
.
isNotEmpty
(
entityType
.
getAllSuperTypes
()))
{
for
(
String
superType
:
entityType
.
getAllSuperTypes
())
{
...
...
@@ -180,6 +181,14 @@ class ExportTypeProcessor {
}
}
private
void
addBusinessMetadataType
(
AtlasEntityType
entityType
,
ExportService
.
ExportContext
context
)
{
for
(
String
bmTypeName
:
entityType
.
getBusinessAttributes
().
keySet
())
{
AtlasBusinessMetadataType
bmType
=
typeRegistry
.
getBusinessMetadataTypeByName
(
bmTypeName
);
addBusinessMetadataType
(
bmType
,
context
);
}
}
private
void
addAttributeTypes
(
AtlasStructType
structType
,
ExportService
.
ExportContext
context
)
{
for
(
AtlasStructDef
.
AtlasAttributeDef
attributeDef
:
structType
.
getStructDef
().
getAttributeDefs
())
{
addType
(
attributeDef
.
getTypeName
(),
context
);
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java
View file @
801d40c3
...
...
@@ -19,6 +19,7 @@ package org.apache.atlas.repository.impexp;
import
org.apache.atlas.exception.AtlasBaseException
;
import
org.apache.atlas.model.impexp.AtlasImportResult
;
import
org.apache.atlas.model.typedef.AtlasBusinessMetadataDef
;
import
org.apache.atlas.model.typedef.AtlasClassificationDef
;
import
org.apache.atlas.model.typedef.AtlasEntityDef
;
import
org.apache.atlas.model.typedef.AtlasEnumDef
;
...
...
@@ -72,6 +73,10 @@ public class ImportTypeDefProcessor {
for
(
AtlasRelationshipDef
def
:
typesDef
.
getRelationshipDefs
())
{
def
.
setGuid
(
null
);
}
for
(
AtlasBusinessMetadataDef
def
:
typesDef
.
getBusinessMetadataDefs
())
{
def
.
setGuid
(
null
);
}
}
private
void
updateMetricsForTypesDef
(
AtlasTypesDef
typeDefinitionMap
,
AtlasImportResult
result
)
{
...
...
@@ -80,5 +85,6 @@ public class ImportTypeDefProcessor {
result
.
incrementMeticsCounter
(
"typedef:entitydef"
,
typeDefinitionMap
.
getEntityDefs
().
size
());
result
.
incrementMeticsCounter
(
"typedef:struct"
,
typeDefinitionMap
.
getStructDefs
().
size
());
result
.
incrementMeticsCounter
(
"typedef:relationship"
,
typeDefinitionMap
.
getRelationshipDefs
().
size
());
result
.
incrementMeticsCounter
(
"typedef:businessmetadata"
,
typeDefinitionMap
.
getBusinessMetadataDefs
().
size
());
}
}
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
View file @
801d40c3
...
...
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import
org.apache.atlas.AtlasErrorCode
;
import
org.apache.atlas.exception.AtlasBaseException
;
import
org.apache.atlas.model.impexp.AtlasImportResult
;
import
org.apache.atlas.model.typedef.AtlasBusinessMetadataDef
;
import
org.apache.atlas.model.typedef.AtlasClassificationDef
;
import
org.apache.atlas.model.typedef.AtlasEntityDef
;
import
org.apache.atlas.model.typedef.AtlasEnumDef
;
...
...
@@ -54,6 +55,7 @@ public class TypeAttributeDifference {
updateClassificationDef
(
typeDefinitionMap
.
getClassificationDefs
(),
result
);
updateEntityDef
(
typeDefinitionMap
.
getEntityDefs
(),
result
);
updateRelationshipDefs
(
typeDefinitionMap
.
getRelationshipDefs
(),
result
);
updateBusinessMetadataDefs
(
typeDefinitionMap
.
getBusinessMetadataDefs
(),
result
);
}
private
void
updateEntityDef
(
List
<
AtlasEntityDef
>
entityDefs
,
AtlasImportResult
result
)
throws
AtlasBaseException
{
...
...
@@ -108,6 +110,16 @@ public class TypeAttributeDifference {
}
}
private
void
updateBusinessMetadataDefs
(
List
<
AtlasBusinessMetadataDef
>
businessMetadataDefs
,
AtlasImportResult
result
)
throws
AtlasBaseException
{
for
(
AtlasBusinessMetadataDef
def
:
businessMetadataDefs
)
{
AtlasBusinessMetadataDef
existing
=
typeRegistry
.
getBusinessMetadataDefByName
(
def
.
getName
());
if
(
existing
!=
null
&&
addAttributes
(
existing
,
def
))
{
typeDefStore
.
updateStructDefByName
(
existing
.
getName
(),
existing
);
result
.
incrementMeticsCounter
(
"typedef:businessmetadatadef:update"
);
}
}
}
@VisibleForTesting
boolean
addElements
(
AtlasEnumDef
existing
,
AtlasEnumDef
incoming
)
throws
AtlasBaseException
{
return
addElements
(
existing
,
getElementsAbsentInExisting
(
existing
,
incoming
));
...
...
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
View file @
801d40c3
...
...
@@ -17,22 +17,33 @@
*/
package
org
.
apache
.
atlas
.
repository
.
impexp
;
import
org.apache.atlas.TestUtilsV2
;
import
org.apache.atlas.exception.AtlasBaseException
;
import
org.apache.atlas.model.impexp.AtlasImportResult
;
import
org.apache.atlas.model.typedef.AtlasBaseTypeDef
;
import
org.apache.atlas.model.typedef.AtlasEntityDef
;
import
org.apache.atlas.model.typedef.AtlasEnumDef
;
import
org.apache.atlas.model.typedef.AtlasStructDef
;
import
org.apache.atlas.model.typedef.AtlasTypesDef
;
import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer
;
import
org.apache.atlas.store.AtlasTypeDefStore
;
import
org.apache.atlas.type.AtlasTypeRegistry
;
import
org.apache.atlas.utils.TestLoadModelUtils
;
import
org.apache.atlas.utils.TestResourceFileUtils
;
import
org.mockito.Mock
;
import
org.mockito.Mockito
;
import
org.mockito.MockitoAnnotations
;
import
org.testng.Assert
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.Test
;
import
java.io.IOException
;
import
java.util.ArrayList
;
import
java.util.HashMap
;
import
java.util.List
;
import
static
org
.
testng
.
Assert
.
assertEquals
;
import
static
org
.
testng
.
Assert
.
assertNotNull
;
public
class
TypeAttributeDifferenceTest
{
private
TypeAttributeDifference
typeAttributeDifference
;
...
...
@@ -161,4 +172,15 @@ public class TypeAttributeDifferenceTest {
private
boolean
invokeUpdate
(
AtlasEnumDef
existing
,
AtlasEnumDef
incoming
)
throws
Exception
{
return
typeAttributeDifference
.
addElements
(
existing
,
incoming
);
}
@Test
public
void
t1
()
throws
IOException
,
AtlasBaseException
{
AtlasTypesDef
typesDef
=
TestResourceFileUtils
.
readObjectFromJson
(
"."
,
"typesDef-bm"
,
AtlasTypesDef
.
class
);
AtlasImportResult
result
=
new
AtlasImportResult
();
AtlasTypesDef
typesToCreate
=
AtlasTypeDefStoreInitializer
.
getTypesToCreate
(
typesDef
,
this
.
typeRegistry
);
typeDefStore
.
createTypesDef
(
typesToCreate
);
typeAttributeDifference
.
updateTypes
(
typesDef
,
result
);
assertNotNull
(
typesDef
);
}
}
This diff is collapsed.
Click to expand it.
repository/src/test/resources/json/typesDef-bm.json
0 → 100644
View file @
801d40c3
{
"enumDefs"
:
[],
"structDefs"
:
[],
"classificationDefs"
:
[],
"entityDefs"
:
[
{
"category"
:
"ENTITY"
,
"name"
:
"rdbms_table"
,
"typeVersion"
:
"1.0"
,
"attributeDefs"
:
[
{
"name"
:
"name"
,
"typeName"
:
"string"
,
"isOptional"
:
false
,
"cardinality"
:
"SINGLE"
,
"valuesMinCount"
:
-1
,
"valuesMaxCount"
:
-1
,
"isUnique"
:
false
,
"isIndexable"
:
false
}
],
"superTypes"
:
[]
},
{
"category"
:
"ENTITY"
,
"name"
:
"rdbms_storage"
,
"typeVersion"
:
"1.0"
,
"attributeDefs"
:
[
{
"name"
:
"name"
,
"typeName"
:
"string"
,
"isOptional"
:
false
,
"cardinality"
:
"SINGLE"
,
"valuesMinCount"
:
-1
,
"valuesMaxCount"
:
-1
,
"isUnique"
:
false
,
"isIndexable"
:
false
}
],
"superTypes"
:
[]
},
{
"category"
:
"ENTITY"
,
"name"
:
"rdbms_db"
,
"typeVersion"
:
"1.0"
,
"attributeDefs"
:
[
{
"name"
:
"name"
,
"typeName"
:
"string"
,
"isOptional"
:
false
,
"cardinality"
:
"SINGLE"
,
"valuesMinCount"
:
-1
,
"valuesMaxCount"
:
-1
,
"isUnique"
:
true
,
"isIndexable"
:
true
},
{
"name"
:
"sd"
,
"typeName"
:
"rdbms_storage"
,
"isOptional"
:
true
,
"cardinality"
:
"SINGLE"
,
"valuesMinCount"
:
-1
,
"valuesMaxCount"
:
-1
,
"isUnique"
:
false
,
"isIndexable"
:
false
,
"constraints"
:
[
{
"type"
:
"ownedRef"
}
],
"options"
:
{
"isSoftReference"
:
"true"
}
},
{
"name"
:
"tables"
,
"typeName"
:
"array<rdbms_table>"
,
"isOptional"
:
true
,
"cardinality"
:
"SINGLE"
,
"valuesMinCount"
:
-1
,
"valuesMaxCount"
:
-1
,
"isUnique"
:
false
,
"isIndexable"
:
false
,
"constraints"
:
[
{
"type"
:
"ownedRef"
}
],
"options"
:
{
"isSoftReference"
:
"true"
}
},
{
"name"
:
"regions"
,
"typeName"
:
"map<string,rdbms_table>"
,
"isOptional"
:
true
,
"cardinality"
:
"SINGLE"
,
"valuesMinCount"
:
-1
,
"valuesMaxCount"
:
-1
,
"isUnique"
:
false
,
"isIndexable"
:
false
,
"constraints"
:
[
{
"type"
:
"ownedRef"
}
],
"options"
:
{
"isSoftReference"
:
"true"
}
}
],
"superTypes"
:
[],
"businessAttributeDefs"
:
{
"B1"
:
[{
"name"
:
"A1"
,
"typeName"
:
"string"
,
"isOptional"
:
true
,
"cardinality"
:
"SINGLE"
,
"valuesMinCount"
:
0
,
"valuesMaxCount"
:
1
,
"isUnique"
:
false
,
"isIndexable"
:
true
,
"includeInNotification"
:
false
,
"searchWeight"
:
5
,
"options"
:
{
"applicableEntityTypes"
:
"[
\"
hive_db
\"
]"
,
"maxStrLength"
:
"50"
}
}]
}
}
],
"businessMetadataDefs"
:
[
{
"category"
:
"BUSINESS_METADATA"
,
"guid"
:
"a0d52995-9ef7-41e7-bfc0-652e0cf3587c"
,
"createdBy"
:
"admin"
,
"updatedBy"
:
"admin"
,
"createTime"
:
1603487492875
,
"updateTime"
:
1603487515791
,
"version"
:
2
,
"name"
:
"B1"
,
"description"
:
"Test B1"
,
"typeVersion"
:
"1.1"
,
"attributeDefs"
:
[
{
"name"
:
"A1"
,
"typeName"
:
"string"
,
"isOptional"
:
true
,
"cardinality"
:
"SINGLE"
,
"valuesMinCount"
:
0
,
"valuesMaxCount"
:
1
,
"isUnique"
:
false
,
"isIndexable"
:
true
,
"includeInNotification"
:
false
,
"searchWeight"
:
5
,
"options"
:
{
"applicableEntityTypes"
:
"[
\"
rdbms_table
\"
]"
,
"maxStrLength"
:
"50"
}
}
]
}
]
}
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment