Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
9735643c
Commit
9735643c
authored
6 years ago
by
Ashutosh Mestry
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-3046: Classification Updater tool. Unique name used.
parent
b7dcfc60
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
59 additions
and
16 deletions
+59
-16
BulkFetchAndUpdate.java
.../main/java/org/apache/atlas/tools/BulkFetchAndUpdate.java
+59
-16
No files found.
tools/classification-updater/src/main/java/org/apache/atlas/tools/BulkFetchAndUpdate.java
View file @
9735643c
...
...
@@ -28,6 +28,8 @@ import org.apache.atlas.model.instance.AtlasClassification;
import
org.apache.atlas.model.instance.AtlasEntityHeader
;
import
org.apache.atlas.model.instance.AtlasEntityHeaders
;
import
org.apache.atlas.model.typedef.AtlasClassificationDef
;
import
org.apache.atlas.model.typedef.AtlasEntityDef
;
import
org.apache.atlas.model.typedef.AtlasStructDef
;
import
org.apache.atlas.model.typedef.AtlasTypesDef
;
import
org.apache.atlas.type.AtlasType
;
import
org.apache.atlas.utils.AtlasJson
;
...
...
@@ -300,6 +302,7 @@ public class BulkFetchAndUpdate {
private
static
final
String
ATTR_NAME_QUALIFIED_NAME
=
"qualifiedName"
;
private
AtlasClientV2
atlasClientV2
;
private
Map
<
String
,
String
>
typeNameUniqueAttributeNameMap
=
new
HashMap
<>();
public
Preparer
(
AtlasClientV2
atlasClientV2
)
{
this
.
atlasClientV2
=
atlasClientV2
;
...
...
@@ -322,7 +325,7 @@ public class BulkFetchAndUpdate {
try
{
classificationDef
.
setGuid
(
null
);
String
json
=
AtlasType
.
toJson
(
classificationDef
);
fileWriter
.
write
(
json
+
"\n"
);
fileWriter
.
write
(
json
);
}
catch
(
Exception
e
)
{
LOG
.
error
(
"Error writing classifications: {}"
,
e
);
displayCrLf
(
"Error writing classifications."
);
...
...
@@ -348,11 +351,11 @@ public class BulkFetchAndUpdate {
}
try
{
AtlasEntityHeaders
response
=
atlasClientV2
.
getEntityHeaders
(
fromTimestamp
);
int
guidHeaderMapSize
=
response
.
getGuidHeaderMap
().
size
();
AtlasEntityHeaders
entityHeaders
=
atlasClientV2
.
getEntityHeaders
(
fromTimestamp
);
int
guidHeaderMapSize
=
entityHeaders
.
getGuidHeaderMap
().
size
();
try
{
displayCrLf
(
"Read entities: "
+
guidHeaderMapSize
);
AtlasEntityHeaders
updatedHeaders
=
removeEntityGuids
(
response
);
AtlasEntityHeaders
updatedHeaders
=
removeEntityGuids
(
entityHeaders
);
fileWriter
.
write
(
AtlasType
.
toJson
(
updatedHeaders
));
displayCrLf
(
"Writing entities: "
+
updatedHeaders
.
getGuidHeaderMap
().
size
());
...
...
@@ -368,12 +371,17 @@ public class BulkFetchAndUpdate {
}
private
AtlasEntityHeaders
removeEntityGuids
(
AtlasEntityHeaders
headers
)
{
Map
<
String
,
AtlasEntityHeader
>
qualifiedName
HeaderMap
=
new
HashMap
<>();
Map
<
String
,
AtlasEntityHeader
>
uniqueNameEntity
HeaderMap
=
new
HashMap
<>();
for
(
AtlasEntityHeader
header
:
headers
.
getGuidHeaderMap
().
values
())
{
String
qualifiedName
=
getQualifiedName
(
header
);
displayCrLf
(
"Processing: "
+
qualifiedName
);
String
uniqueName
=
getUniqueName
(
header
);
if
(
StringUtils
.
isEmpty
(
uniqueName
))
{
displayCrLf
(
"UniqueName is empty. Ignoring: "
+
header
.
getGuid
());
LOG
.
warn
(
"UniqueName is empty. Ignoring: {}"
,
AtlasJson
.
toJson
(
header
));
continue
;
}
displayCrLf
(
"Processing: "
+
uniqueName
);
if
(
header
.
getStatus
()
==
DELETED
)
{
continue
;
}
...
...
@@ -383,12 +391,22 @@ public class BulkFetchAndUpdate {
continue
;
}
boolean
keyFound
=
qualifiedNameHeaderMap
.
containsKey
(
qualifiedName
);
String
key
=
String
.
format
(
"%s:%s"
,
header
.
getTypeName
(),
uniqueName
);
boolean
keyFound
=
uniqueNameEntityHeaderMap
.
containsKey
(
key
);
if
(!
keyFound
)
{
qualifiedNameHeaderMap
.
put
(
qualifiedName
,
header
);
uniqueNameEntityHeaderMap
.
put
(
key
,
header
);
}
updateClassificationsForHeader
(
header
,
uniqueNameEntityHeaderMap
.
get
(
key
),
keyFound
);
displayCrLf
(
"Processing: "
+
uniqueName
);
}
displayCrLf
(
"Processed: "
+
uniqueNameEntityHeaderMap
.
size
());
headers
.
setGuidHeaderMap
(
uniqueNameEntityHeaderMap
);
return
headers
;
}
AtlasEntityHeader
currentHeader
=
qualifiedNameHeaderMap
.
get
(
qualifiedName
);
private
void
updateClassificationsForHeader
(
AtlasEntityHeader
header
,
AtlasEntityHeader
currentHeader
,
boolean
keyFound
)
{
for
(
AtlasClassification
c
:
header
.
getClassifications
())
{
c
.
setEntityGuid
(
null
);
...
...
@@ -403,17 +421,42 @@ public class BulkFetchAndUpdate {
}
}
}
}
displayCrLf
(
"Processing: "
+
qualifiedName
);
private
String
getUniqueName
(
AtlasEntityHeader
header
)
{
String
uniqueAttributeName
=
ATTR_NAME_QUALIFIED_NAME
;
if
(!
header
.
getAttributes
().
containsKey
(
ATTR_NAME_QUALIFIED_NAME
))
{
uniqueAttributeName
=
getUniqueAttribute
(
header
.
getTypeName
());
}
displayCrLf
(
"Processed: "
+
qualifiedNameHeaderMap
.
size
());
headers
.
setGuidHeaderMap
(
qualifiedNameHeaderMap
);
return
headers
;
Object
attrValue
=
header
.
getAttribute
(
uniqueAttributeName
);
if
(
attrValue
==
null
)
{
LOG
.
warn
(
"Unique Attribute Value: empty: {}"
,
AtlasJson
.
toJson
(
header
));
return
StringUtils
.
EMPTY
;
}
return
attrValue
.
toString
();
}
private
String
getUniqueAttribute
(
String
typeName
)
{
try
{
if
(
typeNameUniqueAttributeNameMap
.
containsKey
(
typeName
))
{
return
typeNameUniqueAttributeNameMap
.
get
(
typeName
);
}
AtlasEntityDef
entityDef
=
atlasClientV2
.
getEntityDefByName
(
typeName
);
for
(
AtlasStructDef
.
AtlasAttributeDef
ad
:
entityDef
.
getAttributeDefs
())
{
if
(
ad
.
getIsUnique
())
{
typeNameUniqueAttributeNameMap
.
put
(
typeName
,
ad
.
getName
());
return
ad
.
getName
();
}
}
}
catch
(
AtlasServiceException
e
)
{
LOG
.
error
(
"Error fetching type: {}"
,
typeName
,
e
);
return
null
;
}
private
String
getQualifiedName
(
AtlasEntityHeader
header
)
{
return
(
String
)
header
.
getAttribute
(
ATTR_NAME_QUALIFIED_NAME
);
return
null
;
}
private
List
<
AtlasClassificationDef
>
getAllClassificationsDefs
()
throws
Exception
{
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment