Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
74bfe947
Commit
74bfe947
authored
6 years ago
by
Mandar Ambawane
Committed by
Sarath Subramanian
6 years ago
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-3335 Update Sqoop/Storm hook to use relationship attributes
Signed-off-by:
Sarath Subramanian
<
sarath@apache.org
>
parent
709db096
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
17 additions
and
9 deletions
+17
-9
SqoopHook.java
.../src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
+9
-5
StormAtlasHook.java
...main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
+8
-4
No files found.
addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
View file @
74bfe947
...
...
@@ -70,6 +70,10 @@ public class SqoopHook extends SqoopJobDataPublisher {
public
static
final
String
OUTPUTS
=
"outputs"
;
public
static
final
String
ATTRIBUTE_DB
=
"db"
;
public
static
final
String
RELATIONSHIP_HIVE_TABLE_DB
=
"hive_table_db"
;
public
static
final
String
RELATIONSHIP_DATASET_PROCESS_INPUTS
=
"dataset_process_inputs"
;
public
static
final
String
RELATIONSHIP_PROCESS_DATASET_OUTPUTS
=
"process_dataset_outputs"
;
private
static
final
AtlasHookImpl
atlasHook
;
static
{
...
...
@@ -129,7 +133,7 @@ public class SqoopHook extends SqoopJobDataPublisher {
entHiveTable
.
setAttribute
(
AtlasClient
.
NAME
,
tableName
.
toLowerCase
());
entHiveTable
.
setAttribute
(
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
qualifiedName
);
entHiveTable
.
set
Attribute
(
ATTRIBUTE_DB
,
AtlasTypeUtil
.
getAtlasObjectId
(
entHiveDb
));
entHiveTable
.
set
RelationshipAttribute
(
ATTRIBUTE_DB
,
AtlasTypeUtil
.
getAtlasRelatedObjectId
(
entHiveDb
,
RELATIONSHIP_HIVE_TABLE_DB
));
return
entHiveTable
;
}
...
...
@@ -179,11 +183,11 @@ public class SqoopHook extends SqoopJobDataPublisher {
List
<
AtlasObjectId
>
hiveObjects
=
Collections
.
singletonList
(
AtlasTypeUtil
.
getAtlasObjectId
(
entHiveTable
!=
null
?
entHiveTable
:
entHiveDb
));
if
(
isImportOperation
(
data
))
{
entProcess
.
set
Attribute
(
SqoopHook
.
INPUTS
,
sqoopObjects
);
entProcess
.
set
Attribute
(
SqoopHook
.
OUTPUTS
,
hiveObjects
);
entProcess
.
set
RelationshipAttribute
(
SqoopHook
.
INPUTS
,
AtlasTypeUtil
.
getAtlasRelatedObjectIdList
(
sqoopObjects
,
RELATIONSHIP_DATASET_PROCESS_INPUTS
)
);
entProcess
.
set
RelationshipAttribute
(
SqoopHook
.
OUTPUTS
,
AtlasTypeUtil
.
getAtlasRelatedObjectIdList
(
hiveObjects
,
RELATIONSHIP_PROCESS_DATASET_OUTPUTS
)
);
}
else
{
entProcess
.
set
Attribute
(
SqoopHook
.
INPUTS
,
hiveObjects
);
entProcess
.
set
Attribute
(
SqoopHook
.
OUTPUTS
,
sqoopObjects
);
entProcess
.
set
RelationshipAttribute
(
SqoopHook
.
INPUTS
,
AtlasTypeUtil
.
getAtlasRelatedObjectIdList
(
hiveObjects
,
RELATIONSHIP_DATASET_PROCESS_INPUTS
)
);
entProcess
.
set
RelationshipAttribute
(
SqoopHook
.
OUTPUTS
,
AtlasTypeUtil
.
getAtlasRelatedObjectIdList
(
sqoopObjects
,
RELATIONSHIP_PROCESS_DATASET_OUTPUTS
)
);
}
entProcess
.
setAttribute
(
SqoopHook
.
USER
,
data
.
getUser
());
...
...
This diff is collapsed.
Click to expand it.
addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
View file @
74bfe947
...
...
@@ -67,6 +67,10 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
public
static
final
String
HBASE_NAMESPACE_DEFAULT
=
"default"
;
public
static
final
String
ATTRIBUTE_DB
=
"db"
;
public
static
final
String
RELATIONSHIP_STORM_TOPOLOGY_NODES
=
"storm_topology_nodes"
;
public
static
final
String
RELATIONSHIP_DATASET_PROCESS_INPUTS
=
"dataset_process_inputs"
;
public
static
final
String
RELATIONSHIP_PROCESS_DATASET_OUTPUTS
=
"process_dataset_outputs"
;
/**
* This is the client-side hook that storm fires when a topology is added.
*
...
...
@@ -90,7 +94,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
if
(
CollectionUtils
.
isNotEmpty
(
graphNodes
))
{
// add the connection from topology to the graph
topology
.
set
Attribute
(
"nodes"
,
AtlasTypeUtil
.
getAtlasObjectIds
(
graphNodes
));
topology
.
set
RelationshipAttribute
(
"nodes"
,
AtlasTypeUtil
.
getAtlasRelatedObjectIds
(
graphNodes
,
RELATIONSHIP_STORM_TOPOLOGY_NODES
));
for
(
AtlasEntity
graphNode
:
graphNodes
)
{
entity
.
addReferredEntity
(
graphNode
);
...
...
@@ -144,7 +148,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
}
}
topology
.
set
Attribute
(
"inputs"
,
AtlasTypeUtil
.
getAtlasObjectIds
(
inputs
));
topology
.
set
RelationshipAttribute
(
"inputs"
,
AtlasTypeUtil
.
getAtlasRelatedObjectIds
(
inputs
,
RELATIONSHIP_DATASET_PROCESS_INPUTS
));
}
private
void
addTopologyOutputs
(
StormTopology
stormTopology
,
String
topologyOwner
,
Map
stormConf
,
AtlasEntity
topology
,
AtlasEntityExtInfo
entityExtInfo
)
{
...
...
@@ -162,7 +166,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
}
}
topology
.
set
Attribute
(
"outputs"
,
AtlasTypeUtil
.
getAtlasObjectIds
(
outputs
));
topology
.
set
RelationshipAttribute
(
"outputs"
,
AtlasTypeUtil
.
getAtlasRelatedObjectIds
(
outputs
,
RELATIONSHIP_PROCESS_DATASET_OUTPUTS
));
}
private
AtlasEntity
addDataSet
(
String
dataSetType
,
String
topologyOwner
,
Serializable
instance
,
Map
stormConf
,
AtlasEntityExtInfo
entityExtInfo
)
{
...
...
@@ -272,7 +276,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
ret
=
new
AtlasEntity
(
"hive_table"
);
ret
.
setAttribute
(
AtlasClient
.
NAME
,
tblName
);
ret
.
set
Attribute
(
ATTRIBUTE_DB
,
AtlasTypeUtil
.
getAtlasObjectId
(
dbEntity
));
ret
.
set
RelationshipAttribute
(
ATTRIBUTE_DB
,
AtlasTypeUtil
.
getAtlasRelatedObjectId
(
dbEntity
,
"hive_table_db"
));
ret
.
setAttribute
(
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
HiveMetaStoreBridge
.
getTableQualifiedName
(
metadataNamespace
,
dbName
,
tblName
));
}
}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment