Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
f62ed092
Commit
f62ed092
authored
6 years ago
by
rmani
Committed by
Madhan Neethiraj
6 years ago
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-2649: updated Hive Hook to create lineage between HBase table and Hive table
Signed-off-by:
Madhan Neethiraj
<
madhan@apache.org
>
parent
f15995cc
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
117 additions
and
2 deletions
+117
-2
pom.xml
addons/hive-bridge/pom.xml
+5
-0
BaseHiveEvent.java
...java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+95
-2
CreateTable.java
...n/java/org/apache/atlas/hive/hook/events/CreateTable.java
+17
-0
No files found.
addons/hive-bridge/pom.xml
View file @
f62ed092
...
...
@@ -295,6 +295,11 @@
<version>
${hbase.version}
</version>
</artifactItem>
<artifactItem>
<groupId>
org.apache.hbase
</groupId>
<artifactId>
hbase-server
</artifactId>
<version>
${hbase.version}
</version>
</artifactItem>
<artifactItem>
<groupId>
com.sun.jersey
</groupId>
<artifactId>
jersey-json
</artifactId>
<version>
${jersey.version}
</version>
...
...
This diff is collapsed.
Click to expand it.
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
View file @
f62ed092
...
...
@@ -28,6 +28,7 @@ import org.apache.atlas.model.instance.AtlasStruct;
import
org.apache.atlas.model.notification.HookNotification
;
import
org.apache.atlas.utils.HdfsNameServiceResolver
;
import
org.apache.commons.collections.CollectionUtils
;
import
org.apache.commons.collections.MapUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.hadoop.fs.Path
;
import
org.apache.hadoop.hive.metastore.api.Database
;
...
...
@@ -75,6 +76,8 @@ public abstract class BaseHiveEvent {
public
static
final
String
HIVE_TYPE_SERDE
=
"hive_serde"
;
public
static
final
String
HIVE_TYPE_ORDER
=
"hive_order"
;
public
static
final
String
HDFS_TYPE_PATH
=
"hdfs_path"
;
public
static
final
String
HBASE_TYPE_TABLE
=
"hbase_table"
;
public
static
final
String
HBASE_TYPE_NAMESPACE
=
"hbase_namespace"
;
public
static
final
String
ATTRIBUTE_QUALIFIED_NAME
=
"qualifiedName"
;
public
static
final
String
ATTRIBUTE_NAME
=
"name"
;
...
...
@@ -124,8 +127,14 @@ public abstract class BaseHiveEvent {
public
static
final
String
ATTRIBUTE_DEPENDENCY_TYPE
=
"depenendencyType"
;
public
static
final
String
ATTRIBUTE_EXPRESSION
=
"expression"
;
public
static
final
String
ATTRIBUTE_ALIASES
=
"aliases"
;
public
static
final
String
ATTRIBUTE_URI
=
"uri"
;
public
static
final
String
ATTRIBUTE_STORAGE_HANDLER
=
"storage_handler"
;
public
static
final
String
ATTRIBUTE_NAMESPACE
=
"namespace"
;
public
static
final
String
HBASE_STORAGE_HANDLER_CLASS
=
"org.apache.hadoop.hive.hbase.HBaseStorageHandler"
;
public
static
final
String
HBASE_DEFAULT_NAMESPACE
=
"default"
;
public
static
final
String
HBASE_NAMESPACE_TABLE_DELIMITER
=
":"
;
public
static
final
String
HBASE_PARAM_TABLE_NAME
=
"hbase.table.name"
;
public
static
final
long
MILLIS_CONVERT_FACTOR
=
1000
;
public
static
final
Map
<
Integer
,
String
>
OWNER_TYPE_TO_ENUM_VALUE
=
new
HashMap
<>();
...
...
@@ -156,6 +165,10 @@ public abstract class BaseHiveEvent {
return
table
.
getTTable
()
!=
null
?
(
table
.
getTTable
().
getCreateTime
()
*
MILLIS_CONVERT_FACTOR
)
:
System
.
currentTimeMillis
();
}
public
static
String
getTableOwner
(
Table
table
)
{
return
table
.
getTTable
()
!=
null
?
(
table
.
getOwner
()):
""
;
}
public
static
AtlasObjectId
getObjectId
(
AtlasEntity
entity
)
{
String
qualifiedName
=
(
String
)
entity
.
getAttribute
(
ATTRIBUTE_QUALIFIED_NAME
);
AtlasObjectId
ret
=
new
AtlasObjectId
(
entity
.
getGuid
(),
entity
.
getTypeName
(),
Collections
.
singletonMap
(
ATTRIBUTE_QUALIFIED_NAME
,
qualifiedName
));
...
...
@@ -694,6 +707,53 @@ public abstract class BaseHiveEvent {
return
sb
.
toString
();
}
protected
AtlasEntity
toReferencedHBaseTable
(
Table
table
,
AtlasEntitiesWithExtInfo
entities
)
{
AtlasEntity
ret
=
null
;
HBaseTableInfo
hBaseTableInfo
=
new
HBaseTableInfo
(
table
);
String
hbaseNameSpace
=
hBaseTableInfo
.
getHbaseNameSpace
();
String
hbaseTableName
=
hBaseTableInfo
.
getHbaseTableName
();
if
(
hbaseTableName
!=
null
)
{
AtlasEntity
nsEntity
=
new
AtlasEntity
(
HBASE_TYPE_NAMESPACE
);
nsEntity
.
setAttribute
(
ATTRIBUTE_NAME
,
hbaseNameSpace
);
nsEntity
.
setAttribute
(
ATTRIBUTE_CLUSTER_NAME
,
getClusterName
());
nsEntity
.
setAttribute
(
ATTRIBUTE_QUALIFIED_NAME
,
getHBaseNameSpaceQualifiedName
(
getClusterName
(),
hbaseNameSpace
));
ret
=
new
AtlasEntity
(
HBASE_TYPE_TABLE
);
ret
.
setAttribute
(
ATTRIBUTE_NAME
,
hbaseTableName
);
ret
.
setAttribute
(
ATTRIBUTE_URI
,
hbaseTableName
);
ret
.
setAttribute
(
ATTRIBUTE_NAMESPACE
,
getObjectId
(
nsEntity
));
ret
.
setAttribute
(
ATTRIBUTE_QUALIFIED_NAME
,
getHBaseTableQualifiedName
(
getClusterName
(),
hbaseNameSpace
,
hbaseTableName
));
entities
.
addReferredEntity
(
nsEntity
);
entities
.
addEntity
(
ret
);
}
return
ret
;
}
protected
boolean
isHBaseStore
(
Table
table
)
{
boolean
ret
=
false
;
Map
<
String
,
String
>
parameters
=
table
.
getParameters
();
if
(
MapUtils
.
isNotEmpty
(
parameters
))
{
String
storageHandler
=
parameters
.
get
(
ATTRIBUTE_STORAGE_HANDLER
);
ret
=
(
storageHandler
!=
null
&&
storageHandler
.
equals
(
HBASE_STORAGE_HANDLER_CLASS
));
}
return
ret
;
}
private
static
String
getHBaseTableQualifiedName
(
String
clusterName
,
String
nameSpace
,
String
tableName
)
{
return
String
.
format
(
"%s:%s@%s"
,
nameSpace
.
toLowerCase
(),
tableName
.
toLowerCase
(),
clusterName
);
}
private
static
String
getHBaseNameSpaceQualifiedName
(
String
clusterName
,
String
nameSpace
)
{
return
String
.
format
(
"%s@%s"
,
nameSpace
.
toLowerCase
(),
clusterName
);
}
private
boolean
ignoreHDFSPathsinProcessQualifiedName
()
{
switch
(
context
.
getHiveOperation
())
{
case
LOAD:
...
...
@@ -831,4 +891,37 @@ public abstract class BaseHiveEvent {
}
static
final
Comparator
<
Entity
>
entityComparator
=
new
EntityComparator
();
static
final
class
HBaseTableInfo
{
String
hbaseNameSpace
=
null
;
String
hbaseTableName
=
null
;
HBaseTableInfo
(
Table
table
)
{
Map
<
String
,
String
>
parameters
=
table
.
getParameters
();
if
(
MapUtils
.
isNotEmpty
(
parameters
))
{
hbaseNameSpace
=
HBASE_DEFAULT_NAMESPACE
;
hbaseTableName
=
parameters
.
get
(
HBASE_PARAM_TABLE_NAME
);
if
(
hbaseTableName
!=
null
)
{
if
(
hbaseTableName
.
contains
(
HBASE_NAMESPACE_TABLE_DELIMITER
))
{
String
[]
hbaseTableInfo
=
hbaseTableName
.
split
(
HBASE_NAMESPACE_TABLE_DELIMITER
);
if
(
hbaseTableInfo
.
length
>
1
)
{
hbaseNameSpace
=
hbaseTableInfo
[
0
];
hbaseTableName
=
hbaseTableInfo
[
1
];
}
}
}
}
}
public
String
getHbaseNameSpace
()
{
return
hbaseNameSpace
;
}
public
String
getHbaseTableName
()
{
return
hbaseTableName
;
}
}
}
This diff is collapsed.
Click to expand it.
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
View file @
f62ed092
...
...
@@ -81,6 +81,22 @@ public class CreateTable extends BaseHiveEvent {
if
(
table
!=
null
)
{
AtlasEntity
tblEntity
=
toTableEntity
(
table
,
ret
);
if
(
isHBaseStore
(
table
))
{
// This create lineage to HBase table in case of Hive on HBase
AtlasEntity
hbaseTableEntity
=
toReferencedHBaseTable
(
table
,
ret
);
if
(
hbaseTableEntity
!=
null
)
{
final
AtlasEntity
processEntity
;
if
(
TableType
.
EXTERNAL_TABLE
.
equals
(
table
.
getTableType
()))
{
processEntity
=
getHiveProcessEntity
(
Collections
.
singletonList
(
hbaseTableEntity
),
Collections
.
singletonList
(
tblEntity
));
}
else
{
processEntity
=
getHiveProcessEntity
(
Collections
.
singletonList
(
tblEntity
),
Collections
.
singletonList
(
hbaseTableEntity
));
}
ret
.
addEntity
(
processEntity
);
}
}
else
{
if
(
TableType
.
EXTERNAL_TABLE
.
equals
(
table
.
getTableType
()))
{
AtlasEntity
hdfsPathEntity
=
getHDFSPathEntity
(
table
.
getDataLocation
());
AtlasEntity
processEntity
=
getHiveProcessEntity
(
Collections
.
singletonList
(
hdfsPathEntity
),
Collections
.
singletonList
(
tblEntity
));
...
...
@@ -89,6 +105,7 @@ public class CreateTable extends BaseHiveEvent {
ret
.
addReferredEntity
(
hdfsPathEntity
);
}
}
}
addProcessedEntities
(
ret
);
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment