Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
3f51160f
Commit
3f51160f
authored
Aug 05, 2016
by
Shwetha GS
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-1096 Modify HveMetaStoreBridge.import to use getEntity instead of DSL (sumasai via shwethags)
parent
38fd4f35
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
67 additions
and
72 deletions
+67
-72
HiveMetaStoreBridge.java
...ava/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+24
-44
HiveMetaStoreBridgeTest.java
...org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
+36
-28
HiveMetastoreBridgeIT.java
...a/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java
+6
-0
release-log.txt
release-log.txt
+1
-0
No files found.
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
View file @
3f51160f
...
@@ -54,8 +54,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
...
@@ -54,8 +54,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import
org.apache.hadoop.hive.ql.metadata.Table
;
import
org.apache.hadoop.hive.ql.metadata.Table
;
import
org.apache.hadoop.hive.ql.session.SessionState
;
import
org.apache.hadoop.hive.ql.session.SessionState
;
import
org.apache.hadoop.security.UserGroupInformation
;
import
org.apache.hadoop.security.UserGroupInformation
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONObject
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
...
@@ -72,7 +70,6 @@ public class HiveMetaStoreBridge {
...
@@ -72,7 +70,6 @@ public class HiveMetaStoreBridge {
public
static
final
String
HIVE_CLUSTER_NAME
=
"atlas.cluster.name"
;
public
static
final
String
HIVE_CLUSTER_NAME
=
"atlas.cluster.name"
;
public
static
final
String
DEFAULT_CLUSTER_NAME
=
"primary"
;
public
static
final
String
DEFAULT_CLUSTER_NAME
=
"primary"
;
public
static
final
String
DESCRIPTION_ATTR
=
"description"
;
public
static
final
String
DESCRIPTION_ATTR
=
"description"
;
public
static
final
String
SEARCH_ENTRY_GUID_ATTR
=
"__guid"
;
public
static
final
String
TEMP_TABLE_PREFIX
=
"_temp-"
;
public
static
final
String
TEMP_TABLE_PREFIX
=
"_temp-"
;
...
@@ -214,30 +211,7 @@ public class HiveMetaStoreBridge {
...
@@ -214,30 +211,7 @@ public class HiveMetaStoreBridge {
LOG
.
debug
(
"Getting reference for database {}"
,
databaseName
);
LOG
.
debug
(
"Getting reference for database {}"
,
databaseName
);
String
typeName
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
typeName
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
dslQuery
=
getDatabaseDSLQuery
(
clusterName
,
databaseName
,
typeName
);
return
getEntityReference
(
typeName
,
getDBQualifiedName
(
clusterName
,
databaseName
));
return
getEntityReferenceFromDSL
(
typeName
,
dslQuery
);
}
static
String
getDatabaseDSLQuery
(
String
clusterName
,
String
databaseName
,
String
typeName
)
{
return
String
.
format
(
"%s where %s = '%s' and %s = '%s'"
,
typeName
,
AtlasClient
.
NAME
,
databaseName
.
toLowerCase
(),
AtlasConstants
.
CLUSTER_NAME_ATTRIBUTE
,
clusterName
);
}
private
Referenceable
getEntityReferenceFromDSL
(
String
typeName
,
String
dslQuery
)
throws
Exception
{
AtlasClient
dgiClient
=
getAtlasClient
();
JSONArray
results
=
dgiClient
.
searchByDSL
(
dslQuery
,
1
,
0
);
if
(
results
.
length
()
==
0
)
{
return
null
;
}
else
{
String
guid
;
JSONObject
row
=
results
.
getJSONObject
(
0
);
if
(
row
.
has
(
"$id$"
))
{
guid
=
row
.
getJSONObject
(
"$id$"
).
getString
(
"id"
);
}
else
{
guid
=
row
.
getJSONObject
(
"_col_0"
).
getString
(
"id"
);
}
return
new
Referenceable
(
guid
,
typeName
,
null
);
}
}
}
/**
/**
...
@@ -253,11 +227,16 @@ public class HiveMetaStoreBridge {
...
@@ -253,11 +227,16 @@ public class HiveMetaStoreBridge {
private
String
getCreateTableString
(
Table
table
,
String
location
){
private
String
getCreateTableString
(
Table
table
,
String
location
){
String
colString
=
""
;
String
colString
=
""
;
List
<
FieldSchema
>
colList
=
table
.
getAllCols
();
List
<
FieldSchema
>
colList
=
table
.
getAllCols
();
for
(
FieldSchema
col:
colList
){
if
(
colList
!=
null
)
{
for
(
FieldSchema
col
:
colList
)
{
colString
+=
col
.
getName
()
+
" "
+
col
.
getType
()
+
","
;
colString
+=
col
.
getName
()
+
" "
+
col
.
getType
()
+
","
;
}
}
if
(
colList
.
size
()
>
0
)
{
colString
=
colString
.
substring
(
0
,
colString
.
length
()
-
1
);
colString
=
colString
.
substring
(
0
,
colString
.
length
()
-
1
);
String
query
=
"create external table "
+
table
.
getTableName
()
+
"("
+
colString
+
")"
+
colString
=
"("
+
colString
+
")"
;
}
}
String
query
=
"create external table "
+
table
.
getTableName
()
+
colString
+
" location '"
+
location
+
"'"
;
" location '"
+
location
+
"'"
;
return
query
;
return
query
;
}
}
...
@@ -293,7 +272,7 @@ public class HiveMetaStoreBridge {
...
@@ -293,7 +272,7 @@ public class HiveMetaStoreBridge {
Table
table
=
hiveClient
.
getTable
(
databaseName
,
tableName
);
Table
table
=
hiveClient
.
getTable
(
databaseName
,
tableName
);
Referenceable
tableReferenceable
=
registerTable
(
databaseReferenceable
,
table
);
Referenceable
tableReferenceable
=
registerTable
(
databaseReferenceable
,
table
);
if
(
table
.
getTableType
()
==
TableType
.
EXTERNAL_TABLE
)
{
if
(
table
.
getTableType
()
==
TableType
.
EXTERNAL_TABLE
)
{
String
tableQualifiedName
=
getTableQualifiedName
(
clusterName
,
table
);
String
tableQualifiedName
=
getTable
Process
QualifiedName
(
clusterName
,
table
);
Referenceable
process
=
getProcessReference
(
tableQualifiedName
);
Referenceable
process
=
getProcessReference
(
tableQualifiedName
);
if
(
process
==
null
)
{
if
(
process
==
null
)
{
LOG
.
info
(
"Attempting to register create table process for {}"
,
tableQualifiedName
);
LOG
.
info
(
"Attempting to register create table process for {}"
,
tableQualifiedName
);
...
@@ -347,25 +326,26 @@ public class HiveMetaStoreBridge {
...
@@ -347,25 +326,26 @@ public class HiveMetaStoreBridge {
LOG
.
debug
(
"Getting reference for table {}.{}"
,
hiveTable
.
getDbName
(),
hiveTable
.
getTableName
());
LOG
.
debug
(
"Getting reference for table {}.{}"
,
hiveTable
.
getDbName
(),
hiveTable
.
getTableName
());
String
typeName
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
String
typeName
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
String
dslQuery
=
getTableDSLQuery
(
getClusterName
(),
hiveTable
.
getDbName
(),
hiveTable
.
getTableName
(),
typeName
,
hiveTable
.
isTemporary
());
String
tblQualifiedName
=
getTableQualifiedName
(
getClusterName
(),
hiveTable
.
getDbName
(),
hiveTable
.
getTableName
());
return
getEntityReference
FromDSL
(
typeName
,
dslQuery
);
return
getEntityReference
(
typeName
,
tblQualifiedName
);
}
}
private
Referenceable
getProcessReference
(
String
qualifiedName
)
throws
Exception
{
private
Referenceable
getEntityReference
(
final
String
typeName
,
final
String
tblQualifiedName
)
throws
AtlasServiceException
{
LOG
.
debug
(
"Getting reference for process {}"
,
qualifiedName
);
AtlasClient
dgiClient
=
getAtlasClient
();
String
typeName
=
HiveDataTypes
.
HIVE_PROCESS
.
getName
();
try
{
String
dslQuery
=
getProcessDSLQuery
(
typeName
,
qualifiedName
);
return
dgiClient
.
getEntity
(
typeName
,
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
tblQualifiedName
);
return
getEntityReferenceFromDSL
(
typeName
,
dslQuery
);
}
catch
(
AtlasServiceException
e
)
{
if
(
e
.
getStatus
()
==
ClientResponse
.
Status
.
NOT_FOUND
)
{
return
null
;
}
throw
e
;
}
}
static
String
getProcessDSLQuery
(
String
typeName
,
String
qualifiedName
)
throws
Exception
{
String
dslQuery
=
String
.
format
(
"%s as t where qualifiedName = '%s'"
,
typeName
,
qualifiedName
);
return
dslQuery
;
}
}
static
String
getTableDSLQuery
(
String
clusterName
,
String
dbName
,
String
tableName
,
String
typeName
,
boolean
isTemporary
)
{
private
Referenceable
getProcessReference
(
String
qualifiedName
)
throws
Exception
{
String
entityName
=
getTableQualifiedName
(
clusterName
,
dbName
,
tableName
,
isTemporary
);
LOG
.
debug
(
"Getting reference for process {}"
,
qualifiedName
);
return
String
.
format
(
"%s as t where qualifiedName = '%s'"
,
typeName
,
entityName
);
String
typeName
=
HiveDataTypes
.
HIVE_PROCESS
.
getName
();
return
getEntityReference
(
typeName
,
qualifiedName
);
}
}
/**
/**
...
...
addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
View file @
3f51160f
...
@@ -23,8 +23,10 @@ import org.apache.atlas.AtlasServiceException;
...
@@ -23,8 +23,10 @@ import org.apache.atlas.AtlasServiceException;
import
org.apache.atlas.hive.model.HiveDataModelGenerator
;
import
org.apache.atlas.hive.model.HiveDataModelGenerator
;
import
org.apache.atlas.hive.model.HiveDataTypes
;
import
org.apache.atlas.hive.model.HiveDataTypes
;
import
org.apache.atlas.typesystem.Referenceable
;
import
org.apache.atlas.typesystem.Referenceable
;
import
org.apache.hadoop.fs.Path
;
import
org.apache.hadoop.hive.metastore.TableType
;
import
org.apache.hadoop.hive.metastore.TableType
;
import
org.apache.hadoop.hive.metastore.api.Database
;
import
org.apache.hadoop.hive.metastore.api.Database
;
import
org.apache.hadoop.hive.metastore.api.FieldSchema
;
import
org.apache.hadoop.hive.ql.metadata.Hive
;
import
org.apache.hadoop.hive.ql.metadata.Hive
;
import
org.apache.hadoop.hive.ql.metadata.HiveException
;
import
org.apache.hadoop.hive.ql.metadata.HiveException
;
import
org.apache.hadoop.hive.ql.metadata.Partition
;
import
org.apache.hadoop.hive.ql.metadata.Partition
;
...
@@ -95,13 +97,13 @@ public class HiveMetaStoreBridgeTest {
...
@@ -95,13 +97,13 @@ public class HiveMetaStoreBridgeTest {
returnExistingDatabase
(
TEST_DB_NAME
,
atlasClient
,
CLUSTER_NAME
);
returnExistingDatabase
(
TEST_DB_NAME
,
atlasClient
,
CLUSTER_NAME
);
// return existing table
// return existing table
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getTableDSLQuery
(
CLUSTER_NAME
,
TEST_DB_NAME
,
TEST_TABLE_NAME
,
when
(
atlasClient
.
getEntity
(
HiveDataTypes
.
HIVE_TABLE
.
getName
()
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
false
),
1
,
0
)).
thenReturn
(
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
TEST_DB_NAME
,
TEST_TABLE_NAME
)))
getEntityReference
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
.
thenReturn
(
getEntityReference
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
when
(
atlasClient
.
getEntity
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
)).
thenReturn
(
createTableReference
());
when
(
atlasClient
.
getEntity
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
)).
thenReturn
(
createTableReference
());
String
processQualifiedName
=
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
hiveTables
.
get
(
0
));
String
processQualifiedName
=
HiveMetaStoreBridge
.
getTable
Process
QualifiedName
(
CLUSTER_NAME
,
hiveTables
.
get
(
0
));
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getProcessDSLQuer
y
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
when
(
atlasClient
.
getEntit
y
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
processQualifiedName
),
1
,
0
)).
thenReturn
(
getEntityReference
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
processQualifiedName
)).
thenReturn
(
getEntityReference
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
HiveMetaStoreBridge
bridge
=
new
HiveMetaStoreBridge
(
CLUSTER_NAME
,
hiveClient
,
atlasClient
);
HiveMetaStoreBridge
bridge
=
new
HiveMetaStoreBridge
(
CLUSTER_NAME
,
hiveClient
,
atlasClient
);
bridge
.
importHiveMetadata
(
true
);
bridge
.
importHiveMetadata
(
true
);
...
@@ -114,9 +116,10 @@ public class HiveMetaStoreBridgeTest {
...
@@ -114,9 +116,10 @@ public class HiveMetaStoreBridgeTest {
private
void
returnExistingDatabase
(
String
databaseName
,
AtlasClient
atlasClient
,
String
clusterName
)
private
void
returnExistingDatabase
(
String
databaseName
,
AtlasClient
atlasClient
,
String
clusterName
)
throws
AtlasServiceException
,
JSONException
{
throws
AtlasServiceException
,
JSONException
{
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getDatabaseDSLQuery
(
clusterName
,
databaseName
,
when
(
atlasClient
.
getEntity
(
HiveDataTypes
.
HIVE_DB
.
getName
()),
1
,
0
)).
thenReturn
(
HiveDataTypes
.
HIVE_DB
.
getName
(),
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
getEntityReference
(
"72e06b34-9151-4023-aa9d-b82103a50e76"
));
HiveMetaStoreBridge
.
getDBQualifiedName
(
clusterName
,
databaseName
))).
thenReturn
(
getEntityReference
(
HiveDataTypes
.
HIVE_DB
.
getName
(),
"72e06b34-9151-4023-aa9d-b82103a50e76"
));
}
}
private
List
<
Table
>
setupTables
(
Hive
hiveClient
,
String
databaseName
,
String
...
tableNames
)
throws
HiveException
{
private
List
<
Table
>
setupTables
(
Hive
hiveClient
,
String
databaseName
,
String
...
tableNames
)
throws
HiveException
{
...
@@ -144,12 +147,12 @@ public class HiveMetaStoreBridgeTest {
...
@@ -144,12 +147,12 @@ public class HiveMetaStoreBridgeTest {
returnExistingDatabase
(
TEST_DB_NAME
,
atlasClient
,
CLUSTER_NAME
);
returnExistingDatabase
(
TEST_DB_NAME
,
atlasClient
,
CLUSTER_NAME
);
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getTableDSLQuery
(
CLUSTER_NAME
,
TEST_DB
_NAME
,
when
(
atlasClient
.
getEntity
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
AtlasClient
.
REFERENCEABLE_ATTRIBUTE
_NAME
,
TEST_TABLE_NAME
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
false
),
1
,
0
)).
thenReturn
(
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
TEST_DB_NAME
,
TEST_TABLE_NAME
)
)).
thenReturn
(
getEntityReference
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
getEntityReference
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
String
processQualifiedName
=
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
hiveTable
);
String
processQualifiedName
=
HiveMetaStoreBridge
.
getTable
Process
QualifiedName
(
CLUSTER_NAME
,
hiveTable
);
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getProcessDSLQuery
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
()
,
when
(
atlasClient
.
getEntity
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
processQualifiedName
)
,
1
,
0
)).
thenReturn
(
getEntityReference
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
processQualifiedName
)
).
thenReturn
(
getEntityReference
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
when
(
atlasClient
.
getEntity
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
)).
thenReturn
(
createTableReference
());
when
(
atlasClient
.
getEntity
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
)).
thenReturn
(
createTableReference
());
Partition
partition
=
mock
(
Partition
.
class
);
Partition
partition
=
mock
(
Partition
.
class
);
...
@@ -176,13 +179,13 @@ public class HiveMetaStoreBridgeTest {
...
@@ -176,13 +179,13 @@ public class HiveMetaStoreBridgeTest {
returnExistingDatabase
(
TEST_DB_NAME
,
atlasClient
,
CLUSTER_NAME
);
returnExistingDatabase
(
TEST_DB_NAME
,
atlasClient
,
CLUSTER_NAME
);
when
(
hiveClient
.
getTable
(
TEST_DB_NAME
,
TEST_TABLE_NAME
)).
thenThrow
(
new
RuntimeException
(
"Timeout while reading data from hive metastore"
));
when
(
hiveClient
.
getTable
(
TEST_DB_NAME
,
TEST_TABLE_NAME
)).
thenThrow
(
new
RuntimeException
(
"Timeout while reading data from hive metastore"
));
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getTableDSLQuery
(
CLUSTER_NAME
,
TEST_DB_NAME
,
when
(
atlasClient
.
getEntity
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
TEST_DB_NAME
,
table2Name
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
false
),
1
,
0
)).
thenReturn
(
table2Name
)
)).
thenReturn
(
getEntityReference
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
getEntityReference
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
when
(
atlasClient
.
getEntity
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
)).
thenReturn
(
createTableReference
());
when
(
atlasClient
.
getEntity
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
)).
thenReturn
(
createTableReference
());
String
processQualifiedName
=
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
hiveTables
.
get
(
1
));
String
processQualifiedName
=
HiveMetaStoreBridge
.
getTable
Process
QualifiedName
(
CLUSTER_NAME
,
hiveTables
.
get
(
1
));
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getProcessDSLQuery
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
()
,
when
(
atlasClient
.
getEntity
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
processQualifiedName
)
,
1
,
0
)).
thenReturn
(
getEntityReference
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
processQualifiedName
)
).
thenReturn
(
getEntityReference
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
HiveMetaStoreBridge
bridge
=
new
HiveMetaStoreBridge
(
CLUSTER_NAME
,
hiveClient
,
atlasClient
);
HiveMetaStoreBridge
bridge
=
new
HiveMetaStoreBridge
(
CLUSTER_NAME
,
hiveClient
,
atlasClient
);
try
{
try
{
...
@@ -201,13 +204,13 @@ public class HiveMetaStoreBridgeTest {
...
@@ -201,13 +204,13 @@ public class HiveMetaStoreBridgeTest {
returnExistingDatabase
(
TEST_DB_NAME
,
atlasClient
,
CLUSTER_NAME
);
returnExistingDatabase
(
TEST_DB_NAME
,
atlasClient
,
CLUSTER_NAME
);
when
(
hiveClient
.
getTable
(
TEST_DB_NAME
,
TEST_TABLE_NAME
)).
thenThrow
(
new
RuntimeException
(
"Timeout while reading data from hive metastore"
));
when
(
hiveClient
.
getTable
(
TEST_DB_NAME
,
TEST_TABLE_NAME
)).
thenThrow
(
new
RuntimeException
(
"Timeout while reading data from hive metastore"
));
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getTableDSLQuery
(
CLUSTER_NAME
,
TEST_DB_NAME
,
when
(
atlasClient
.
getEntity
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
TEST_DB_NAME
,
table2Name
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
false
),
10
,
0
)).
thenReturn
(
table2Name
)
)).
thenReturn
(
getEntityReference
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
getEntityReference
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
when
(
atlasClient
.
getEntity
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
)).
thenReturn
(
createTableReference
());
when
(
atlasClient
.
getEntity
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
)).
thenReturn
(
createTableReference
());
String
processQualifiedName
=
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
hiveTables
.
get
(
1
));
String
processQualifiedName
=
HiveMetaStoreBridge
.
getTableQualifiedName
(
CLUSTER_NAME
,
hiveTables
.
get
(
1
));
when
(
atlasClient
.
searchByDSL
(
HiveMetaStoreBridge
.
getProcessDSLQuery
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
()
,
when
(
atlasClient
.
getEntity
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
processQualifiedName
)
,
10
,
0
)).
thenReturn
(
getEntityReference
(
"82e06b34-9151-4023-aa9d-b82103a50e77"
))
;
processQualifiedName
)
).
thenReturn
(
getEntityReference
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
"82e06b34-9151-4023-aa9d-b82103a50e77"
));
;
HiveMetaStoreBridge
bridge
=
new
HiveMetaStoreBridge
(
CLUSTER_NAME
,
hiveClient
,
atlasClient
);
HiveMetaStoreBridge
bridge
=
new
HiveMetaStoreBridge
(
CLUSTER_NAME
,
hiveClient
,
atlasClient
);
try
{
try
{
...
@@ -218,8 +221,8 @@ public class HiveMetaStoreBridgeTest {
...
@@ -218,8 +221,8 @@ public class HiveMetaStoreBridgeTest {
}
}
}
}
private
JSONArray
getEntityReference
(
String
id
)
throws
JSONException
{
private
Referenceable
getEntityReference
(
String
typeName
,
String
id
)
throws
JSONException
{
return
new
JSONArray
(
String
.
format
(
"[{\"$id$\":{\"id\":\"%s\"}}]"
,
id
)
);
return
new
Referenceable
(
id
,
typeName
,
null
);
}
}
private
Referenceable
createTableReference
()
{
private
Referenceable
createTableReference
()
{
...
@@ -232,7 +235,12 @@ public class HiveMetaStoreBridgeTest {
...
@@ -232,7 +235,12 @@ public class HiveMetaStoreBridgeTest {
private
Table
createTestTable
(
String
databaseName
,
String
tableName
)
throws
HiveException
{
private
Table
createTestTable
(
String
databaseName
,
String
tableName
)
throws
HiveException
{
Table
table
=
new
Table
(
databaseName
,
tableName
);
Table
table
=
new
Table
(
databaseName
,
tableName
);
table
.
setInputFormatClass
(
TextInputFormat
.
class
);
table
.
setInputFormatClass
(
TextInputFormat
.
class
);
table
.
setFields
(
new
ArrayList
<
FieldSchema
>()
{{
add
(
new
FieldSchema
(
"col1"
,
"string"
,
"comment1"
));
}
});
table
.
setTableType
(
TableType
.
EXTERNAL_TABLE
);
table
.
setTableType
(
TableType
.
EXTERNAL_TABLE
);
table
.
setDataLocation
(
new
Path
(
"somehdfspath"
));
return
table
;
return
table
;
}
}
...
...
addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java
View file @
3f51160f
...
@@ -52,6 +52,8 @@ public class HiveMetastoreBridgeIT extends HiveITBase {
...
@@ -52,6 +52,8 @@ public class HiveMetastoreBridgeIT extends HiveITBase {
assertEquals
(
outputs
.
size
(),
1
);
assertEquals
(
outputs
.
size
(),
1
);
assertEquals
(
outputs
.
get
(
0
).
getId
().
_getId
(),
tableId
);
assertEquals
(
outputs
.
get
(
0
).
getId
().
_getId
(),
tableId
);
int
tableCount
=
atlasClient
.
listEntities
(
HiveDataTypes
.
HIVE_TABLE
.
getName
()).
size
();
//Now import using import tool - should be no-op
//Now import using import tool - should be no-op
hiveMetaStoreBridge
.
importTable
(
atlasClient
.
getEntity
(
dbId
),
DEFAULT_DB
,
tableName
,
true
);
hiveMetaStoreBridge
.
importTable
(
atlasClient
.
getEntity
(
dbId
),
DEFAULT_DB
,
tableName
,
true
);
String
tableId2
=
assertTableIsRegistered
(
DEFAULT_DB
,
tableName
);
String
tableId2
=
assertTableIsRegistered
(
DEFAULT_DB
,
tableName
);
...
@@ -61,6 +63,10 @@ public class HiveMetastoreBridgeIT extends HiveITBase {
...
@@ -61,6 +63,10 @@ public class HiveMetastoreBridgeIT extends HiveITBase {
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
AtlasClient
.
REFERENCEABLE_ATTRIBUTE_NAME
,
getTableProcessQualifiedName
(
DEFAULT_DB
,
tableName
),
null
);
getTableProcessQualifiedName
(
DEFAULT_DB
,
tableName
),
null
);
assertEquals
(
processId2
,
processId
);
assertEquals
(
processId2
,
processId
);
//assert that table is de-duped and no new entity is created
int
newTableCount
=
atlasClient
.
listEntities
(
HiveDataTypes
.
HIVE_TABLE
.
getName
()).
size
();
assertEquals
(
newTableCount
,
tableCount
);
}
}
@Test
@Test
...
...
release-log.txt
View file @
3f51160f
...
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
...
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
ALL CHANGES:
ATLAS-1096 Modify HveMetaStoreBridge.import to use getEntity instead of DSL (sumasai via shwethags)
ATLAS-1091 Improvement in DSL search functionality. (kevalbhatt)
ATLAS-1091 Improvement in DSL search functionality. (kevalbhatt)
ATLAS-1080 Regression - UI - hive_storagedesc is shown as "undefined" in UI.(kevalbhatt)
ATLAS-1080 Regression - UI - hive_storagedesc is shown as "undefined" in UI.(kevalbhatt)
ATLAS-1089 Storm hook should handle cyclic references in topology object (mneethiraj via sumasai)
ATLAS-1089 Storm hook should handle cyclic references in topology object (mneethiraj via sumasai)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment