Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
fee7c5cc
Commit
fee7c5cc
authored
May 30, 2015
by
Shwetha GS
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
taking all hive entity names as lowercase
parent
45c22514
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
23 additions
and
15 deletions
+23
-15
HiveMetaStoreBridge.java
...ache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
+3
-3
HiveHook.java
...n/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
+7
-7
HiveHookIT.java
...java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
+13
-5
No files found.
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
View file @
fee7c5cc
...
...
@@ -158,7 +158,7 @@ public class HiveMetaStoreBridge {
String
typeName
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
dslQuery
=
String
.
format
(
"%s where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_DB
.
getName
(),
databaseName
,
clusterName
);
databaseName
.
toLowerCase
()
,
clusterName
);
return
getEntityReferenceFromDSL
(
typeName
,
dslQuery
);
}
...
...
@@ -194,7 +194,7 @@ public class HiveMetaStoreBridge {
String
dslQuery
=
String
.
format
(
"%s as t where name = '%s', dbName where name = '%s' and "
+
"clusterName = '%s' select t"
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
,
dbName
,
clusterName
);
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
.
toLowerCase
(),
dbName
.
toLowerCase
()
,
clusterName
);
return
getEntityReferenceFromDSL
(
typeName
,
dslQuery
);
// String dbType = HiveDataTypes.HIVE_DB.getName();
...
...
@@ -230,7 +230,7 @@ public class HiveMetaStoreBridge {
String
gremlinQuery
=
String
.
format
(
"g.V.has('__typeName', '%s').has('%s.values', %s).as('p')."
+
"out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+
".has('%s.clusterName', '%s').back('p').toList()"
,
typeName
,
typeName
,
valuesStr
,
typeName
,
tableType
,
tableName
,
tableType
,
dbType
,
dbName
,
dbType
,
clusterName
);
tableType
,
tableName
.
toLowerCase
(),
tableType
,
dbType
,
dbName
.
toLowerCase
()
,
dbType
,
clusterName
);
return
getEntityReferenceFromGremlin
(
typeName
,
gremlinQuery
);
}
...
...
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
View file @
fee7c5cc
...
...
@@ -243,11 +243,13 @@ public class HiveHook implements ExecuteWithHookContext {
return
;
}
Referenceable
dbReferenceable
=
dgiBridge
.
registerDatabase
(
oldTable
.
getDbName
()
.
toLowerCase
()
);
Referenceable
dbReferenceable
=
dgiBridge
.
registerDatabase
(
oldTable
.
getDbName
());
Referenceable
tableReferenceable
=
dgiBridge
.
registerTable
(
dbReferenceable
,
oldTable
.
getDbName
(),
oldTable
.
getTableName
());
LOG
.
info
(
"Updating entity name {}.{} to {}"
,
oldTable
.
getDbName
(),
oldTable
.
getTableName
(),
newTable
.
getTableName
());
dgiBridge
.
getMetadataServiceClient
().
updateEntity
(
tableReferenceable
.
getId
().
_getId
(),
"name"
,
newTable
.
getTableName
());
newTable
.
getTableName
()
.
toLowerCase
()
);
}
private
void
handleCreateTable
(
HiveMetaStoreBridge
dgiBridge
,
HiveEvent
event
)
throws
Exception
{
...
...
@@ -255,9 +257,7 @@ public class HiveHook implements ExecuteWithHookContext {
if
(
entity
.
getType
()
==
Entity
.
Type
.
TABLE
)
{
Table
table
=
entity
.
getTable
();
//TODO table.getDbName().toLowerCase() is required as hive stores in lowercase,
// but table.getDbName() is not lowercase
Referenceable
dbReferenceable
=
dgiBridge
.
registerDatabase
(
table
.
getDbName
().
toLowerCase
());
Referenceable
dbReferenceable
=
dgiBridge
.
registerDatabase
(
table
.
getDbName
());
dgiBridge
.
registerTable
(
dbReferenceable
,
table
.
getDbName
(),
table
.
getTableName
());
}
}
...
...
@@ -297,7 +297,7 @@ public class HiveHook implements ExecuteWithHookContext {
for
(
ReadEntity
readEntity
:
inputs
)
{
if
(
readEntity
.
getType
()
==
Entity
.
Type
.
TABLE
)
{
Table
table
=
readEntity
.
getTable
();
String
dbName
=
table
.
getDbName
()
.
toLowerCase
()
;
String
dbName
=
table
.
getDbName
();
source
.
add
(
dgiBridge
.
registerTable
(
dbName
,
table
.
getTableName
()));
}
if
(
readEntity
.
getType
()
==
Entity
.
Type
.
PARTITION
)
{
...
...
@@ -309,7 +309,7 @@ public class HiveHook implements ExecuteWithHookContext {
for
(
WriteEntity
writeEntity
:
outputs
)
{
if
(
writeEntity
.
getType
()
==
Entity
.
Type
.
TABLE
||
writeEntity
.
getType
()
==
Entity
.
Type
.
PARTITION
)
{
Table
table
=
writeEntity
.
getTable
();
String
dbName
=
table
.
getDbName
()
.
toLowerCase
()
;
String
dbName
=
table
.
getDbName
();
target
.
add
(
dgiBridge
.
registerTable
(
dbName
,
table
.
getTableName
()));
}
if
(
writeEntity
.
getType
()
==
Entity
.
Type
.
PARTITION
)
{
...
...
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
View file @
fee7c5cc
...
...
@@ -26,8 +26,10 @@ import org.apache.hadoop.metadata.MetadataServiceClient;
import
org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.apache.hadoop.metadata.typesystem.Referenceable
;
import
org.apache.log4j.spi.LoggerFactory
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONObject
;
import
org.slf4j.Logger
;
import
org.testng.Assert
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.Test
;
...
...
@@ -36,6 +38,8 @@ import java.io.File;
import
java.util.Map
;
public
class
HiveHookIT
{
public
static
final
Logger
LOG
=
org
.
slf4j
.
LoggerFactory
.
getLogger
(
HiveHookIT
.
class
);
private
static
final
String
DGI_URL
=
"http://localhost:21000/"
;
private
static
final
String
CLUSTER_NAME
=
"test"
;
public
static
final
String
DEFAULT_DB
=
"default"
;
...
...
@@ -178,7 +182,7 @@ public class HiveHookIT {
}
private
String
random
()
{
return
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
(
);
return
RandomStringUtils
.
randomAlphanumeric
(
10
);
}
private
String
file
(
String
tag
)
throws
Exception
{
...
...
@@ -259,26 +263,30 @@ public class HiveHookIT {
}
private
String
assertTableIsRegistered
(
String
dbName
,
String
tableName
,
boolean
registered
)
throws
Exception
{
LOG
.
debug
(
"Searching for table {}.{}"
,
dbName
,
tableName
);
String
query
=
String
.
format
(
"%s as t where name = '%s', dbName where name = '%s' and clusterName = '%s'"
+
" select t"
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
,
dbName
,
CLUSTER_NAME
);
+
" select t"
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
.
toLowerCase
(),
dbName
.
toLowerCase
(),
CLUSTER_NAME
);
return
assertEntityIsRegistered
(
query
,
registered
);
}
private
String
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
LOG
.
debug
(
"Searching for database {}"
,
dbName
);
String
query
=
String
.
format
(
"%s where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_DB
.
getName
(),
dbName
,
CLUSTER_NAME
);
dbName
.
toLowerCase
()
,
CLUSTER_NAME
);
return
assertEntityIsRegistered
(
query
,
true
);
}
private
void
assertPartitionIsRegistered
(
String
dbName
,
String
tableName
,
String
value
)
throws
Exception
{
String
typeName
=
HiveDataTypes
.
HIVE_PARTITION
.
getName
();
String
dbType
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
tableType
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
LOG
.
debug
(
"Searching for partition of {}.{} with values {}"
,
dbName
,
tableName
,
value
);
String
gremlinQuery
=
String
.
format
(
"g.V.has('__typeName', '%s').has('%s.values', ['%s']).as('p')."
+
"out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+
".has('%s.clusterName', '%s').back('p').toList()"
,
typeName
,
typeName
,
value
,
typeName
,
tableType
,
tableName
,
tableType
,
dbType
,
dbName
,
dbType
,
CLUSTER_NAME
);
tableType
,
tableName
.
toLowerCase
(),
tableType
,
dbType
,
dbName
.
toLowerCase
()
,
dbType
,
CLUSTER_NAME
);
JSONObject
response
=
dgiCLient
.
searchByGremlin
(
gremlinQuery
);
JSONArray
results
=
response
.
getJSONArray
(
MetadataServiceClient
.
RESULTS
);
Assert
.
assertEquals
(
results
.
length
(),
1
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment