Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
3e695bad
Commit
3e695bad
authored
May 26, 2015
by
Shwetha GS
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
handling all hive query operations in hive hook
parent
1abeba45
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
266 additions
and
88 deletions
+266
-88
pom.xml
addons/hive-bridge/pom.xml
+19
-19
HiveMetaStoreBridge.java
...ache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
+87
-37
HiveHook.java
...n/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
+31
-17
HiveHookIT.java
...java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
+119
-12
MetadataServiceClient.java
...ava/org/apache/hadoop/metadata/MetadataServiceClient.java
+1
-1
pom.xml
pom.xml
+5
-0
GraphBackedDiscoveryService.java
...metadata/discovery/graph/GraphBackedDiscoveryService.java
+2
-0
GraphBackedMetadataRepository.java
...adata/repository/graph/GraphBackedMetadataRepository.java
+0
-0
GraphBackedSearchIndexer.java
...p/metadata/repository/graph/GraphBackedSearchIndexer.java
+1
-1
log4j.xml
typesystem/src/main/resources/log4j.xml
+1
-1
No files found.
addons/hive-bridge/pom.xml
View file @
3e695bad
...
@@ -40,25 +40,6 @@
...
@@ -40,25 +40,6 @@
<dependencies>
<dependencies>
<dependency>
<dependency>
<groupId>
org.apache.hadoop.metadata
</groupId>
<artifactId>
metadata-client
</artifactId>
<version>
${version}
</version>
<exclusions>
<exclusion>
<groupId>
com.google.guava
</groupId>
<artifactId>
guava
</artifactId>
</exclusion>
</exclusions>
<scope>
runtime
</scope>
<type>
test-jar
</type>
</dependency>
<dependency>
<groupId>
org.apache.hadoop.metadata
</groupId>
<artifactId>
metadata-typesystem
</artifactId>
</dependency>
<dependency>
<groupId>
org.apache.hadoop
</groupId>
<groupId>
org.apache.hadoop
</groupId>
<artifactId>
hadoop-minikdc
</artifactId>
<artifactId>
hadoop-minikdc
</artifactId>
<version>
${hadoop.version}
</version>
<version>
${hadoop.version}
</version>
...
@@ -99,6 +80,25 @@
...
@@ -99,6 +80,25 @@
</dependency>
</dependency>
<dependency>
<dependency>
<groupId>
org.apache.hadoop.metadata
</groupId>
<artifactId>
metadata-client
</artifactId>
<version>
${version}
</version>
<exclusions>
<exclusion>
<groupId>
com.google.guava
</groupId>
<artifactId>
guava
</artifactId>
</exclusion>
</exclusions>
<scope>
runtime
</scope>
<type>
test-jar
</type>
</dependency>
<dependency>
<groupId>
org.apache.hadoop.metadata
</groupId>
<artifactId>
metadata-typesystem
</artifactId>
</dependency>
<dependency>
<groupId>
org.apache.hadoop
</groupId>
<groupId>
org.apache.hadoop
</groupId>
<artifactId>
hadoop-client
</artifactId>
<artifactId>
hadoop-client
</artifactId>
<version>
${hadoop.version}
</version>
<version>
${hadoop.version}
</version>
...
...
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
View file @
3e695bad
...
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
...
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import
org.apache.hadoop.hive.ql.metadata.Partition
;
import
org.apache.hadoop.hive.ql.metadata.Partition
;
import
org.apache.hadoop.hive.ql.metadata.Table
;
import
org.apache.hadoop.hive.ql.metadata.Table
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.MetadataServiceException
;
import
org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator
;
import
org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.apache.hadoop.metadata.typesystem.Referenceable
;
import
org.apache.hadoop.metadata.typesystem.Referenceable
;
...
@@ -91,31 +92,6 @@ public class HiveMetaStoreBridge {
...
@@ -91,31 +92,6 @@ public class HiveMetaStoreBridge {
}
}
}
}
/**
* Gets reference for the database
*
*
* @param databaseName
* @param clusterName cluster name
* @return Reference for database if exists, else null
* @throws Exception
*/
private
Referenceable
getDatabaseReference
(
String
databaseName
,
String
clusterName
)
throws
Exception
{
LOG
.
debug
(
"Getting reference for database {}"
,
databaseName
);
String
typeName
=
HiveDataTypes
.
HIVE_DB
.
getName
();
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
String
dslQuery
=
String
.
format
(
"%s where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_DB
.
getName
(),
databaseName
,
clusterName
);
JSONArray
results
=
dgiClient
.
searchByDSL
(
dslQuery
);
if
(
results
.
length
()
==
0
)
{
return
null
;
}
else
{
String
guid
=
getGuidFromDSLResponse
(
results
.
getJSONObject
(
0
));
return
new
Referenceable
(
guid
,
typeName
,
null
);
}
}
public
Referenceable
registerDatabase
(
String
databaseName
)
throws
Exception
{
public
Referenceable
registerDatabase
(
String
databaseName
)
throws
Exception
{
Referenceable
dbRef
=
getDatabaseReference
(
databaseName
,
clusterName
);
Referenceable
dbRef
=
getDatabaseReference
(
databaseName
,
clusterName
);
if
(
dbRef
==
null
)
{
if
(
dbRef
==
null
)
{
...
@@ -169,6 +145,35 @@ public class HiveMetaStoreBridge {
...
@@ -169,6 +145,35 @@ public class HiveMetaStoreBridge {
}
}
/**
/**
* Gets reference for the database
*
*
* @param databaseName
* @param clusterName cluster name
* @return Reference for database if exists, else null
* @throws Exception
*/
private
Referenceable
getDatabaseReference
(
String
databaseName
,
String
clusterName
)
throws
Exception
{
LOG
.
debug
(
"Getting reference for database {}"
,
databaseName
);
String
typeName
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
dslQuery
=
String
.
format
(
"%s where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_DB
.
getName
(),
databaseName
,
clusterName
);
return
getEntityReferenceFromDSL
(
typeName
,
dslQuery
);
}
private
Referenceable
getEntityReferenceFromDSL
(
String
typeName
,
String
dslQuery
)
throws
Exception
{
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
JSONArray
results
=
dgiClient
.
searchByDSL
(
dslQuery
);
if
(
results
.
length
()
==
0
)
{
return
null
;
}
else
{
String
guid
=
getGuidFromDSLResponse
(
results
.
getJSONObject
(
0
));
return
new
Referenceable
(
guid
,
typeName
,
null
);
}
}
/**
* Gets reference for the table
* Gets reference for the table
*
*
* @param dbName
* @param dbName
...
@@ -180,19 +185,47 @@ public class HiveMetaStoreBridge {
...
@@ -180,19 +185,47 @@ public class HiveMetaStoreBridge {
LOG
.
debug
(
"Getting reference for table {}.{}"
,
dbName
,
tableName
);
LOG
.
debug
(
"Getting reference for table {}.{}"
,
dbName
,
tableName
);
String
typeName
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
String
typeName
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
String
query
=
String
.
format
(
"%s where name = '%s', dbName where name = '%s' and clusterName = '%s'"
,
// String dslQuery = String.format("%s as t where name = '%s' dbName where name = '%s' and "
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
,
dbName
,
clusterName
);
// + "clusterName = '%s' select t",
JSONArray
results
=
dgiClient
.
searchByDSL
(
query
);
// HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, clusterName);
String
dbType
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
gremlinQuery
=
String
.
format
(
"g.V.has('__typeName', '%s').has('%s.name', '%s').as('t').out"
+
"('__%s.dbName').has('%s.name', '%s').has('%s.clusterName', '%s').back('t').toList()"
,
typeName
,
typeName
,
tableName
,
typeName
,
dbType
,
dbName
,
dbType
,
clusterName
);
return
getEntityReferenceFromGremlin
(
typeName
,
gremlinQuery
);
}
private
Referenceable
getEntityReferenceFromGremlin
(
String
typeName
,
String
gremlinQuery
)
throws
MetadataServiceException
,
JSONException
{
MetadataServiceClient
client
=
getMetadataServiceClient
();
JSONObject
response
=
client
.
searchByGremlin
(
gremlinQuery
);
JSONArray
results
=
response
.
getJSONArray
(
MetadataServiceClient
.
RESULTS
);
if
(
results
.
length
()
==
0
)
{
if
(
results
.
length
()
==
0
)
{
return
null
;
return
null
;
}
else
{
}
//There should be just one instance with the given name
String
guid
=
results
.
getJSONObject
(
0
).
getString
(
"__guid"
);
String
guid
=
getGuidFromDSLResponse
(
results
.
getJSONObject
(
0
));
LOG
.
debug
(
"Got reference for table {}.{} = {}"
,
dbName
,
tableName
,
guid
);
return
new
Referenceable
(
guid
,
typeName
,
null
);
return
new
Referenceable
(
guid
,
typeName
,
null
);
}
}
private
Referenceable
getPartitionReference
(
String
dbName
,
String
tableName
,
List
<
String
>
values
)
throws
Exception
{
String
valuesStr
=
"['"
+
StringUtils
.
join
(
values
,
"', '"
)
+
"']"
;
LOG
.
debug
(
"Getting reference for partition for {}.{} with values {}"
,
dbName
,
tableName
,
valuesStr
);
String
typeName
=
HiveDataTypes
.
HIVE_PARTITION
.
getName
();
// String dslQuery = String.format("%s as p where values = %s, tableName where name = '%s', "
// + "dbName where name = '%s' and clusterName = '%s' select p", typeName, valuesStr, tableName,
// dbName, clusterName);
String
dbType
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
tableType
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
String
gremlinQuery
=
String
.
format
(
"g.V.has('__typeName', '%s').has('%s.values', %s).as('p')."
+
"out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+
".has('%s.clusterName', '%s').back('p').toList()"
,
typeName
,
typeName
,
valuesStr
,
typeName
,
tableType
,
tableName
,
tableType
,
dbType
,
dbName
,
dbType
,
clusterName
);
return
getEntityReferenceFromGremlin
(
typeName
,
gremlinQuery
);
}
}
private
String
getGuidFromDSLResponse
(
JSONObject
jsonObject
)
throws
JSONException
{
private
String
getGuidFromDSLResponse
(
JSONObject
jsonObject
)
throws
JSONException
{
...
@@ -292,14 +325,27 @@ public class HiveMetaStoreBridge {
...
@@ -292,14 +325,27 @@ public class HiveMetaStoreBridge {
}
}
}
}
//todo should be idempotent
public
Referenceable
registerPartition
(
Partition
partition
)
throws
Exception
{
String
dbName
=
partition
.
getTable
().
getDbName
();
String
tableName
=
partition
.
getTable
().
getTableName
();
Referenceable
dbRef
=
registerDatabase
(
dbName
);
Referenceable
tableRef
=
registerTable
(
dbName
,
tableName
);
Referenceable
sdRef
=
getSDForTable
(
dbName
,
tableName
);
return
importPartition
(
partition
,
dbRef
,
tableRef
,
sdRef
);
}
private
Referenceable
importPartition
(
Partition
hivePart
,
private
Referenceable
importPartition
(
Partition
hivePart
,
Referenceable
dbReferenceable
,
Referenceable
dbReferenceable
,
Referenceable
tableReferenceable
,
Referenceable
tableReferenceable
,
Referenceable
sdReferenceable
)
throws
Exception
{
Referenceable
sdReferenceable
)
throws
Exception
{
LOG
.
info
(
"Importing partition for {}.{} with values {}"
,
dbReferenceable
,
tableReferenceable
,
LOG
.
info
(
"Importing partition for {}.{} with values {}"
,
dbReferenceable
,
tableReferenceable
,
StringUtils
.
join
(
hivePart
.
getValues
(),
","
));
StringUtils
.
join
(
hivePart
.
getValues
(),
","
));
Referenceable
partRef
=
new
Referenceable
(
HiveDataTypes
.
HIVE_PARTITION
.
getName
());
String
dbName
=
hivePart
.
getTable
().
getDbName
();
String
tableName
=
hivePart
.
getTable
().
getTableName
();
Referenceable
partRef
=
getPartitionReference
(
dbName
,
tableName
,
hivePart
.
getValues
());
if
(
partRef
==
null
)
{
partRef
=
new
Referenceable
(
HiveDataTypes
.
HIVE_PARTITION
.
getName
());
partRef
.
set
(
"values"
,
hivePart
.
getValues
());
partRef
.
set
(
"values"
,
hivePart
.
getValues
());
partRef
.
set
(
"dbName"
,
dbReferenceable
);
partRef
.
set
(
"dbName"
,
dbReferenceable
);
...
@@ -315,8 +361,12 @@ public class HiveMetaStoreBridge {
...
@@ -315,8 +361,12 @@ public class HiveMetaStoreBridge {
partRef
.
set
(
"sd"
,
sdReferenceable
);
partRef
.
set
(
"sd"
,
sdReferenceable
);
partRef
.
set
(
"parameters"
,
hivePart
.
getParameters
());
partRef
.
set
(
"parameters"
,
hivePart
.
getParameters
());
partRef
=
createInstance
(
partRef
);
return
createInstance
(
partRef
);
}
else
{
LOG
.
info
(
"Partition {}.{} with values {} is already registered with id {}"
,
dbName
,
tableName
,
StringUtils
.
join
(
hivePart
.
getValues
(),
","
),
partRef
.
getId
().
id
);
}
return
partRef
;
}
}
private
void
importIndexes
(
String
db
,
String
table
,
private
void
importIndexes
(
String
db
,
String
table
,
...
...
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
View file @
3e695bad
...
@@ -189,17 +189,28 @@ public class HiveHook implements ExecuteWithHookContext {
...
@@ -189,17 +189,28 @@ public class HiveHook implements ExecuteWithHookContext {
switch
(
event
.
operation
)
{
switch
(
event
.
operation
)
{
case
CREATEDATABASE:
case
CREATEDATABASE:
Set
<
WriteEntity
>
outputs
=
event
.
outputs
;
handleCreateDB
(
dgiBridge
,
event
);
for
(
WriteEntity
entity
:
outputs
)
{
if
(
entity
.
getType
()
==
Entity
.
Type
.
DATABASE
)
{
dgiBridge
.
registerDatabase
(
entity
.
getDatabase
().
getName
());
}
}
break
;
break
;
case
CREATETABLE:
case
CREATETABLE:
outputs
=
event
.
outputs
;
handleCreateTable
(
dgiBridge
,
event
);
for
(
WriteEntity
entity
:
outputs
)
{
break
;
case
CREATETABLE_AS_SELECT:
case
CREATEVIEW:
case
LOAD:
case
EXPORT:
case
IMPORT:
case
QUERY:
registerProcess
(
dgiBridge
,
event
);
break
;
default
:
}
}
private
void
handleCreateTable
(
HiveMetaStoreBridge
dgiBridge
,
HiveEvent
event
)
throws
Exception
{
for
(
WriteEntity
entity
:
event
.
outputs
)
{
if
(
entity
.
getType
()
==
Entity
.
Type
.
TABLE
)
{
if
(
entity
.
getType
()
==
Entity
.
Type
.
TABLE
)
{
Table
table
=
entity
.
getTable
();
Table
table
=
entity
.
getTable
();
...
@@ -209,17 +220,17 @@ public class HiveHook implements ExecuteWithHookContext {
...
@@ -209,17 +220,17 @@ public class HiveHook implements ExecuteWithHookContext {
dgiBridge
.
registerTable
(
dbReferenceable
,
table
.
getDbName
(),
table
.
getTableName
());
dgiBridge
.
registerTable
(
dbReferenceable
,
table
.
getDbName
(),
table
.
getTableName
());
}
}
}
}
break
;
}
case
CREATETABLE_AS_SELECT:
registerCTAS
(
dgiBridge
,
event
);
break
;
default
:
private
void
handleCreateDB
(
HiveMetaStoreBridge
dgiBridge
,
HiveEvent
event
)
throws
Exception
{
for
(
WriteEntity
entity
:
event
.
outputs
)
{
if
(
entity
.
getType
()
==
Entity
.
Type
.
DATABASE
)
{
dgiBridge
.
registerDatabase
(
entity
.
getDatabase
().
getName
());
}
}
}
}
}
private
void
register
CTAS
(
HiveMetaStoreBridge
dgiBridge
,
HiveEvent
event
)
throws
Exception
{
private
void
register
Process
(
HiveMetaStoreBridge
dgiBridge
,
HiveEvent
event
)
throws
Exception
{
Set
<
ReadEntity
>
inputs
=
event
.
inputs
;
Set
<
ReadEntity
>
inputs
=
event
.
inputs
;
Set
<
WriteEntity
>
outputs
=
event
.
outputs
;
Set
<
WriteEntity
>
outputs
=
event
.
outputs
;
...
@@ -243,7 +254,7 @@ public class HiveHook implements ExecuteWithHookContext {
...
@@ -243,7 +254,7 @@ public class HiveHook implements ExecuteWithHookContext {
processReferenceable
.
set
(
"userName"
,
event
.
user
);
processReferenceable
.
set
(
"userName"
,
event
.
user
);
List
<
Referenceable
>
source
=
new
ArrayList
<>();
List
<
Referenceable
>
source
=
new
ArrayList
<>();
for
(
ReadEntity
readEntity
:
inputs
)
{
for
(
ReadEntity
readEntity
:
inputs
)
{
if
(
readEntity
.
getTyp
()
==
Entity
.
Type
.
TABLE
)
{
if
(
readEntity
.
getTyp
e
()
==
Entity
.
Type
.
TABLE
)
{
Table
table
=
readEntity
.
getTable
();
Table
table
=
readEntity
.
getTable
();
String
dbName
=
table
.
getDbName
().
toLowerCase
();
String
dbName
=
table
.
getDbName
().
toLowerCase
();
source
.
add
(
dgiBridge
.
registerTable
(
dbName
,
table
.
getTableName
()));
source
.
add
(
dgiBridge
.
registerTable
(
dbName
,
table
.
getTableName
()));
...
@@ -252,11 +263,14 @@ public class HiveHook implements ExecuteWithHookContext {
...
@@ -252,11 +263,14 @@ public class HiveHook implements ExecuteWithHookContext {
processReferenceable
.
set
(
"inputTables"
,
source
);
processReferenceable
.
set
(
"inputTables"
,
source
);
List
<
Referenceable
>
target
=
new
ArrayList
<>();
List
<
Referenceable
>
target
=
new
ArrayList
<>();
for
(
WriteEntity
writeEntity
:
outputs
)
{
for
(
WriteEntity
writeEntity
:
outputs
)
{
if
(
writeEntity
.
getTyp
()
==
Entity
.
Type
.
TABLE
)
{
if
(
writeEntity
.
getTyp
e
()
==
Entity
.
Type
.
TABLE
||
writeEntity
.
getType
()
==
Entity
.
Type
.
PARTITION
)
{
Table
table
=
writeEntity
.
getTable
();
Table
table
=
writeEntity
.
getTable
();
String
dbName
=
table
.
getDbName
().
toLowerCase
();
String
dbName
=
table
.
getDbName
().
toLowerCase
();
target
.
add
(
dgiBridge
.
registerTable
(
dbName
,
table
.
getTableName
()));
target
.
add
(
dgiBridge
.
registerTable
(
dbName
,
table
.
getTableName
()));
}
}
if
(
writeEntity
.
getType
()
==
Entity
.
Type
.
PARTITION
)
{
dgiBridge
.
registerPartition
(
writeEntity
.
getPartition
());
}
}
}
processReferenceable
.
set
(
"outputTables"
,
target
);
processReferenceable
.
set
(
"outputTables"
,
target
);
processReferenceable
.
set
(
"queryText"
,
queryStr
);
processReferenceable
.
set
(
"queryText"
,
queryStr
);
...
...
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
View file @
3e695bad
...
@@ -26,10 +26,13 @@ import org.apache.hadoop.metadata.MetadataServiceClient;
...
@@ -26,10 +26,13 @@ import org.apache.hadoop.metadata.MetadataServiceClient;
import
org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge
;
import
org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONObject
;
import
org.testng.Assert
;
import
org.testng.Assert
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.Test
;
import
org.testng.annotations.Test
;
import
java.io.File
;
public
class
HiveHookIT
{
public
class
HiveHookIT
{
private
static
final
String
DGI_URL
=
"http://localhost:21000/"
;
private
static
final
String
DGI_URL
=
"http://localhost:21000/"
;
private
static
final
String
CLUSTER_NAME
=
"test"
;
private
static
final
String
CLUSTER_NAME
=
"test"
;
...
@@ -59,6 +62,9 @@ public class HiveHookIT {
...
@@ -59,6 +62,9 @@ public class HiveHookIT {
hiveConf
.
set
(
"javax.jdo.option.ConnectionURL"
,
"jdbc:derby:./target/metastore_db;create=true"
);
hiveConf
.
set
(
"javax.jdo.option.ConnectionURL"
,
"jdbc:derby:./target/metastore_db;create=true"
);
hiveConf
.
set
(
"hive.hook.dgi.synchronous"
,
"true"
);
hiveConf
.
set
(
"hive.hook.dgi.synchronous"
,
"true"
);
hiveConf
.
set
(
HiveMetaStoreBridge
.
HIVE_CLUSTER_NAME
,
CLUSTER_NAME
);
hiveConf
.
set
(
HiveMetaStoreBridge
.
HIVE_CLUSTER_NAME
,
CLUSTER_NAME
);
//weird, hive prepends test_ to table name
hiveConf
.
set
(
"hive.test.mode"
,
"true"
);
hiveConf
.
set
(
"fs.pfile.impl"
,
"org.apache.hadoop.fs.ProxyLocalFileSystem"
);
return
hiveConf
;
return
hiveConf
;
}
}
...
@@ -69,7 +75,7 @@ public class HiveHookIT {
...
@@ -69,7 +75,7 @@ public class HiveHookIT {
@Test
@Test
public
void
testCreateDatabase
()
throws
Exception
{
public
void
testCreateDatabase
()
throws
Exception
{
String
dbName
=
"db"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
dbName
=
"db"
+
random
();
runCommand
(
"create database "
+
dbName
);
runCommand
(
"create database "
+
dbName
);
assertDatabaseIsRegistered
(
dbName
);
assertDatabaseIsRegistered
(
dbName
);
...
@@ -77,15 +83,15 @@ public class HiveHookIT {
...
@@ -77,15 +83,15 @@ public class HiveHookIT {
@Test
@Test
public
void
testCreateTable
()
throws
Exception
{
public
void
testCreateTable
()
throws
Exception
{
String
dbName
=
"db"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
dbName
=
"db"
+
random
();
runCommand
(
"create database "
+
dbName
);
runCommand
(
"create database "
+
dbName
);
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
tableName
=
"table"
+
random
();
runCommand
(
"create table "
+
dbName
+
"."
+
tableName
+
"(id int, name string)"
);
runCommand
(
"create table "
+
dbName
+
"."
+
tableName
+
"(id int, name string)"
);
assertTableIsRegistered
(
dbName
,
tableName
);
assertTableIsRegistered
(
dbName
,
tableName
);
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
tableName
=
"table"
+
random
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
runCommand
(
"create table "
+
tableName
+
"(id int, name string)
partitioned by(dt string)
"
);
assertTableIsRegistered
(
"default"
,
tableName
);
assertTableIsRegistered
(
"default"
,
tableName
);
//Create table where database doesn't exist, will create database instance as well
//Create table where database doesn't exist, will create database instance as well
...
@@ -94,10 +100,10 @@ public class HiveHookIT {
...
@@ -94,10 +100,10 @@ public class HiveHookIT {
@Test
@Test
public
void
testCTAS
()
throws
Exception
{
public
void
testCTAS
()
throws
Exception
{
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
tableName
=
"table"
+
random
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
String
ctasTableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
ctasTableName
=
"table"
+
random
();
String
query
=
"create table "
+
ctasTableName
+
" as select * from "
+
tableName
;
String
query
=
"create table "
+
ctasTableName
+
" as select * from "
+
tableName
;
runCommand
(
query
);
runCommand
(
query
);
...
@@ -105,24 +111,125 @@ public class HiveHookIT {
...
@@ -105,24 +111,125 @@ public class HiveHookIT {
assertProcessIsRegistered
(
query
);
assertProcessIsRegistered
(
query
);
}
}
@Test
public
void
testCreateView
()
throws
Exception
{
String
tableName
=
"table"
+
random
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
String
viewName
=
"table"
+
random
();
String
query
=
"create view "
+
viewName
+
" as select * from "
+
tableName
;
runCommand
(
query
);
assertTableIsRegistered
(
"default"
,
viewName
);
assertProcessIsRegistered
(
query
);
}
@Test
public
void
testLoadData
()
throws
Exception
{
String
tableName
=
"table"
+
random
();
runCommand
(
"create table test_"
+
tableName
+
"(id int, name string)"
);
String
loadFile
=
file
(
"load"
);
String
query
=
"load data local inpath 'file://"
+
loadFile
+
"' into table "
+
tableName
;
runCommand
(
query
);
assertProcessIsRegistered
(
query
);
}
@Test
public
void
testInsert
()
throws
Exception
{
String
tableName
=
"table"
+
random
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string) partitioned by(dt string)"
);
String
insertTableName
=
"table"
+
random
();
runCommand
(
"create table test_"
+
insertTableName
+
"(name string) partitioned by(dt string)"
);
String
query
=
"insert into "
+
insertTableName
+
" partition(dt = '2015-01-01') select name from "
+
tableName
+
" where dt = '2015-01-01'"
;
runCommand
(
query
);
assertProcessIsRegistered
(
query
);
assertPartitionIsRegistered
(
"default"
,
"test_"
+
insertTableName
,
"2015-01-01"
);
}
private
String
random
()
{
return
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
}
private
String
file
(
String
tag
)
throws
Exception
{
String
filename
=
"./target/"
+
tag
+
"-data-"
+
random
();
File
file
=
new
File
(
filename
);
file
.
createNewFile
();
return
file
.
getAbsolutePath
();
}
private
String
mkdir
(
String
tag
)
throws
Exception
{
String
filename
=
"./target/"
+
tag
+
"-data-"
+
random
();
File
file
=
new
File
(
filename
);
file
.
mkdirs
();
return
file
.
getAbsolutePath
();
}
@Test
public
void
testExportImport
()
throws
Exception
{
String
tableName
=
"table"
+
random
();
runCommand
(
"create table test_"
+
tableName
+
"(name string)"
);
String
filename
=
"pfile://"
+
mkdir
(
"export"
);
String
query
=
"export table "
+
tableName
+
" to '"
+
filename
+
"'"
;
runCommand
(
query
);
assertProcessIsRegistered
(
query
);
tableName
=
"table"
+
random
();
runCommand
(
"create table "
+
tableName
+
"(name string)"
);
query
=
"import table "
+
tableName
+
" from '"
+
filename
+
"'"
;
runCommand
(
query
);
assertProcessIsRegistered
(
query
);
}
@Test
public
void
testSelect
()
throws
Exception
{
String
tableName
=
"table"
+
random
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
String
query
=
"select * from "
+
tableName
;
runCommand
(
query
);
assertProcessIsRegistered
(
query
);
}
private
void
assertProcessIsRegistered
(
String
queryStr
)
throws
Exception
{
private
void
assertProcessIsRegistered
(
String
queryStr
)
throws
Exception
{
String
dslQuery
=
String
.
format
(
"%s where queryText =
'%s'
"
,
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
queryStr
);
String
dslQuery
=
String
.
format
(
"%s where queryText =
\"%s\"
"
,
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
queryStr
);
assert
Instance
IsRegistered
(
dslQuery
);
assert
Entity
IsRegistered
(
dslQuery
);
}
}
private
void
assertTableIsRegistered
(
String
dbName
,
String
tableName
)
throws
Exception
{
private
void
assertTableIsRegistered
(
String
dbName
,
String
tableName
)
throws
Exception
{
String
query
=
String
.
format
(
"%s where name = '%s', dbName where name = '%s' and clusterName = '%s'"
,
String
query
=
String
.
format
(
"%s where name = '%s', dbName where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
,
dbName
,
CLUSTER_NAME
);
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
,
dbName
,
CLUSTER_NAME
);
assert
Instance
IsRegistered
(
query
);
assert
Entity
IsRegistered
(
query
);
}
}
private
void
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
private
void
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
String
query
=
String
.
format
(
"%s where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_DB
.
getName
(),
String
query
=
String
.
format
(
"%s where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_DB
.
getName
(),
dbName
,
CLUSTER_NAME
);
dbName
,
CLUSTER_NAME
);
assertInstanceIsRegistered
(
query
);
assertEntityIsRegistered
(
query
);
}
private
void
assertPartitionIsRegistered
(
String
dbName
,
String
tableName
,
String
value
)
throws
Exception
{
String
typeName
=
HiveDataTypes
.
HIVE_PARTITION
.
getName
();
String
dbType
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
tableType
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
String
gremlinQuery
=
String
.
format
(
"g.V.has('__typeName', '%s').has('%s.values', ['%s']).as('p')."
+
"out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+
".has('%s.clusterName', '%s').back('p').toList()"
,
typeName
,
typeName
,
value
,
typeName
,
tableType
,
tableName
,
tableType
,
dbType
,
dbName
,
dbType
,
CLUSTER_NAME
);
JSONObject
response
=
dgiCLient
.
searchByGremlin
(
gremlinQuery
);
JSONArray
results
=
response
.
getJSONArray
(
MetadataServiceClient
.
RESULTS
);
Assert
.
assertEquals
(
results
.
length
(),
1
);
}
}
private
void
assert
Instance
IsRegistered
(
String
dslQuery
)
throws
Exception
{
private
void
assert
Entity
IsRegistered
(
String
dslQuery
)
throws
Exception
{
JSONArray
results
=
dgiCLient
.
searchByDSL
(
dslQuery
);
JSONArray
results
=
dgiCLient
.
searchByDSL
(
dslQuery
);
Assert
.
assertEquals
(
results
.
length
(),
1
);
Assert
.
assertEquals
(
results
.
length
(),
1
);
}
}
...
...
client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
View file @
3e695bad
...
@@ -200,7 +200,7 @@ public class MetadataServiceClient {
...
@@ -200,7 +200,7 @@ public class MetadataServiceClient {
public
Referenceable
getEntity
(
String
guid
)
throws
MetadataServiceException
{
public
Referenceable
getEntity
(
String
guid
)
throws
MetadataServiceException
{
JSONObject
jsonResponse
=
callAPI
(
API
.
GET_ENTITY
,
null
,
guid
);
JSONObject
jsonResponse
=
callAPI
(
API
.
GET_ENTITY
,
null
,
guid
);
try
{
try
{
String
entityInstanceDefinition
=
jsonResponse
.
getString
(
MetadataServiceClient
.
GUID
);
String
entityInstanceDefinition
=
jsonResponse
.
getString
(
MetadataServiceClient
.
DEFINITION
);
return
InstanceSerialization
.
fromJsonReferenceable
(
entityInstanceDefinition
,
true
);
return
InstanceSerialization
.
fromJsonReferenceable
(
entityInstanceDefinition
,
true
);
}
catch
(
JSONException
e
)
{
}
catch
(
JSONException
e
)
{
throw
new
MetadataServiceException
(
e
);
throw
new
MetadataServiceException
(
e
);
...
...
pom.xml
View file @
3e695bad
...
@@ -101,6 +101,9 @@
...
@@ -101,6 +101,9 @@
<StagingId>
apache-staging
</StagingId>
<StagingId>
apache-staging
</StagingId>
<StagingName>
Apache Release Distribution Repository
</StagingName>
<StagingName>
Apache Release Distribution Repository
</StagingName>
<StagingUrl>
https://repository.apache.org/content/groups/staging
</StagingUrl>
<StagingUrl>
https://repository.apache.org/content/groups/staging
</StagingUrl>
<!-- skips checkstyle and find bugs -->
<skipCheck>
false
</skipCheck>
</properties>
</properties>
<profiles>
<profiles>
...
@@ -971,6 +974,7 @@
...
@@ -971,6 +974,7 @@
</goals>
</goals>
<phase>
verify
</phase>
<phase>
verify
</phase>
<configuration>
<configuration>
<skip>
${skipCheck}
</skip>
<consoleOutput>
true
</consoleOutput>
<consoleOutput>
true
</consoleOutput>
<includeTestSourceDirectory>
true
</includeTestSourceDirectory>
<includeTestSourceDirectory>
true
</includeTestSourceDirectory>
<configLocation>
src/build/checkstyle.xml
</configLocation>
<configLocation>
src/build/checkstyle.xml
</configLocation>
...
@@ -988,6 +992,7 @@
...
@@ -988,6 +992,7 @@
<!--debug>true</debug -->
<!--debug>true</debug -->
<xmlOutput>
true
</xmlOutput>
<xmlOutput>
true
</xmlOutput>
<failOnError>
false
</failOnError>
<failOnError>
false
</failOnError>
<skip>
${skipCheck}
</skip>
</configuration>
</configuration>
<executions>
<executions>
<execution>
<execution>
...
...
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
View file @
3e695bad
...
@@ -23,6 +23,8 @@ import com.thinkaurelius.titan.core.TitanIndexQuery;
...
@@ -23,6 +23,8 @@ import com.thinkaurelius.titan.core.TitanIndexQuery;
import
com.thinkaurelius.titan.core.TitanProperty
;
import
com.thinkaurelius.titan.core.TitanProperty
;
import
com.thinkaurelius.titan.core.TitanVertex
;
import
com.thinkaurelius.titan.core.TitanVertex
;
import
com.tinkerpop.blueprints.Vertex
;
import
com.tinkerpop.blueprints.Vertex
;
import
com.tinkerpop.gremlin.groovy.Gremlin
;
import
com.tinkerpop.gremlin.java.GremlinPipeline
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.discovery.DiscoveryException
;
import
org.apache.hadoop.metadata.discovery.DiscoveryException
;
import
org.apache.hadoop.metadata.discovery.DiscoveryService
;
import
org.apache.hadoop.metadata.discovery.DiscoveryService
;
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
View file @
3e695bad
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
View file @
3e695bad
...
@@ -199,7 +199,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
...
@@ -199,7 +199,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
break
;
break
;
case
ENUM:
case
ENUM:
createVertexMixedIndex
(
propertyName
,
Integer
.
class
);
createVertexMixedIndex
(
propertyName
,
String
.
class
);
break
;
break
;
case
ARRAY:
case
ARRAY:
...
...
typesystem/src/main/resources/log4j.xml
View file @
3e695bad
...
@@ -85,7 +85,7 @@
...
@@ -85,7 +85,7 @@
</logger>
</logger>
<root>
<root>
<priority
value=
"
debug
"
/>
<priority
value=
"
info
"
/>
<appender-ref
ref=
"console"
/>
<appender-ref
ref=
"console"
/>
</root>
</root>
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment