Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
113313e2
Commit
113313e2
authored
May 26, 2015
by
Suma S
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #101 from hortonworks/master
Merging from master to DAL
parents
cee73f6c
63fd0a25
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
85 additions
and
127 deletions
+85
-127
pom.xml
addons/hive-bridge/pom.xml
+1
-1
HiveMetaStoreBridge.java
...ache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
+25
-35
HiveHook.java
...n/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
+0
-0
HiveDataModelGenerator.java
...he/hadoop/metadata/hive/model/HiveDataModelGenerator.java
+2
-2
Bridge-Hive.twiki
addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki
+9
-1
HiveHookIT.java
...java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
+22
-13
SSLAndKerberosHiveHookIT.java
...e/hadoop/metadata/hive/hook/SSLAndKerberosHiveHookIT.java
+0
-34
SSLHiveHookIT.java
...a/org/apache/hadoop/metadata/hive/hook/SSLHiveHookIT.java
+0
-34
MetadataServiceClient.java
...ava/org/apache/hadoop/metadata/MetadataServiceClient.java
+1
-0
GraphBackedTypeStore.java
...p/metadata/repository/typestore/GraphBackedTypeStore.java
+3
-3
EnumType.java
...org/apache/hadoop/metadata/typesystem/types/EnumType.java
+2
-1
TypesSerialization.scala
.../hadoop/metadata/typesystem/json/TypesSerialization.scala
+1
-1
Servlets.java
...in/java/org/apache/hadoop/metadata/web/util/Servlets.java
+16
-2
EntityJerseyResourceIT.java
...hadoop/metadata/web/resources/EntityJerseyResourceIT.java
+3
-0
No files found.
addons/hive-bridge/pom.xml
View file @
113313e2
...
@@ -33,7 +33,7 @@
...
@@ -33,7 +33,7 @@
<packaging>
jar
</packaging>
<packaging>
jar
</packaging>
<properties>
<properties>
<hive.version>
1.
1
.0
</hive.version>
<hive.version>
1.
2
.0
</hive.version>
<calcite.version>
0.9.2-incubating
</calcite.version>
<calcite.version>
0.9.2-incubating
</calcite.version>
<hadoop.version>
2.6.0
</hadoop.version>
<hadoop.version>
2.6.0
</hadoop.version>
</properties>
</properties>
...
...
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
View file @
113313e2
...
@@ -32,13 +32,10 @@ import org.apache.hadoop.hive.ql.metadata.Table;
...
@@ -32,13 +32,10 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator
;
import
org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance
;
import
org.apache.hadoop.metadata.typesystem.Referenceable
;
import
org.apache.hadoop.metadata.typesystem.Referenceable
;
import
org.apache.hadoop.metadata.typesystem.Struct
;
import
org.apache.hadoop.metadata.typesystem.Struct
;
import
org.apache.hadoop.metadata.typesystem.json.InstanceSerialization
;
import
org.apache.hadoop.metadata.typesystem.json.InstanceSerialization
;
import
org.apache.hadoop.metadata.typesystem.json.Serialization
;
import
org.apache.hadoop.metadata.typesystem.persistence.Id
;
import
org.apache.hadoop.metadata.typesystem.persistence.Id
;
import
org.apache.hadoop.metadata.typesystem.types.TypeSystem
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONException
;
import
org.codehaus.jettison.json.JSONException
;
import
org.codehaus.jettison.json.JSONObject
;
import
org.codehaus.jettison.json.JSONObject
;
...
@@ -55,20 +52,9 @@ import java.util.Set;
...
@@ -55,20 +52,9 @@ import java.util.Set;
*/
*/
public
class
HiveMetaStoreBridge
{
public
class
HiveMetaStoreBridge
{
private
static
final
String
DEFAULT_DGI_URL
=
"http://localhost:21000/"
;
private
static
final
String
DEFAULT_DGI_URL
=
"http://localhost:21000/"
;
public
static
final
String
HIVE_CLUSTER_NAME
=
"hive.cluster.name"
;
public
static
class
Pair
<
S
,
T
>
{
public
static
final
String
DEFAULT_CLUSTER_NAME
=
"primary"
;
public
S
first
;
private
final
String
clusterName
;
public
T
second
;
public
Pair
(
S
first
,
T
second
)
{
this
.
first
=
first
;
this
.
second
=
second
;
}
public
static
<
S
,
T
>
Pair
of
(
S
first
,
T
second
)
{
return
new
Pair
(
first
,
second
);
}
}
public
static
final
String
DGI_URL_PROPERTY
=
"hive.hook.dgi.url"
;
public
static
final
String
DGI_URL_PROPERTY
=
"hive.hook.dgi.url"
;
...
@@ -82,6 +68,7 @@ public class HiveMetaStoreBridge {
...
@@ -82,6 +68,7 @@ public class HiveMetaStoreBridge {
* @param hiveConf
* @param hiveConf
*/
*/
public
HiveMetaStoreBridge
(
HiveConf
hiveConf
)
throws
Exception
{
public
HiveMetaStoreBridge
(
HiveConf
hiveConf
)
throws
Exception
{
clusterName
=
hiveConf
.
get
(
HIVE_CLUSTER_NAME
,
DEFAULT_CLUSTER_NAME
);
hiveClient
=
Hive
.
get
(
hiveConf
);
hiveClient
=
Hive
.
get
(
hiveConf
);
metadataServiceClient
=
new
MetadataServiceClient
(
hiveConf
.
get
(
DGI_URL_PROPERTY
,
DEFAULT_DGI_URL
));
metadataServiceClient
=
new
MetadataServiceClient
(
hiveConf
.
get
(
DGI_URL_PROPERTY
,
DEFAULT_DGI_URL
));
}
}
...
@@ -107,16 +94,20 @@ public class HiveMetaStoreBridge {
...
@@ -107,16 +94,20 @@ public class HiveMetaStoreBridge {
/**
/**
* Gets reference for the database
* Gets reference for the database
*
*
* @param dbName database name
*
* @param databaseName
* @param clusterName cluster name
* @return Reference for database if exists, else null
* @return Reference for database if exists, else null
* @throws Exception
* @throws Exception
*/
*/
private
Referenceable
getDatabaseReference
(
String
d
b
Name
)
throws
Exception
{
private
Referenceable
getDatabaseReference
(
String
d
atabaseName
,
String
cluster
Name
)
throws
Exception
{
LOG
.
debug
(
"Getting reference for database {}"
,
d
b
Name
);
LOG
.
debug
(
"Getting reference for database {}"
,
d
atabase
Name
);
String
typeName
=
HiveDataTypes
.
HIVE_DB
.
getName
();
String
typeName
=
HiveDataTypes
.
HIVE_DB
.
getName
();
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
JSONArray
results
=
dgiClient
.
rawSearch
(
typeName
,
"name"
,
dbName
);
String
dslQuery
=
String
.
format
(
"%s where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_DB
.
getName
(),
databaseName
,
clusterName
);
JSONArray
results
=
dgiClient
.
searchByDSL
(
dslQuery
);
if
(
results
.
length
()
==
0
)
{
if
(
results
.
length
()
==
0
)
{
return
null
;
return
null
;
}
else
{
}
else
{
...
@@ -126,13 +117,14 @@ public class HiveMetaStoreBridge {
...
@@ -126,13 +117,14 @@ public class HiveMetaStoreBridge {
}
}
public
Referenceable
registerDatabase
(
String
databaseName
)
throws
Exception
{
public
Referenceable
registerDatabase
(
String
databaseName
)
throws
Exception
{
Referenceable
dbRef
=
getDatabaseReference
(
databaseName
);
Referenceable
dbRef
=
getDatabaseReference
(
databaseName
,
clusterName
);
if
(
dbRef
==
null
)
{
if
(
dbRef
==
null
)
{
LOG
.
info
(
"Importing objects from databaseName : "
+
databaseName
);
LOG
.
info
(
"Importing objects from databaseName : "
+
databaseName
);
Database
hiveDB
=
hiveClient
.
getDatabase
(
databaseName
);
Database
hiveDB
=
hiveClient
.
getDatabase
(
databaseName
);
dbRef
=
new
Referenceable
(
HiveDataTypes
.
HIVE_DB
.
getName
());
dbRef
=
new
Referenceable
(
HiveDataTypes
.
HIVE_DB
.
getName
());
dbRef
.
set
(
"name"
,
hiveDB
.
getName
());
dbRef
.
set
(
"name"
,
hiveDB
.
getName
());
dbRef
.
set
(
"clusterName"
,
clusterName
);
dbRef
.
set
(
"description"
,
hiveDB
.
getDescription
());
dbRef
.
set
(
"description"
,
hiveDB
.
getDescription
());
dbRef
.
set
(
"locationUri"
,
hiveDB
.
getLocationUri
());
dbRef
.
set
(
"locationUri"
,
hiveDB
.
getLocationUri
());
dbRef
.
set
(
"parameters"
,
hiveDB
.
getParameters
());
dbRef
.
set
(
"parameters"
,
hiveDB
.
getParameters
());
...
@@ -168,7 +160,7 @@ public class HiveMetaStoreBridge {
...
@@ -168,7 +160,7 @@ public class HiveMetaStoreBridge {
Referenceable
tableReferenceable
=
registerTable
(
databaseReferenceable
,
databaseName
,
tableName
);
Referenceable
tableReferenceable
=
registerTable
(
databaseReferenceable
,
databaseName
,
tableName
);
// Import Partitions
// Import Partitions
Referenceable
sdReferenceable
=
getSDForTable
(
database
Referenceabl
e
,
tableName
);
Referenceable
sdReferenceable
=
getSDForTable
(
database
Nam
e
,
tableName
);
importPartitions
(
databaseName
,
tableName
,
databaseReferenceable
,
tableReferenceable
,
sdReferenceable
);
importPartitions
(
databaseName
,
tableName
,
databaseReferenceable
,
tableReferenceable
,
sdReferenceable
);
// Import Indexes
// Import Indexes
...
@@ -179,28 +171,26 @@ public class HiveMetaStoreBridge {
...
@@ -179,28 +171,26 @@ public class HiveMetaStoreBridge {
/**
/**
* Gets reference for the table
* Gets reference for the table
*
*
* @param db
Ref
* @param db
Name
* @param tableName table name
* @param tableName table name
* @return table reference if exists, else null
* @return table reference if exists, else null
* @throws Exception
* @throws Exception
*/
*/
private
Referenceable
getTableReference
(
Referenceable
dbRef
,
String
tableName
)
throws
Exception
{
private
Referenceable
getTableReference
(
String
dbName
,
String
tableName
)
throws
Exception
{
LOG
.
debug
(
"Getting reference for table {}.{}"
,
db
Ref
,
tableName
);
LOG
.
debug
(
"Getting reference for table {}.{}"
,
db
Name
,
tableName
);
String
typeName
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
String
typeName
=
HiveDataTypes
.
HIVE_TABLE
.
getName
();
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
//todo DSL support for reference doesn't work. is the usage right?
String
query
=
String
.
format
(
"%s where name = '%s', dbName where name = '%s' and clusterName = '%s'"
,
// String query = String.format("%s where dbName = \"%s\" and tableName = \"%s\"", typeName, dbRef.getId().id,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
,
dbName
,
clusterName
);
// tableName);
String
query
=
String
.
format
(
"%s where name = \"%s\""
,
typeName
,
tableName
);
JSONArray
results
=
dgiClient
.
searchByDSL
(
query
);
JSONArray
results
=
dgiClient
.
searchByDSL
(
query
);
if
(
results
.
length
()
==
0
)
{
if
(
results
.
length
()
==
0
)
{
return
null
;
return
null
;
}
else
{
}
else
{
//There should be just one instance with the given name
//There should be just one instance with the given name
String
guid
=
getGuidFromDSLResponse
(
results
.
getJSONObject
(
0
));
String
guid
=
getGuidFromDSLResponse
(
results
.
getJSONObject
(
0
));
LOG
.
debug
(
"Got reference for table {}.{} = {}"
,
db
Ref
,
tableName
,
guid
);
LOG
.
debug
(
"Got reference for table {}.{} = {}"
,
db
Name
,
tableName
,
guid
);
return
new
Referenceable
(
guid
,
typeName
,
null
);
return
new
Referenceable
(
guid
,
typeName
,
null
);
}
}
}
}
...
@@ -209,10 +199,10 @@ public class HiveMetaStoreBridge {
...
@@ -209,10 +199,10 @@ public class HiveMetaStoreBridge {
return
jsonObject
.
getJSONObject
(
"$id$"
).
getString
(
"id"
);
return
jsonObject
.
getJSONObject
(
"$id$"
).
getString
(
"id"
);
}
}
private
Referenceable
getSDForTable
(
Referenceable
dbRef
,
String
tableName
)
throws
Exception
{
private
Referenceable
getSDForTable
(
String
dbName
,
String
tableName
)
throws
Exception
{
Referenceable
tableRef
=
getTableReference
(
db
Ref
,
tableName
);
Referenceable
tableRef
=
getTableReference
(
db
Name
,
tableName
);
if
(
tableRef
==
null
)
{
if
(
tableRef
==
null
)
{
throw
new
IllegalArgumentException
(
"Table "
+
db
Ref
+
"."
+
tableName
+
" doesn't exist"
);
throw
new
IllegalArgumentException
(
"Table "
+
db
Name
+
"."
+
tableName
+
" doesn't exist"
);
}
}
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
MetadataServiceClient
dgiClient
=
getMetadataServiceClient
();
...
@@ -228,7 +218,7 @@ public class HiveMetaStoreBridge {
...
@@ -228,7 +218,7 @@ public class HiveMetaStoreBridge {
public
Referenceable
registerTable
(
Referenceable
dbReference
,
String
dbName
,
String
tableName
)
throws
Exception
{
public
Referenceable
registerTable
(
Referenceable
dbReference
,
String
dbName
,
String
tableName
)
throws
Exception
{
LOG
.
info
(
"Attempting to register table ["
+
tableName
+
"]"
);
LOG
.
info
(
"Attempting to register table ["
+
tableName
+
"]"
);
Referenceable
tableRef
=
getTableReference
(
db
Referenc
e
,
tableName
);
Referenceable
tableRef
=
getTableReference
(
db
Nam
e
,
tableName
);
if
(
tableRef
==
null
)
{
if
(
tableRef
==
null
)
{
LOG
.
info
(
"Importing objects from "
+
dbName
+
"."
+
tableName
);
LOG
.
info
(
"Importing objects from "
+
dbName
+
"."
+
tableName
);
...
...
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
View file @
113313e2
This diff is collapsed.
Click to expand it.
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java
View file @
113313e2
...
@@ -280,6 +280,8 @@ public class HiveDataModelGenerator {
...
@@ -280,6 +280,8 @@ public class HiveDataModelGenerator {
AttributeDefinition
[]
attributeDefinitions
=
new
AttributeDefinition
[]{
AttributeDefinition
[]
attributeDefinitions
=
new
AttributeDefinition
[]{
new
AttributeDefinition
(
"name"
,
DataTypes
.
STRING_TYPE
.
getName
(),
new
AttributeDefinition
(
"name"
,
DataTypes
.
STRING_TYPE
.
getName
(),
Multiplicity
.
REQUIRED
,
false
,
null
),
Multiplicity
.
REQUIRED
,
false
,
null
),
new
AttributeDefinition
(
"clusterName"
,
DataTypes
.
STRING_TYPE
.
getName
(),
Multiplicity
.
REQUIRED
,
false
,
null
),
new
AttributeDefinition
(
"description"
,
DataTypes
.
STRING_TYPE
.
getName
(),
new
AttributeDefinition
(
"description"
,
DataTypes
.
STRING_TYPE
.
getName
(),
Multiplicity
.
OPTIONAL
,
false
,
null
),
Multiplicity
.
OPTIONAL
,
false
,
null
),
new
AttributeDefinition
(
"locationUri"
,
DataTypes
.
STRING_TYPE
.
getName
(),
new
AttributeDefinition
(
"locationUri"
,
DataTypes
.
STRING_TYPE
.
getName
(),
...
@@ -322,8 +324,6 @@ public class HiveDataModelGenerator {
...
@@ -322,8 +324,6 @@ public class HiveDataModelGenerator {
AttributeDefinition
[]
attributeDefinitions
=
new
AttributeDefinition
[]{
AttributeDefinition
[]
attributeDefinitions
=
new
AttributeDefinition
[]{
new
AttributeDefinition
(
"name"
,
DataTypes
.
STRING_TYPE
.
getName
(),
new
AttributeDefinition
(
"name"
,
DataTypes
.
STRING_TYPE
.
getName
(),
Multiplicity
.
REQUIRED
,
false
,
null
),
Multiplicity
.
REQUIRED
,
false
,
null
),
//new AttributeDefinition("type", DefinedTypes.HIVE_TYPE.getName(), Multiplicity
// .REQUIRED, false, null),
new
AttributeDefinition
(
"type"
,
DataTypes
.
STRING_TYPE
.
getName
(),
new
AttributeDefinition
(
"type"
,
DataTypes
.
STRING_TYPE
.
getName
(),
Multiplicity
.
REQUIRED
,
false
,
null
),
Multiplicity
.
REQUIRED
,
false
,
null
),
new
AttributeDefinition
(
"comment"
,
DataTypes
.
STRING_TYPE
.
getName
(),
new
AttributeDefinition
(
"comment"
,
DataTypes
.
STRING_TYPE
.
getName
(),
...
...
addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki
View file @
113313e2
...
@@ -29,6 +29,10 @@ hive conf directory:
...
@@ -29,6 +29,10 @@ hive conf directory:
<name>hive.hook.dgi.url</name>
<name>hive.hook.dgi.url</name>
<value>http://localhost:21000/</value>
<value>http://localhost:21000/</value>
</property>
</property>
<property>
<name>hive.cluster.name</name>
<value>primary</value>
</property>
</verbatim>
</verbatim>
Usage: <dgi package>/bin/import-hive.sh. The logs are in <dgi package>/logs/import-hive.log
Usage: <dgi package>/bin/import-hive.sh. The logs are in <dgi package>/logs/import-hive.log
...
@@ -44,12 +48,16 @@ The hook submits the request to a thread pool executor to avoid blocking the com
...
@@ -44,12 +48,16 @@ The hook submits the request to a thread pool executor to avoid blocking the com
<value>org.apache.hadoop.metadata.hive.hook.HiveHook</value>
<value>org.apache.hadoop.metadata.hive.hook.HiveHook</value>
</property>
</property>
</verbatim>
</verbatim>
* Add the following propert
y
in hive-ste.xml with the DGI endpoint for your set-up
* Add the following propert
ies
in hive-ste.xml with the DGI endpoint for your set-up
<verbatim>
<verbatim>
<property>
<property>
<name>hive.hook.dgi.url</name>
<name>hive.hook.dgi.url</name>
<value>http://localhost:21000/</value>
<value>http://localhost:21000/</value>
</property>
</property>
<property>
<name>hive.cluster.name</name>
<value>primary</value>
</property>
</verbatim>
</verbatim>
* Add 'export HIVE_AUX_JARS_PATH=<dgi package>/hook/hive' in hive-env.sh
* Add 'export HIVE_AUX_JARS_PATH=<dgi package>/hook/hive' in hive-env.sh
...
...
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
View file @
113313e2
...
@@ -24,16 +24,15 @@ import org.apache.hadoop.hive.ql.Driver;
...
@@ -24,16 +24,15 @@ import org.apache.hadoop.hive.ql.Driver;
import
org.apache.hadoop.hive.ql.session.SessionState
;
import
org.apache.hadoop.hive.ql.session.SessionState
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge
;
import
org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge
;
import
org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONObject
;
import
org.testng.Assert
;
import
org.testng.Assert
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.Test
;
import
org.testng.annotations.Test
;
public
class
HiveHookIT
{
public
class
HiveHookIT
{
private
static
final
String
DGI_URL
=
"http://localhost:21000/"
;
private
static
final
String
DGI_URL
=
"http://localhost:21000/"
;
private
static
final
String
CLUSTER_NAME
=
"test"
;
private
Driver
driver
;
private
Driver
driver
;
private
MetadataServiceClient
dgiCLient
;
private
MetadataServiceClient
dgiCLient
;
private
SessionState
ss
;
private
SessionState
ss
;
...
@@ -59,6 +58,7 @@ public class HiveHookIT {
...
@@ -59,6 +58,7 @@ public class HiveHookIT {
hiveConf
.
set
(
HiveMetaStoreBridge
.
DGI_URL_PROPERTY
,
DGI_URL
);
hiveConf
.
set
(
HiveMetaStoreBridge
.
DGI_URL_PROPERTY
,
DGI_URL
);
hiveConf
.
set
(
"javax.jdo.option.ConnectionURL"
,
"jdbc:derby:./target/metastore_db;create=true"
);
hiveConf
.
set
(
"javax.jdo.option.ConnectionURL"
,
"jdbc:derby:./target/metastore_db;create=true"
);
hiveConf
.
set
(
"hive.hook.dgi.synchronous"
,
"true"
);
hiveConf
.
set
(
"hive.hook.dgi.synchronous"
,
"true"
);
hiveConf
.
set
(
HiveMetaStoreBridge
.
HIVE_CLUSTER_NAME
,
CLUSTER_NAME
);
return
hiveConf
;
return
hiveConf
;
}
}
...
@@ -82,11 +82,11 @@ public class HiveHookIT {
...
@@ -82,11 +82,11 @@ public class HiveHookIT {
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
dbName
+
"."
+
tableName
+
"(id int, name string)"
);
runCommand
(
"create table "
+
dbName
+
"."
+
tableName
+
"(id int, name string)"
);
assertTableIsRegistered
(
tableName
);
assertTableIsRegistered
(
dbName
,
tableName
);
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
assertTableIsRegistered
(
tableName
);
assertTableIsRegistered
(
"default"
,
tableName
);
//Create table where database doesn't exist, will create database instance as well
//Create table where database doesn't exist, will create database instance as well
assertDatabaseIsRegistered
(
"default"
);
assertDatabaseIsRegistered
(
"default"
);
...
@@ -97,24 +97,33 @@ public class HiveHookIT {
...
@@ -97,24 +97,33 @@ public class HiveHookIT {
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
String
new
TableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
ctas
TableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
query
=
"create table "
+
new
TableName
+
" as select * from "
+
tableName
;
String
query
=
"create table "
+
ctas
TableName
+
" as select * from "
+
tableName
;
runCommand
(
query
);
runCommand
(
query
);
assertTableIsRegistered
(
new
TableName
);
assertTableIsRegistered
(
"default"
,
ctas
TableName
);
assert
InstanceIsRegistered
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
"queryText"
,
query
);
assert
ProcessIsRegistered
(
query
);
}
}
private
void
assertTableIsRegistered
(
String
tableName
)
throws
Exception
{
private
void
assertProcessIsRegistered
(
String
queryStr
)
throws
Exception
{
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
"name"
,
tableName
);
String
dslQuery
=
String
.
format
(
"%s where queryText = '%s'"
,
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
queryStr
);
assertInstanceIsRegistered
(
dslQuery
);
}
private
void
assertTableIsRegistered
(
String
dbName
,
String
tableName
)
throws
Exception
{
String
query
=
String
.
format
(
"%s where name = '%s', dbName where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
tableName
,
dbName
,
CLUSTER_NAME
);
assertInstanceIsRegistered
(
query
);
}
}
private
void
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
private
void
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_DB
.
getName
(),
"name"
,
dbName
);
String
query
=
String
.
format
(
"%s where name = '%s' and clusterName = '%s'"
,
HiveDataTypes
.
HIVE_DB
.
getName
(),
dbName
,
CLUSTER_NAME
);
assertInstanceIsRegistered
(
query
);
}
}
private
void
assertInstanceIsRegistered
(
String
typeName
,
String
colName
,
String
colValue
)
throws
Exception
{
private
void
assertInstanceIsRegistered
(
String
dslQuery
)
throws
Exception
{
JSONArray
results
=
dgiCLient
.
rawSearch
(
typeName
,
colName
,
colValue
);
JSONArray
results
=
dgiCLient
.
searchByDSL
(
dslQuery
);
Assert
.
assertEquals
(
results
.
length
(),
1
);
Assert
.
assertEquals
(
results
.
length
(),
1
);
}
}
}
}
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLAndKerberosHiveHookIT.java
View file @
113313e2
...
@@ -205,40 +205,6 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
...
@@ -205,40 +205,6 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
assertDatabaseIsRegistered
(
dbName
);
assertDatabaseIsRegistered
(
dbName
);
}
}
@Test
public
void
testCreateTable
()
throws
Exception
{
String
dbName
=
"db"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create database "
+
dbName
);
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
dbName
+
"."
+
tableName
+
"(id int, name string)"
);
assertTableIsRegistered
(
tableName
);
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
assertTableIsRegistered
(
tableName
);
//Create table where database doesn't exist, will create database instance as well
assertDatabaseIsRegistered
(
"default"
);
}
@Test
public
void
testCTAS
()
throws
Exception
{
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
String
newTableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
query
=
"create table "
+
newTableName
+
" as select * from "
+
tableName
;
runCommand
(
query
);
assertTableIsRegistered
(
newTableName
);
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
"queryText"
,
query
);
}
private
void
assertTableIsRegistered
(
String
tableName
)
throws
Exception
{
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
"name"
,
tableName
);
}
private
void
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
private
void
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_DB
.
getName
(),
"name"
,
dbName
);
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_DB
.
getName
(),
"name"
,
dbName
);
}
}
...
...
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLHiveHookIT.java
View file @
113313e2
...
@@ -208,40 +208,6 @@ public class SSLHiveHookIT {
...
@@ -208,40 +208,6 @@ public class SSLHiveHookIT {
assertDatabaseIsRegistered
(
dbName
);
assertDatabaseIsRegistered
(
dbName
);
}
}
@Test
public
void
testCreateTable
()
throws
Exception
{
String
dbName
=
"db"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create database "
+
dbName
);
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
dbName
+
"."
+
tableName
+
"(id int, name string)"
);
assertTableIsRegistered
(
tableName
);
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
assertTableIsRegistered
(
tableName
);
//Create table where database doesn't exist, will create database instance as well
assertDatabaseIsRegistered
(
"default"
);
}
@Test
public
void
testCTAS
()
throws
Exception
{
String
tableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
);
String
newTableName
=
"table"
+
RandomStringUtils
.
randomAlphanumeric
(
5
).
toLowerCase
();
String
query
=
"create table "
+
newTableName
+
" as select * from "
+
tableName
;
runCommand
(
query
);
assertTableIsRegistered
(
newTableName
);
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_PROCESS
.
getName
(),
"queryText"
,
query
);
}
private
void
assertTableIsRegistered
(
String
tableName
)
throws
Exception
{
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_TABLE
.
getName
(),
"name"
,
tableName
);
}
private
void
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
private
void
assertDatabaseIsRegistered
(
String
dbName
)
throws
Exception
{
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_DB
.
getName
(),
"name"
,
dbName
);
assertInstanceIsRegistered
(
HiveDataTypes
.
HIVE_DB
.
getName
(),
"name"
,
dbName
);
}
}
...
...
client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
View file @
113313e2
...
@@ -55,6 +55,7 @@ public class MetadataServiceClient {
...
@@ -55,6 +55,7 @@ public class MetadataServiceClient {
public
static
final
String
DEFINITION
=
"definition"
;
public
static
final
String
DEFINITION
=
"definition"
;
public
static
final
String
ERROR
=
"error"
;
public
static
final
String
ERROR
=
"error"
;
public
static
final
String
STACKTRACE
=
"stackTrace"
;
public
static
final
String
REQUEST_ID
=
"requestId"
;
public
static
final
String
REQUEST_ID
=
"requestId"
;
public
static
final
String
RESULTS
=
"results"
;
public
static
final
String
RESULTS
=
"results"
;
public
static
final
String
COUNT
=
"count"
;
public
static
final
String
COUNT
=
"count"
;
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java
View file @
113313e2
...
@@ -53,8 +53,8 @@ import java.util.Iterator;
...
@@ -53,8 +53,8 @@ import java.util.Iterator;
import
java.util.List
;
import
java.util.List
;
public
class
GraphBackedTypeStore
implements
ITypeStore
{
public
class
GraphBackedTypeStore
implements
ITypeStore
{
public
static
final
String
VERTEX_TYPE
=
"typeSystem"
;
public
static
final
String
VERTEX_TYPE
=
Constants
.
INTERNAL_PROPERTY_KEY_PREFIX
+
"typeSystem"
;
private
static
final
String
PROPERTY_PREFIX
=
"type."
;
private
static
final
String
PROPERTY_PREFIX
=
Constants
.
INTERNAL_PROPERTY_KEY_PREFIX
+
"type."
;
public
static
final
String
SUPERTYPE_EDGE_LABEL
=
PROPERTY_PREFIX
+
".supertype"
;
public
static
final
String
SUPERTYPE_EDGE_LABEL
=
PROPERTY_PREFIX
+
".supertype"
;
public
static
final
String
SUBTYPE_EDGE_LABEL
=
PROPERTY_PREFIX
+
".subtype"
;
public
static
final
String
SUBTYPE_EDGE_LABEL
=
PROPERTY_PREFIX
+
".subtype"
;
...
@@ -188,7 +188,7 @@ public class GraphBackedTypeStore implements ITypeStore {
...
@@ -188,7 +188,7 @@ public class GraphBackedTypeStore implements ITypeStore {
break
;
break
;
default
:
default
:
throw
new
IllegalArgumentException
(
"
Unhandled type category
"
+
attrDataType
.
getTypeCategory
());
throw
new
IllegalArgumentException
(
"
Attribute cannot reference instances of type :
"
+
attrDataType
.
getTypeCategory
());
}
}
for
(
IDataType
attrType
:
attrDataTypes
)
{
for
(
IDataType
attrType
:
attrDataTypes
)
{
...
...
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumType.java
View file @
113313e2
...
@@ -21,6 +21,7 @@ package org.apache.hadoop.metadata.typesystem.types;
...
@@ -21,6 +21,7 @@ package org.apache.hadoop.metadata.typesystem.types;
import
com.google.common.collect.ImmutableCollection
;
import
com.google.common.collect.ImmutableCollection
;
import
com.google.common.collect.ImmutableMap
;
import
com.google.common.collect.ImmutableMap
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataException
;
import
scala.math.BigInt
;
public
class
EnumType
extends
AbstractDataType
<
EnumValue
>
{
public
class
EnumType
extends
AbstractDataType
<
EnumValue
>
{
...
@@ -54,7 +55,7 @@ public class EnumType extends AbstractDataType<EnumValue> {
...
@@ -54,7 +55,7 @@ public class EnumType extends AbstractDataType<EnumValue> {
EnumValue
e
=
null
;
EnumValue
e
=
null
;
if
(
val
instanceof
EnumValue
)
{
if
(
val
instanceof
EnumValue
)
{
e
=
valueMap
.
get
(((
EnumValue
)
val
).
value
);
e
=
valueMap
.
get
(((
EnumValue
)
val
).
value
);
}
else
if
(
val
instanceof
Integer
)
{
}
else
if
(
val
instanceof
Integer
||
val
instanceof
BigInt
)
{
e
=
ordinalMap
.
get
(
val
);
e
=
ordinalMap
.
get
(
val
);
}
else
if
(
val
instanceof
String
)
{
}
else
if
(
val
instanceof
String
)
{
e
=
valueMap
.
get
(
val
);
e
=
valueMap
.
get
(
val
);
...
...
typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerialization.scala
View file @
113313e2
...
@@ -126,7 +126,7 @@ object TypesSerialization {
...
@@ -126,7 +126,7 @@ object TypesSerialization {
private
def
convertAttributeInfoToAttributeDef
(
aInfo
:
AttributeInfo
)
=
{
private
def
convertAttributeInfoToAttributeDef
(
aInfo
:
AttributeInfo
)
=
{
new
AttributeDefinition
(
aInfo
.
name
,
aInfo
.
dataType
().
getName
,
aInfo
.
multiplicity
,
new
AttributeDefinition
(
aInfo
.
name
,
aInfo
.
dataType
().
getName
,
aInfo
.
multiplicity
,
aInfo
.
isComposite
,
aInfo
.
reverseAttributeName
)
aInfo
.
isComposite
,
aInfo
.
isUnique
,
aInfo
.
isIndexable
,
aInfo
.
reverseAttributeName
)
}
}
private
def
convertEnumTypeToEnumTypeDef
(
et
:
EnumType
)
=
{
private
def
convertEnumTypeToEnumTypeDef
(
et
:
EnumType
)
=
{
...
...
webapp/src/main/java/org/apache/hadoop/metadata/web/util/Servlets.java
View file @
113313e2
...
@@ -30,6 +30,7 @@ import javax.servlet.http.HttpServletRequest;
...
@@ -30,6 +30,7 @@ import javax.servlet.http.HttpServletRequest;
import
javax.ws.rs.core.MediaType
;
import
javax.ws.rs.core.MediaType
;
import
javax.ws.rs.core.Response
;
import
javax.ws.rs.core.Response
;
import
java.io.IOException
;
import
java.io.IOException
;
import
java.io.PrintWriter
;
import
java.io.StringWriter
;
import
java.io.StringWriter
;
/**
/**
...
@@ -98,7 +99,20 @@ public final class Servlets {
...
@@ -98,7 +99,20 @@ public final class Servlets {
}
}
public
static
Response
getErrorResponse
(
Throwable
e
,
Response
.
Status
status
)
{
public
static
Response
getErrorResponse
(
Throwable
e
,
Response
.
Status
status
)
{
return
getErrorResponse
(
e
.
getMessage
(),
status
);
Response
response
=
getErrorResponse
(
e
.
getMessage
(),
status
);
JSONObject
responseJson
=
(
JSONObject
)
response
.
getEntity
();
try
{
responseJson
.
put
(
MetadataServiceClient
.
STACKTRACE
,
printStackTrace
(
e
));
}
catch
(
JSONException
e1
)
{
LOG
.
warn
(
"Could not construct error Json rensponse"
,
e1
);
}
return
response
;
}
private
static
String
printStackTrace
(
Throwable
t
)
{
StringWriter
sw
=
new
StringWriter
();
t
.
printStackTrace
(
new
PrintWriter
(
sw
));
return
sw
.
toString
();
}
}
public
static
Response
getErrorResponse
(
String
message
,
Response
.
Status
status
)
{
public
static
Response
getErrorResponse
(
String
message
,
Response
.
Status
status
)
{
...
@@ -108,7 +122,7 @@ public final class Servlets {
...
@@ -108,7 +122,7 @@ public final class Servlets {
errorJson
.
put
(
MetadataServiceClient
.
ERROR
,
errorEntity
);
errorJson
.
put
(
MetadataServiceClient
.
ERROR
,
errorEntity
);
errorEntity
=
errorJson
;
errorEntity
=
errorJson
;
}
catch
(
JSONException
jsonE
)
{
}
catch
(
JSONException
jsonE
)
{
LOG
.
warn
(
"Could not construct error Json rensponse"
);
LOG
.
warn
(
"Could not construct error Json rensponse"
,
jsonE
);
}
}
return
Response
return
Response
.
status
(
status
)
.
status
(
status
)
...
...
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
View file @
113313e2
...
@@ -199,6 +199,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
...
@@ -199,6 +199,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
JSONObject
response
=
new
JSONObject
(
responseAsString
);
JSONObject
response
=
new
JSONObject
(
responseAsString
);
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
ERROR
));
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
ERROR
));
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
STACKTRACE
));
}
}
@Test
(
dependsOnMethods
=
"testSubmitEntity"
)
@Test
(
dependsOnMethods
=
"testSubmitEntity"
)
...
@@ -238,6 +239,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
...
@@ -238,6 +239,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
JSONObject
response
=
new
JSONObject
(
responseAsString
);
JSONObject
response
=
new
JSONObject
(
responseAsString
);
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
ERROR
));
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
ERROR
));
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
STACKTRACE
));
}
}
@Test
@Test
...
@@ -395,6 +397,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
...
@@ -395,6 +397,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
JSONObject
response
=
new
JSONObject
(
responseAsString
);
JSONObject
response
=
new
JSONObject
(
responseAsString
);
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
ERROR
));
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
ERROR
));
Assert
.
assertNotNull
(
response
.
get
(
MetadataServiceClient
.
STACKTRACE
));
}
}
private
void
createHiveTypes
()
throws
Exception
{
private
void
createHiveTypes
()
throws
Exception
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment