Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
74c54aa9
Commit
74c54aa9
authored
9 years ago
by
Arpit Gupta
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #124 from sumashivaprasad/BUG_37105
Fixed eager initilization of singletons, added fix for hive table com…
parents
6dd20ef7
4f7e672c
master
No related merge requests found
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
34 additions
and
20 deletions
+34
-20
HiveMetaStoreBridge.java
...ache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
+8
-3
HiveDataModelGenerator.java
...he/hadoop/metadata/hive/model/HiveDataModelGenerator.java
+9
-3
HiveHookIT.java
...java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
+8
-1
RepositoryMetadataModule.java
.../org/apache/hadoop/metadata/RepositoryMetadataModule.java
+7
-7
DefaultMetadataService.java
...ache/hadoop/metadata/services/DefaultMetadataService.java
+2
-1
HiveLineageServiceTest.java
...che/hadoop/metadata/discovery/HiveLineageServiceTest.java
+0
-5
No files found.
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
View file @
74c54aa9
...
...
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.metastore.api.Index;
import
org.apache.hadoop.hive.metastore.api.Order
;
import
org.apache.hadoop.hive.metastore.api.SerDeInfo
;
import
org.apache.hadoop.hive.metastore.api.StorageDescriptor
;
import
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants
;
import
org.apache.hadoop.hive.ql.metadata.Hive
;
import
org.apache.hadoop.hive.ql.metadata.Partition
;
import
org.apache.hadoop.hive.ql.metadata.Table
;
...
...
@@ -263,11 +264,13 @@ public class HiveMetaStoreBridge {
tableRef
=
new
Referenceable
(
HiveDataTypes
.
HIVE_TABLE
.
getName
());
tableRef
.
set
(
"name"
,
hiveTable
.
getTableName
());
tableRef
.
set
(
"owner"
,
hiveTable
.
getOwner
());
//todo fix
tableRef
.
set
(
"createTime"
,
hiveTable
.
get
LastAccessTime
(
));
tableRef
.
set
(
"createTime"
,
hiveTable
.
get
Metadata
().
getProperty
(
hive_metastoreConstants
.
DDL_TIME
));
tableRef
.
set
(
"lastAccessTime"
,
hiveTable
.
getLastAccessTime
());
tableRef
.
set
(
"retention"
,
hiveTable
.
getRetention
());
tableRef
.
set
(
HiveDataModelGenerator
.
COMMENT
,
hiveTable
.
getParameters
().
get
(
HiveDataModelGenerator
.
COMMENT
));
// add reference to the database
tableRef
.
set
(
"dbName"
,
dbReference
);
...
...
@@ -416,6 +419,8 @@ public class HiveMetaStoreBridge {
serdeInfoStruct
.
set
(
"parameters"
,
serdeInfo
.
getParameters
());
sdReferenceable
.
set
(
"serdeInfo"
,
serdeInfoStruct
);
sdReferenceable
.
set
(
HiveDataModelGenerator
.
STORAGE_NUM_BUCKETS
,
storageDesc
.
getNumBuckets
());
sdReferenceable
.
set
(
HiveDataModelGenerator
.
STORAGE_IS_STORED_AS_SUB_DIRS
,
storageDesc
.
isStoredAsSubDirectories
());
// Will need to revisit this after we fix typesystem.
/*
...
...
@@ -480,7 +485,7 @@ public class HiveMetaStoreBridge {
Referenceable
colReferenceable
=
new
Referenceable
(
HiveDataTypes
.
HIVE_COLUMN
.
getName
());
colReferenceable
.
set
(
"name"
,
fs
.
getName
());
colReferenceable
.
set
(
"type"
,
fs
.
getType
());
colReferenceable
.
set
(
"comment"
,
fs
.
getComment
());
colReferenceable
.
set
(
HiveDataModelGenerator
.
COMMENT
,
fs
.
getComment
());
colList
.
add
(
createInstance
(
colReferenceable
));
}
...
...
This diff is collapsed.
Click to expand it.
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java
View file @
74c54aa9
...
...
@@ -55,6 +55,10 @@ public class HiveDataModelGenerator {
private
final
Map
<
String
,
EnumTypeDefinition
>
enumTypeDefinitionMap
;
private
final
Map
<
String
,
StructTypeDefinition
>
structTypeDefinitionMap
;
public
static
final
String
COMMENT
=
"comment"
;
public
static
final
String
STORAGE_NUM_BUCKETS
=
"numBuckets"
;
public
static
final
String
STORAGE_IS_STORED_AS_SUB_DIRS
=
"storedAsSubDirectories"
;
public
HiveDataModelGenerator
()
{
classTypeDefinitions
=
new
HashMap
<>();
enumTypeDefinitionMap
=
new
HashMap
<>();
...
...
@@ -237,7 +241,7 @@ public class HiveDataModelGenerator {
Multiplicity
.
OPTIONAL
,
false
,
null
),
new
AttributeDefinition
(
"compressed"
,
DataTypes
.
BOOLEAN_TYPE
.
getName
(),
Multiplicity
.
REQUIRED
,
false
,
null
),
new
AttributeDefinition
(
"numBuckets"
,
DataTypes
.
INT_TYPE
.
getName
(),
new
AttributeDefinition
(
STORAGE_NUM_BUCKETS
,
DataTypes
.
INT_TYPE
.
getName
(),
Multiplicity
.
OPTIONAL
,
false
,
null
),
new
AttributeDefinition
(
"serdeInfo"
,
HiveDataTypes
.
HIVE_SERDE
.
getName
(),
Multiplicity
.
OPTIONAL
,
false
,
null
),
...
...
@@ -251,7 +255,7 @@ public class HiveDataModelGenerator {
Multiplicity
.
OPTIONAL
,
false
,
null
),
//new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(),
// Multiplicity.OPTIONAL, false, null),
new
AttributeDefinition
(
"storedAsSubDirectories"
,
DataTypes
.
BOOLEAN_TYPE
.
getName
(),
new
AttributeDefinition
(
STORAGE_IS_STORED_AS_SUB_DIRS
,
DataTypes
.
BOOLEAN_TYPE
.
getName
(),
Multiplicity
.
OPTIONAL
,
false
,
null
),
};
...
...
@@ -326,7 +330,7 @@ public class HiveDataModelGenerator {
Multiplicity
.
REQUIRED
,
false
,
null
),
new
AttributeDefinition
(
"type"
,
DataTypes
.
STRING_TYPE
.
getName
(),
Multiplicity
.
REQUIRED
,
false
,
null
),
new
AttributeDefinition
(
"comment"
,
DataTypes
.
STRING_TYPE
.
getName
(),
new
AttributeDefinition
(
COMMENT
,
DataTypes
.
STRING_TYPE
.
getName
(),
Multiplicity
.
OPTIONAL
,
false
,
null
),
};
HierarchicalTypeDefinition
<
ClassType
>
definition
=
...
...
@@ -377,6 +381,8 @@ public class HiveDataModelGenerator {
Multiplicity
.
OPTIONAL
,
false
,
null
),
new
AttributeDefinition
(
"lastAccessTime"
,
DataTypes
.
INT_TYPE
.
getName
(),
Multiplicity
.
OPTIONAL
,
false
,
null
),
new
AttributeDefinition
(
COMMENT
,
DataTypes
.
STRING_TYPE
.
getName
(),
Multiplicity
.
OPTIONAL
,
false
,
null
),
new
AttributeDefinition
(
"retention"
,
DataTypes
.
INT_TYPE
.
getName
(),
Multiplicity
.
OPTIONAL
,
false
,
null
),
new
AttributeDefinition
(
"sd"
,
HiveDataTypes
.
HIVE_STORAGEDESC
.
getName
(),
...
...
This diff is collapsed.
Click to expand it.
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
View file @
74c54aa9
...
...
@@ -21,12 +21,15 @@ package org.apache.hadoop.metadata.hive.hook;
import
org.apache.commons.lang.RandomStringUtils
;
import
org.apache.hadoop.hive.conf.HiveConf
;
import
org.apache.hadoop.hive.metastore.TableType
;
import
org.apache.hadoop.hive.metastore.api.StorageDescriptor
;
import
org.apache.hadoop.hive.ql.Driver
;
import
org.apache.hadoop.hive.ql.session.SessionState
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge
;
import
org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator
;
import
org.apache.hadoop.metadata.hive.model.HiveDataTypes
;
import
org.apache.hadoop.metadata.typesystem.Referenceable
;
import
org.apache.hadoop.metadata.typesystem.persistence.Id
;
import
org.apache.log4j.spi.LoggerFactory
;
import
org.codehaus.jettison.json.JSONArray
;
import
org.codehaus.jettison.json.JSONObject
;
...
...
@@ -118,7 +121,7 @@ public class HiveHookIT {
private
String
createTable
(
boolean
partition
)
throws
Exception
{
String
tableName
=
tableName
();
runCommand
(
"create table "
+
tableName
+
"(id int, name string)"
+
(
partition
?
" partitioned by(dt string)"
runCommand
(
"create table "
+
tableName
+
"(id int, name string)
comment 'table comment'
"
+
(
partition
?
" partitioned by(dt string)"
:
""
));
return
tableName
;
}
...
...
@@ -137,6 +140,10 @@ public class HiveHookIT {
String
tableId
=
assertTableIsRegistered
(
DEFAULT_DB
,
tableName
);
Referenceable
tableRef
=
dgiCLient
.
getEntity
(
tableId
);
Assert
.
assertEquals
(
tableRef
.
get
(
"tableType"
),
TableType
.
MANAGED_TABLE
.
name
());
Assert
.
assertEquals
(
tableRef
.
get
(
HiveDataModelGenerator
.
COMMENT
),
"table comment"
);
final
Id
sdId
=
(
Id
)
tableRef
.
get
(
"sd"
);
Referenceable
sdRef
=
dgiCLient
.
getEntity
(
sdId
.
id
);
Assert
.
assertEquals
(
sdRef
.
get
(
HiveDataModelGenerator
.
STORAGE_IS_STORED_AS_SUB_DIRS
),
false
);
//Create table where database doesn't exist, will create database instance as well
assertDatabaseIsRegistered
(
DEFAULT_DB
);
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
View file @
74c54aa9
...
...
@@ -48,28 +48,28 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
ThrowingProviderBinder
.
create
(
binder
())
.
bind
(
GraphProvider
.
class
,
TitanGraph
.
class
)
.
to
(
TitanGraphProvider
.
class
)
.
in
(
Scopes
.
SINGLETON
);
.
asEagerSingleton
(
);
// allow for dynamic binding of the metadata repo & graph service
// bind the MetadataRepositoryService interface to an implementation
bind
(
MetadataRepository
.
class
).
to
(
GraphBackedMetadataRepository
.
class
);
bind
(
MetadataRepository
.
class
).
to
(
GraphBackedMetadataRepository
.
class
)
.
asEagerSingleton
()
;
// bind the ITypeStore interface to an implementation
bind
(
ITypeStore
.
class
).
to
(
GraphBackedTypeStore
.
class
);
bind
(
ITypeStore
.
class
).
to
(
GraphBackedTypeStore
.
class
)
.
asEagerSingleton
()
;
// bind the GraphService interface to an implementation
// bind(GraphService.class).to(graphServiceClass);
// bind the MetadataService interface to an implementation
bind
(
MetadataService
.
class
).
to
(
DefaultMetadataService
.
class
);
bind
(
MetadataService
.
class
).
to
(
DefaultMetadataService
.
class
)
.
asEagerSingleton
()
;
// bind the DiscoveryService interface to an implementation
bind
(
DiscoveryService
.
class
).
to
(
GraphBackedDiscoveryService
.
class
);
bind
(
DiscoveryService
.
class
).
to
(
GraphBackedDiscoveryService
.
class
)
.
asEagerSingleton
()
;
bind
(
SearchIndexer
.
class
).
to
(
GraphBackedSearchIndexer
.
class
);
bind
(
SearchIndexer
.
class
).
to
(
GraphBackedSearchIndexer
.
class
)
.
asEagerSingleton
()
;
bind
(
LineageService
.
class
).
to
(
HiveLineageService
.
class
);
bind
(
LineageService
.
class
).
to
(
HiveLineageService
.
class
)
.
asEagerSingleton
()
;
MethodInterceptor
interceptor
=
new
GraphTransactionInterceptor
();
requestInjection
(
interceptor
);
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
View file @
74c54aa9
...
...
@@ -89,10 +89,11 @@ public class DefaultMetadataService implements MetadataService {
private
void
restoreTypeSystem
()
{
LOG
.
info
(
"Restoring type system from the store"
);
try
{
createSuperTypes
();
TypesDef
typesDef
=
typeStore
.
restore
();
typeSystem
.
defineTypes
(
typesDef
);
createSuperTypes
();
}
catch
(
MetadataException
e
)
{
throw
new
RuntimeException
(
e
);
}
...
...
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/hadoop/metadata/discovery/HiveLineageServiceTest.java
View file @
74c54aa9
...
...
@@ -57,11 +57,6 @@ import java.util.List;
@Guice
(
modules
=
RepositoryMetadataModule
.
class
)
public
class
HiveLineageServiceTest
{
static
{
// this would override super types creation if not first thing
TypeSystem
.
getInstance
().
reset
();
}
@Inject
private
DefaultMetadataService
metadataService
;
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment