Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
e3935d5f
Commit
e3935d5f
authored
May 29, 2015
by
Shwetha GS
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
using guice AOP for graph transaction
parent
84c8636d
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
179 additions
and
128 deletions
+179
-128
GraphTransaction.java
...ain/java/org/apache/hadoop/metadata/GraphTransaction.java
+26
-0
GraphTransactionInterceptor.java
...g/apache/hadoop/metadata/GraphTransactionInterceptor.java
+53
-0
RepositoryMetadataModule.java
.../org/apache/hadoop/metadata/RepositoryMetadataModule.java
+13
-31
HiveLineageService.java
.../apache/hadoop/metadata/discovery/HiveLineageService.java
+6
-0
GraphBackedDiscoveryService.java
...metadata/discovery/graph/GraphBackedDiscoveryService.java
+4
-0
GraphBackedMetadataRepository.java
...adata/repository/graph/GraphBackedMetadataRepository.java
+8
-20
TitanGraphProvider.java
.../hadoop/metadata/repository/graph/TitanGraphProvider.java
+2
-0
GraphBackedTypeStore.java
...p/metadata/repository/typestore/GraphBackedTypeStore.java
+67
-76
application.properties
repository/src/test/resources/application.properties
+0
-1
No files found.
repository/src/main/java/org/apache/hadoop/metadata/GraphTransaction.java
0 → 100644
View file @
e3935d5f
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
hadoop
.
metadata
;
import
java.lang.annotation.ElementType
;
import
java.lang.annotation.Retention
;
import
java.lang.annotation.RetentionPolicy
;
import
java.lang.annotation.Target
;
@Retention
(
RetentionPolicy
.
RUNTIME
)
@Target
(
ElementType
.
METHOD
)
public
@interface
GraphTransaction
{}
repository/src/main/java/org/apache/hadoop/metadata/GraphTransactionInterceptor.java
0 → 100644
View file @
e3935d5f
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
hadoop
.
metadata
;
import
com.google.inject.Inject
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
org.aopalliance.intercept.MethodInterceptor
;
import
org.aopalliance.intercept.MethodInvocation
;
import
org.apache.hadoop.metadata.repository.graph.GraphProvider
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
public
class
GraphTransactionInterceptor
implements
MethodInterceptor
{
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
GraphTransactionInterceptor
.
class
);
private
TitanGraph
titanGraph
;
@Inject
GraphProvider
<
TitanGraph
>
graphProvider
;
public
Object
invoke
(
MethodInvocation
invocation
)
throws
Throwable
{
if
(
titanGraph
==
null
)
{
titanGraph
=
graphProvider
.
get
();
}
try
{
LOG
.
debug
(
"graph rollback to cleanup previous state"
);
titanGraph
.
rollback
();
//cleanup previous state
Object
response
=
invocation
.
proceed
();
titanGraph
.
commit
();
LOG
.
debug
(
"graph commit"
);
return
response
;
}
catch
(
Throwable
t
){
titanGraph
.
rollback
();
LOG
.
debug
(
"graph rollback"
);
throw
t
;
}
}
}
repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
View file @
e3935d5f
...
@@ -19,8 +19,10 @@
...
@@ -19,8 +19,10 @@
package
org
.
apache
.
hadoop
.
metadata
;
package
org
.
apache
.
hadoop
.
metadata
;
import
com.google.inject.Scopes
;
import
com.google.inject.Scopes
;
import
com.google.inject.matcher.Matchers
;
import
com.google.inject.throwingproviders.ThrowingProviderBinder
;
import
com.google.inject.throwingproviders.ThrowingProviderBinder
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
org.aopalliance.intercept.MethodInterceptor
;
import
org.apache.hadoop.metadata.discovery.DiscoveryService
;
import
org.apache.hadoop.metadata.discovery.DiscoveryService
;
import
org.apache.hadoop.metadata.discovery.HiveLineageService
;
import
org.apache.hadoop.metadata.discovery.HiveLineageService
;
import
org.apache.hadoop.metadata.discovery.LineageService
;
import
org.apache.hadoop.metadata.discovery.LineageService
;
...
@@ -40,31 +42,7 @@ import org.apache.hadoop.metadata.services.MetadataService;
...
@@ -40,31 +42,7 @@ import org.apache.hadoop.metadata.services.MetadataService;
* Guice module for Repository module.
* Guice module for Repository module.
*/
*/
public
class
RepositoryMetadataModule
extends
com
.
google
.
inject
.
AbstractModule
{
public
class
RepositoryMetadataModule
extends
com
.
google
.
inject
.
AbstractModule
{
@Override
// Graph Service implementation class
// private Class<? extends GraphService> graphServiceClass;
// MetadataRepositoryService implementation class
private
Class
<?
extends
MetadataRepository
>
metadataRepoClass
;
private
Class
<?
extends
ITypeStore
>
typeStore
;
private
Class
<?
extends
MetadataService
>
metadataService
;
private
Class
<?
extends
DiscoveryService
>
discoveryService
;
private
Class
<?
extends
SearchIndexer
>
searchIndexer
;
private
Class
<?
extends
LineageService
>
lineageService
;
public
RepositoryMetadataModule
()
{
// GraphServiceConfigurator gsp = new GraphServiceConfigurator();
// get the impl classes for the repo and the graph service
// this.graphServiceClass = gsp.getImplClass();
this
.
metadataRepoClass
=
GraphBackedMetadataRepository
.
class
;
this
.
typeStore
=
GraphBackedTypeStore
.
class
;
this
.
metadataService
=
DefaultMetadataService
.
class
;
this
.
discoveryService
=
GraphBackedDiscoveryService
.
class
;
this
.
searchIndexer
=
GraphBackedSearchIndexer
.
class
;
this
.
lineageService
=
HiveLineageService
.
class
;
}
protected
void
configure
()
{
protected
void
configure
()
{
// special wiring for Titan Graph
// special wiring for Titan Graph
ThrowingProviderBinder
.
create
(
binder
())
ThrowingProviderBinder
.
create
(
binder
())
...
@@ -75,22 +53,26 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
...
@@ -75,22 +53,26 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// allow for dynamic binding of the metadata repo & graph service
// allow for dynamic binding of the metadata repo & graph service
// bind the MetadataRepositoryService interface to an implementation
// bind the MetadataRepositoryService interface to an implementation
bind
(
MetadataRepository
.
class
).
to
(
metadataRepoC
lass
);
bind
(
MetadataRepository
.
class
).
to
(
GraphBackedMetadataRepository
.
c
lass
);
// bind the ITypeStore interface to an implementation
// bind the ITypeStore interface to an implementation
bind
(
ITypeStore
.
class
).
to
(
typeStore
);
bind
(
ITypeStore
.
class
).
to
(
GraphBackedTypeStore
.
class
);
// bind the GraphService interface to an implementation
// bind the GraphService interface to an implementation
// bind(GraphService.class).to(graphServiceClass);
// bind(GraphService.class).to(graphServiceClass);
// bind the MetadataService interface to an implementation
// bind the MetadataService interface to an implementation
bind
(
MetadataService
.
class
).
to
(
metadataService
);
bind
(
MetadataService
.
class
).
to
(
DefaultMetadataService
.
class
);
// bind the DiscoveryService interface to an implementation
// bind the DiscoveryService interface to an implementation
bind
(
DiscoveryService
.
class
).
to
(
discoveryService
);
bind
(
DiscoveryService
.
class
).
to
(
GraphBackedDiscoveryService
.
class
);
bind
(
SearchIndexer
.
class
).
to
(
GraphBackedSearchIndexer
.
class
);
bind
(
SearchIndexer
.
class
).
to
(
searchIndexer
);
bind
(
LineageService
.
class
).
to
(
HiveLineageService
.
class
);
bind
(
LineageService
.
class
).
to
(
lineageService
);
MethodInterceptor
interceptor
=
new
GraphTransactionInterceptor
();
requestInjection
(
interceptor
);
bindInterceptor
(
Matchers
.
any
(),
Matchers
.
annotatedWith
(
GraphTransaction
.
class
),
interceptor
);
}
}
}
}
repository/src/main/java/org/apache/hadoop/metadata/discovery/HiveLineageService.java
View file @
e3935d5f
...
@@ -20,6 +20,7 @@ package org.apache.hadoop.metadata.discovery;
...
@@ -20,6 +20,7 @@ package org.apache.hadoop.metadata.discovery;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
org.apache.commons.configuration.PropertiesConfiguration
;
import
org.apache.commons.configuration.PropertiesConfiguration
;
import
org.apache.hadoop.metadata.GraphTransaction
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.PropertiesUtil
;
import
org.apache.hadoop.metadata.PropertiesUtil
;
import
org.apache.hadoop.metadata.discovery.graph.DefaultGraphPersistenceStrategy
;
import
org.apache.hadoop.metadata.discovery.graph.DefaultGraphPersistenceStrategy
;
...
@@ -96,6 +97,7 @@ public class HiveLineageService implements LineageService {
...
@@ -96,6 +97,7 @@ public class HiveLineageService implements LineageService {
* @return Lineage Outputs as JSON
* @return Lineage Outputs as JSON
*/
*/
@Override
@Override
@GraphTransaction
public
String
getOutputs
(
String
tableName
)
throws
DiscoveryException
{
public
String
getOutputs
(
String
tableName
)
throws
DiscoveryException
{
LOG
.
info
(
"Fetching lineage outputs for tableName={}"
,
tableName
);
LOG
.
info
(
"Fetching lineage outputs for tableName={}"
,
tableName
);
...
@@ -121,6 +123,7 @@ public class HiveLineageService implements LineageService {
...
@@ -121,6 +123,7 @@ public class HiveLineageService implements LineageService {
* @return Outputs Graph as JSON
* @return Outputs Graph as JSON
*/
*/
@Override
@Override
@GraphTransaction
public
String
getOutputsGraph
(
String
tableName
)
throws
DiscoveryException
{
public
String
getOutputsGraph
(
String
tableName
)
throws
DiscoveryException
{
LOG
.
info
(
"Fetching lineage outputs graph for tableName={}"
,
tableName
);
LOG
.
info
(
"Fetching lineage outputs graph for tableName={}"
,
tableName
);
...
@@ -139,6 +142,7 @@ public class HiveLineageService implements LineageService {
...
@@ -139,6 +142,7 @@ public class HiveLineageService implements LineageService {
* @return Lineage Inputs as JSON
* @return Lineage Inputs as JSON
*/
*/
@Override
@Override
@GraphTransaction
public
String
getInputs
(
String
tableName
)
throws
DiscoveryException
{
public
String
getInputs
(
String
tableName
)
throws
DiscoveryException
{
LOG
.
info
(
"Fetching lineage inputs for tableName={}"
,
tableName
);
LOG
.
info
(
"Fetching lineage inputs for tableName={}"
,
tableName
);
...
@@ -164,6 +168,7 @@ public class HiveLineageService implements LineageService {
...
@@ -164,6 +168,7 @@ public class HiveLineageService implements LineageService {
* @return Inputs Graph as JSON
* @return Inputs Graph as JSON
*/
*/
@Override
@Override
@GraphTransaction
public
String
getInputsGraph
(
String
tableName
)
throws
DiscoveryException
{
public
String
getInputsGraph
(
String
tableName
)
throws
DiscoveryException
{
LOG
.
info
(
"Fetching lineage inputs graph for tableName={}"
,
tableName
);
LOG
.
info
(
"Fetching lineage inputs graph for tableName={}"
,
tableName
);
...
@@ -182,6 +187,7 @@ public class HiveLineageService implements LineageService {
...
@@ -182,6 +187,7 @@ public class HiveLineageService implements LineageService {
* @return Schema as JSON
* @return Schema as JSON
*/
*/
@Override
@Override
@GraphTransaction
public
String
getSchema
(
String
tableName
)
throws
DiscoveryException
{
public
String
getSchema
(
String
tableName
)
throws
DiscoveryException
{
// todo - validate if indeed this is a table type and exists
// todo - validate if indeed this is a table type and exists
String
schemaQuery
=
HIVE_TABLE_TYPE_NAME
String
schemaQuery
=
HIVE_TABLE_TYPE_NAME
...
...
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
View file @
e3935d5f
...
@@ -25,6 +25,7 @@ import com.thinkaurelius.titan.core.TitanVertex;
...
@@ -25,6 +25,7 @@ import com.thinkaurelius.titan.core.TitanVertex;
import
com.tinkerpop.blueprints.Vertex
;
import
com.tinkerpop.blueprints.Vertex
;
import
com.tinkerpop.gremlin.groovy.Gremlin
;
import
com.tinkerpop.gremlin.groovy.Gremlin
;
import
com.tinkerpop.gremlin.java.GremlinPipeline
;
import
com.tinkerpop.gremlin.java.GremlinPipeline
;
import
org.apache.hadoop.metadata.GraphTransaction
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.MetadataServiceClient
;
import
org.apache.hadoop.metadata.discovery.DiscoveryException
;
import
org.apache.hadoop.metadata.discovery.DiscoveryException
;
import
org.apache.hadoop.metadata.discovery.DiscoveryService
;
import
org.apache.hadoop.metadata.discovery.DiscoveryService
;
...
@@ -82,6 +83,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
...
@@ -82,6 +83,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
//http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query
//http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query
// .html#query-string-syntax for query syntax
// .html#query-string-syntax for query syntax
@Override
@Override
// @GraphTransaction
public
String
searchByFullText
(
String
query
)
throws
DiscoveryException
{
public
String
searchByFullText
(
String
query
)
throws
DiscoveryException
{
String
graphQuery
=
String
.
format
(
"v.%s:(%s)"
,
Constants
.
ENTITY_TEXT_PROPERTY_KEY
,
query
);
String
graphQuery
=
String
.
format
(
"v.%s:(%s)"
,
Constants
.
ENTITY_TEXT_PROPERTY_KEY
,
query
);
LOG
.
debug
(
"Full text query: {}"
,
graphQuery
);
LOG
.
debug
(
"Full text query: {}"
,
graphQuery
);
...
@@ -118,6 +120,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
...
@@ -118,6 +120,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
* @return JSON representing the type and results.
* @return JSON representing the type and results.
*/
*/
@Override
@Override
@GraphTransaction
public
String
searchByDSL
(
String
dslQuery
)
throws
DiscoveryException
{
public
String
searchByDSL
(
String
dslQuery
)
throws
DiscoveryException
{
LOG
.
info
(
"Executing dsl query={}"
,
dslQuery
);
LOG
.
info
(
"Executing dsl query={}"
,
dslQuery
);
try
{
try
{
...
@@ -155,6 +158,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
...
@@ -155,6 +158,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
* @throws org.apache.hadoop.metadata.discovery.DiscoveryException
* @throws org.apache.hadoop.metadata.discovery.DiscoveryException
*/
*/
@Override
@Override
@GraphTransaction
public
List
<
Map
<
String
,
String
>>
searchByGremlin
(
String
gremlinQuery
)
public
List
<
Map
<
String
,
String
>>
searchByGremlin
(
String
gremlinQuery
)
throws
DiscoveryException
{
throws
DiscoveryException
{
LOG
.
info
(
"Executing gremlin query={}"
,
gremlinQuery
);
LOG
.
info
(
"Executing gremlin query={}"
,
gremlinQuery
);
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
View file @
e3935d5f
...
@@ -27,6 +27,7 @@ import com.tinkerpop.blueprints.Edge;
...
@@ -27,6 +27,7 @@ import com.tinkerpop.blueprints.Edge;
import
com.tinkerpop.blueprints.GraphQuery
;
import
com.tinkerpop.blueprints.GraphQuery
;
import
com.tinkerpop.blueprints.Vertex
;
import
com.tinkerpop.blueprints.Vertex
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.hadoop.metadata.GraphTransaction
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.repository.Constants
;
import
org.apache.hadoop.metadata.repository.Constants
;
import
org.apache.hadoop.metadata.repository.MetadataRepository
;
import
org.apache.hadoop.metadata.repository.MetadataRepository
;
...
@@ -137,27 +138,23 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -137,27 +138,23 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
@Override
@Override
@GraphTransaction
public
String
createEntity
(
IReferenceableInstance
typedInstance
)
throws
RepositoryException
{
public
String
createEntity
(
IReferenceableInstance
typedInstance
)
throws
RepositoryException
{
LOG
.
info
(
"adding entity={}"
,
typedInstance
);
LOG
.
info
(
"adding entity={}"
,
typedInstance
);
try
{
try
{
titanGraph
.
rollback
();
final
String
guid
=
instanceToGraphMapper
.
mapTypedInstanceToGraph
(
typedInstance
);
final
String
guid
=
instanceToGraphMapper
.
mapTypedInstanceToGraph
(
typedInstance
);
titanGraph
.
commit
();
// commit if there are no errors
return
guid
;
return
guid
;
}
catch
(
MetadataException
e
)
{
}
catch
(
MetadataException
e
)
{
titanGraph
.
rollback
();
throw
new
RepositoryException
(
e
);
throw
new
RepositoryException
(
e
);
}
}
}
}
@Override
@Override
@GraphTransaction
public
ITypedReferenceableInstance
getEntityDefinition
(
String
guid
)
throws
RepositoryException
{
public
ITypedReferenceableInstance
getEntityDefinition
(
String
guid
)
throws
RepositoryException
{
LOG
.
info
(
"Retrieving entity with guid={}"
,
guid
);
LOG
.
info
(
"Retrieving entity with guid={}"
,
guid
);
try
{
try
{
titanGraph
.
rollback
();
// clean up before starting a query
Vertex
instanceVertex
=
getVertexForGUID
(
guid
);
Vertex
instanceVertex
=
getVertexForGUID
(
guid
);
LOG
.
debug
(
"Found a vertex {} for guid {}"
,
instanceVertex
,
guid
);
LOG
.
debug
(
"Found a vertex {} for guid {}"
,
instanceVertex
,
guid
);
...
@@ -206,9 +203,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -206,9 +203,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
* @throws RepositoryException
* @throws RepositoryException
*/
*/
@Override
@Override
@GraphTransaction
public
List
<
String
>
getTraitNames
(
String
guid
)
throws
RepositoryException
{
public
List
<
String
>
getTraitNames
(
String
guid
)
throws
RepositoryException
{
LOG
.
info
(
"Retrieving trait names for entity={}"
,
guid
);
LOG
.
info
(
"Retrieving trait names for entity={}"
,
guid
);
titanGraph
.
rollback
();
// clean up before starting a query
Vertex
instanceVertex
=
getVertexForGUID
(
guid
);
Vertex
instanceVertex
=
getVertexForGUID
(
guid
);
return
getTraitNames
(
instanceVertex
);
return
getTraitNames
(
instanceVertex
);
}
}
...
@@ -231,6 +228,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -231,6 +228,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
* @throws RepositoryException
* @throws RepositoryException
*/
*/
@Override
@Override
@GraphTransaction
public
void
addTrait
(
String
guid
,
public
void
addTrait
(
String
guid
,
ITypedStruct
traitInstance
)
throws
RepositoryException
{
ITypedStruct
traitInstance
)
throws
RepositoryException
{
Preconditions
.
checkNotNull
(
traitInstance
,
"Trait instance cannot be null"
);
Preconditions
.
checkNotNull
(
traitInstance
,
"Trait instance cannot be null"
);
...
@@ -238,22 +236,18 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -238,22 +236,18 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG
.
info
(
"Adding a new trait={} for entity={}"
,
traitName
,
guid
);
LOG
.
info
(
"Adding a new trait={} for entity={}"
,
traitName
,
guid
);
try
{
try
{
titanGraph
.
rollback
();
// clean up before starting a query
Vertex
instanceVertex
=
getVertexForGUID
(
guid
);
Vertex
instanceVertex
=
getVertexForGUID
(
guid
);
// add the trait instance as a new vertex
// add the trait instance as a new vertex
final
String
typeName
=
getTypeName
(
instanceVertex
);
final
String
typeName
=
getTypeName
(
instanceVertex
);
instanceToGraphMapper
.
mapTraitInstanceToVertex
(
traitInstance
,
instanceToGraphMapper
.
mapTraitInstanceToVertex
(
traitInstance
,
getIdFromVertex
(
typeName
,
instanceVertex
),
getIdFromVertex
(
typeName
,
instanceVertex
),
typeName
,
instanceVertex
,
Collections
.<
Id
,
Vertex
>
emptyMap
());
typeName
,
instanceVertex
,
Collections
.<
Id
,
Vertex
>
emptyMap
());
// update the traits in entity once adding trait instance is successful
// update the traits in entity once adding trait instance is successful
((
TitanVertex
)
instanceVertex
)
((
TitanVertex
)
instanceVertex
)
.
addProperty
(
Constants
.
TRAIT_NAMES_PROPERTY_KEY
,
traitName
);
.
addProperty
(
Constants
.
TRAIT_NAMES_PROPERTY_KEY
,
traitName
);
titanGraph
.
commit
();
// commit if there are no errors
}
catch
(
MetadataException
e
)
{
}
catch
(
MetadataException
e
)
{
titanGraph
.
rollback
();
throw
new
RepositoryException
(
e
);
throw
new
RepositoryException
(
e
);
}
}
}
}
...
@@ -266,11 +260,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -266,11 +260,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
* @throws RepositoryException
* @throws RepositoryException
*/
*/
@Override
@Override
@GraphTransaction
public
void
deleteTrait
(
String
guid
,
String
traitNameToBeDeleted
)
public
void
deleteTrait
(
String
guid
,
String
traitNameToBeDeleted
)
throws
RepositoryException
{
throws
RepositoryException
{
LOG
.
info
(
"Deleting trait={} from entity={}"
,
traitNameToBeDeleted
,
guid
);
LOG
.
info
(
"Deleting trait={} from entity={}"
,
traitNameToBeDeleted
,
guid
);
try
{
try
{
titanGraph
.
rollback
();
// clean up before starting a query
Vertex
instanceVertex
=
getVertexForGUID
(
guid
);
Vertex
instanceVertex
=
getVertexForGUID
(
guid
);
List
<
String
>
traitNames
=
getTraitNames
(
instanceVertex
);
List
<
String
>
traitNames
=
getTraitNames
(
instanceVertex
);
...
@@ -297,11 +291,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -297,11 +291,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
traitNames
.
remove
(
traitNameToBeDeleted
);
traitNames
.
remove
(
traitNameToBeDeleted
);
updateTraits
(
instanceVertex
,
traitNames
);
updateTraits
(
instanceVertex
,
traitNames
);
}
}
titanGraph
.
commit
();
// commit if there are no errors
}
}
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
titanGraph
.
rollback
();
throw
new
RepositoryException
(
e
);
throw
new
RepositoryException
(
e
);
}
}
}
}
...
@@ -318,11 +309,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -318,11 +309,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
@Override
@Override
@GraphTransaction
public
void
updateEntity
(
String
guid
,
String
property
,
String
value
)
throws
RepositoryException
{
public
void
updateEntity
(
String
guid
,
String
property
,
String
value
)
throws
RepositoryException
{
LOG
.
info
(
"Adding property {} for entity guid {}"
,
property
,
guid
);
LOG
.
info
(
"Adding property {} for entity guid {}"
,
property
,
guid
);
try
{
try
{
titanGraph
.
rollback
();
// clean up before starting a query
Vertex
instanceVertex
=
GraphHelper
.
findVertexByGUID
(
titanGraph
,
guid
);
Vertex
instanceVertex
=
GraphHelper
.
findVertexByGUID
(
titanGraph
,
guid
);
if
(
instanceVertex
==
null
)
{
if
(
instanceVertex
==
null
)
{
throw
new
RepositoryException
(
"Could not find a vertex for guid "
+
guid
);
throw
new
RepositoryException
(
"Could not find a vertex for guid "
+
guid
);
...
@@ -349,11 +340,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
...
@@ -349,11 +340,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
instanceToGraphMapper
.
mapAttributesToVertex
(
getIdFromVertex
(
typeName
,
instanceVertex
),
instance
,
instanceToGraphMapper
.
mapAttributesToVertex
(
getIdFromVertex
(
typeName
,
instanceVertex
),
instance
,
instanceVertex
,
new
HashMap
<
Id
,
Vertex
>(),
attributeInfo
,
attributeInfo
.
dataType
());
instanceVertex
,
new
HashMap
<
Id
,
Vertex
>(),
attributeInfo
,
attributeInfo
.
dataType
());
titanGraph
.
commit
();
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
throw
new
RepositoryException
(
e
);
throw
new
RepositoryException
(
e
);
}
finally
{
titanGraph
.
rollback
();
}
}
}
}
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java
View file @
e3935d5f
...
@@ -18,6 +18,7 @@
...
@@ -18,6 +18,7 @@
package
org
.
apache
.
hadoop
.
metadata
.
repository
.
graph
;
package
org
.
apache
.
hadoop
.
metadata
.
repository
.
graph
;
import
com.google.inject.Provides
;
import
com.thinkaurelius.titan.core.TitanFactory
;
import
com.thinkaurelius.titan.core.TitanFactory
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.configuration.Configuration
;
...
@@ -64,6 +65,7 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
...
@@ -64,6 +65,7 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
@Override
@Override
@Singleton
@Singleton
@Provides
public
TitanGraph
get
()
{
public
TitanGraph
get
()
{
Configuration
config
;
Configuration
config
;
try
{
try
{
...
...
repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java
View file @
e3935d5f
...
@@ -25,6 +25,7 @@ import com.tinkerpop.blueprints.Direction;
...
@@ -25,6 +25,7 @@ import com.tinkerpop.blueprints.Direction;
import
com.tinkerpop.blueprints.Edge
;
import
com.tinkerpop.blueprints.Edge
;
import
com.tinkerpop.blueprints.Vertex
;
import
com.tinkerpop.blueprints.Vertex
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.hadoop.metadata.GraphTransaction
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.MetadataException
;
import
org.apache.hadoop.metadata.repository.Constants
;
import
org.apache.hadoop.metadata.repository.Constants
;
import
org.apache.hadoop.metadata.repository.graph.GraphProvider
;
import
org.apache.hadoop.metadata.repository.graph.GraphProvider
;
...
@@ -76,40 +77,35 @@ public class GraphBackedTypeStore implements ITypeStore {
...
@@ -76,40 +77,35 @@ public class GraphBackedTypeStore implements ITypeStore {
}
}
@Override
@Override
@GraphTransaction
public
void
store
(
TypeSystem
typeSystem
,
ImmutableList
<
String
>
typeNames
)
throws
MetadataException
{
public
void
store
(
TypeSystem
typeSystem
,
ImmutableList
<
String
>
typeNames
)
throws
MetadataException
{
try
{
ImmutableList
<
String
>
coreTypes
=
typeSystem
.
getCoreTypes
();
ImmutableList
<
String
>
coreTypes
=
typeSystem
.
getCoreTypes
();
for
(
String
typeName
:
typeNames
)
{
titanGraph
.
rollback
();
//Cleanup previous state
if
(!
coreTypes
.
contains
(
typeName
))
{
for
(
String
typeName
:
typeNames
)
{
IDataType
dataType
=
typeSystem
.
getDataType
(
IDataType
.
class
,
typeName
);
if
(!
coreTypes
.
contains
(
typeName
))
{
LOG
.
debug
(
"Processing {}.{} in type store"
,
dataType
.
getTypeCategory
(),
dataType
.
getName
());
IDataType
dataType
=
typeSystem
.
getDataType
(
IDataType
.
class
,
typeName
);
switch
(
dataType
.
getTypeCategory
())
{
LOG
.
debug
(
"Processing {}.{} in type store"
,
dataType
.
getTypeCategory
(),
dataType
.
getName
());
case
ENUM:
switch
(
dataType
.
getTypeCategory
())
{
storeInGraph
((
EnumType
)
dataType
);
case
ENUM:
break
;
storeInGraph
((
EnumType
)
dataType
);
break
;
case
STRUCT:
StructType
structType
=
(
StructType
)
dataType
;
case
STRUCT:
storeInGraph
(
typeSystem
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
StructType
structType
=
(
StructType
)
dataType
;
ImmutableList
.
copyOf
(
structType
.
infoToNameMap
.
keySet
()),
ImmutableList
.<
String
>
of
());
storeInGraph
(
typeSystem
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
break
;
ImmutableList
.
copyOf
(
structType
.
infoToNameMap
.
keySet
()),
ImmutableList
.<
String
>
of
());
break
;
case
TRAIT:
case
CLASS:
case
TRAIT:
HierarchicalType
type
=
(
HierarchicalType
)
dataType
;
case
CLASS:
storeInGraph
(
typeSystem
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
HierarchicalType
type
=
(
HierarchicalType
)
dataType
;
type
.
immediateAttrs
,
type
.
superTypes
);
storeInGraph
(
typeSystem
,
dataType
.
getTypeCategory
(),
dataType
.
getName
(),
break
;
type
.
immediateAttrs
,
type
.
superTypes
);
break
;
default
:
//Ignore primitive/collection types as they are covered under references
break
;
default
:
//Ignore primitive/collection types as they are covered under references
break
;
}
}
}
}
}
titanGraph
.
commit
();
}
finally
{
titanGraph
.
rollback
();
}
}
}
}
...
@@ -206,54 +202,49 @@ public class GraphBackedTypeStore implements ITypeStore {
...
@@ -206,54 +202,49 @@ public class GraphBackedTypeStore implements ITypeStore {
}
}
@Override
@Override
@GraphTransaction
public
TypesDef
restore
()
throws
MetadataException
{
public
TypesDef
restore
()
throws
MetadataException
{
try
{
//Get all vertices for type system
titanGraph
.
rollback
();
//Cleanup previous state
Iterator
vertices
=
//Get all vertices for type system
titanGraph
.
query
().
has
(
Constants
.
VERTEX_TYPE_PROPERTY_KEY
,
VERTEX_TYPE
).
vertices
().
iterator
();
Iterator
vertices
=
titanGraph
.
query
().
has
(
Constants
.
VERTEX_TYPE_PROPERTY_KEY
,
VERTEX_TYPE
).
vertices
().
iterator
();
ImmutableList
.
Builder
<
EnumTypeDefinition
>
enums
=
ImmutableList
.
builder
();
ImmutableList
.
Builder
<
StructTypeDefinition
>
structs
=
ImmutableList
.
builder
();
ImmutableList
.
Builder
<
EnumTypeDefinition
>
enums
=
ImmutableList
.
builder
();
ImmutableList
.
Builder
<
HierarchicalTypeDefinition
<
ClassType
>>
classTypes
=
ImmutableList
.
builder
();
ImmutableList
.
Builder
<
StructTypeDefinition
>
structs
=
ImmutableList
.
builder
();
ImmutableList
.
Builder
<
HierarchicalTypeDefinition
<
TraitType
>>
traits
=
ImmutableList
.
builder
();
ImmutableList
.
Builder
<
HierarchicalTypeDefinition
<
ClassType
>>
classTypes
=
ImmutableList
.
builder
();
ImmutableList
.
Builder
<
HierarchicalTypeDefinition
<
TraitType
>>
traits
=
ImmutableList
.
builder
();
while
(
vertices
.
hasNext
())
{
Vertex
vertex
=
(
Vertex
)
vertices
.
next
();
while
(
vertices
.
hasNext
())
{
DataTypes
.
TypeCategory
typeCategory
=
vertex
.
getProperty
(
Constants
.
TYPE_CATEGORY_PROPERTY_KEY
);
Vertex
vertex
=
(
Vertex
)
vertices
.
next
();
String
typeName
=
vertex
.
getProperty
(
Constants
.
TYPENAME_PROPERTY_KEY
);
DataTypes
.
TypeCategory
typeCategory
=
vertex
.
getProperty
(
Constants
.
TYPE_CATEGORY_PROPERTY_KEY
);
LOG
.
info
(
"Restoring type {}.{}"
,
typeCategory
,
typeName
);
String
typeName
=
vertex
.
getProperty
(
Constants
.
TYPENAME_PROPERTY_KEY
);
switch
(
typeCategory
)
{
LOG
.
info
(
"Restoring type {}.{}"
,
typeCategory
,
typeName
);
case
ENUM:
switch
(
typeCategory
)
{
enums
.
add
(
getEnumType
(
vertex
));
case
ENUM:
break
;
enums
.
add
(
getEnumType
(
vertex
));
break
;
case
STRUCT:
AttributeDefinition
[]
attributes
=
getAttributes
(
vertex
);
case
STRUCT:
structs
.
add
(
new
StructTypeDefinition
(
typeName
,
attributes
));
AttributeDefinition
[]
attributes
=
getAttributes
(
vertex
);
break
;
structs
.
add
(
new
StructTypeDefinition
(
typeName
,
attributes
));
break
;
case
CLASS:
ImmutableList
<
String
>
superTypes
=
getSuperTypes
(
vertex
);
case
CLASS:
attributes
=
getAttributes
(
vertex
);
ImmutableList
<
String
>
superTypes
=
getSuperTypes
(
vertex
);
classTypes
.
add
(
new
HierarchicalTypeDefinition
(
ClassType
.
class
,
typeName
,
superTypes
,
attributes
));
attributes
=
getAttributes
(
vertex
);
break
;
classTypes
.
add
(
new
HierarchicalTypeDefinition
(
ClassType
.
class
,
typeName
,
superTypes
,
attributes
));
break
;
case
TRAIT:
superTypes
=
getSuperTypes
(
vertex
);
case
TRAIT:
attributes
=
getAttributes
(
vertex
);
superTypes
=
getSuperTypes
(
vertex
);
traits
.
add
(
new
HierarchicalTypeDefinition
(
TraitType
.
class
,
typeName
,
superTypes
,
attributes
));
attributes
=
getAttributes
(
vertex
);
break
;
traits
.
add
(
new
HierarchicalTypeDefinition
(
TraitType
.
class
,
typeName
,
superTypes
,
attributes
));
break
;
default
:
throw
new
IllegalArgumentException
(
"Unhandled type category "
+
typeCategory
);
default
:
throw
new
IllegalArgumentException
(
"Unhandled type category "
+
typeCategory
);
}
}
}
titanGraph
.
commit
();
return
TypeUtils
.
getTypesDef
(
enums
.
build
(),
structs
.
build
(),
traits
.
build
(),
classTypes
.
build
());
}
finally
{
titanGraph
.
rollback
();
}
}
return
TypeUtils
.
getTypesDef
(
enums
.
build
(),
structs
.
build
(),
traits
.
build
(),
classTypes
.
build
());
}
}
private
EnumTypeDefinition
getEnumType
(
Vertex
vertex
)
{
private
EnumTypeDefinition
getEnumType
(
Vertex
vertex
)
{
...
...
repository/src/test/resources/application.properties
View file @
e3935d5f
...
@@ -27,7 +27,6 @@ metadata.graph.index.search.backend=elasticsearch
...
@@ -27,7 +27,6 @@ metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory
=
./target/data/es
metadata.graph.index.search.directory
=
./target/data/es
metadata.graph.index.search.elasticsearch.client-only
=
false
metadata.graph.index.search.elasticsearch.client-only
=
false
metadata.graph.index.search.elasticsearch.local-mode
=
true
metadata.graph.index.search.elasticsearch.local-mode
=
true
metadata.graph.index.search.elasticsearch.create.sleep
=
1000
######### Hive Lineage Configs #########
######### Hive Lineage Configs #########
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment