Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
8936008d
Commit
8936008d
authored
Dec 11, 2014
by
Venkatesh Seetharam
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ISSUE-3 Externalize graph configuration. Contributed by Venkatesh Seetharam
parent
5273b41f
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
31 additions
and
127 deletions
+31
-127
ServiceInitializer.java
...rg/apache/hadoop/metadata/service/ServiceInitializer.java
+3
-5
Services.java
...ain/java/org/apache/hadoop/metadata/service/Services.java
+4
-2
GraphBackedMetadataRepositoryService.java
...tadata/services/GraphBackedMetadataRepositoryService.java
+3
-48
TitanGraphService.java
...rg/apache/hadoop/metadata/services/TitanGraphService.java
+13
-62
application.properties
webapp/src/main/resources/application.properties
+8
-10
No files found.
common/src/main/java/org/apache/hadoop/metadata/service/ServiceInitializer.java
View file @
8936008d
...
...
@@ -36,15 +36,13 @@ public class ServiceInitializer {
public
void
initialize
()
throws
MetadataException
{
String
serviceClassNames
;
try
{
PropertiesConfiguration
configuration
=
new
PropertiesConfiguration
(
"application.properties"
);
PropertiesConfiguration
configuration
=
new
PropertiesConfiguration
(
"application.properties"
);
serviceClassNames
=
configuration
.
getString
(
"application.services"
);
}
catch
(
ConfigurationException
e
)
{
throw
new
Metadata
Exception
(
"unable to get server properties"
);
throw
new
Runtime
Exception
(
"unable to get server properties"
);
}
serviceClassNames
=
"org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService"
;
for
(
String
serviceClassName
:
serviceClassNames
.
split
(
","
))
{
serviceClassName
=
serviceClassName
.
trim
();
if
(
serviceClassName
.
isEmpty
())
{
...
...
common/src/main/java/org/apache/hadoop/metadata/service/Services.java
View file @
8936008d
...
...
@@ -59,7 +59,8 @@ public final class Services implements Iterable<Service> {
if
(
services
.
containsKey
(
serviceName
))
{
return
(
T
)
services
.
get
(
serviceName
);
}
else
{
throw
new
NoSuchElementException
(
"Service "
+
serviceName
+
" not registered with registry"
);
throw
new
NoSuchElementException
(
"Service "
+
serviceName
+
" not registered with registry"
);
}
}
...
...
@@ -79,7 +80,8 @@ public final class Services implements Iterable<Service> {
String
serviceClassName
;
try
{
PropertiesConfiguration
configuration
=
new
PropertiesConfiguration
(
"application.properties"
);
PropertiesConfiguration
configuration
=
new
PropertiesConfiguration
(
"application.properties"
);
serviceClassName
=
configuration
.
getString
(
serviceName
+
".impl"
);
}
catch
(
ConfigurationException
e
)
{
throw
new
MetadataException
(
"unable to get server properties"
);
...
...
repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryService.java
View file @
8936008d
...
...
@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.services;
import
com.google.common.base.Preconditions
;
import
com.tinkerpop.blueprints.Graph
;
import
com.tinkerpop.blueprints.GraphQuery
;
import
com.tinkerpop.blueprints.TransactionalGraph
;
import
com.tinkerpop.blueprints.Vertex
;
import
org.apache.hadoop.metadata.service.Services
;
...
...
@@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory;
import
java.io.IOException
;
import
java.util.Collections
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.UUID
;
...
...
@@ -102,6 +100,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
public
String
submitEntity
(
String
entity
,
String
entityType
)
{
LOG
.
info
(
"adding entity={} type={}"
,
entity
,
entityType
);
Map
<
String
,
String
>
properties
=
(
Map
<
String
,
String
>)
JSONValue
.
parse
(
entity
);
final
String
entityName
=
properties
.
get
(
"entityName"
);
...
...
@@ -132,6 +131,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
public
String
getEntityDefinition
(
String
entityName
,
String
entityType
)
{
LOG
.
info
(
"Retrieving entity name={} type={}"
,
entityName
,
entityType
);
Vertex
entityVertex
=
GraphUtils
.
findVertex
(
getBlueprintsGraph
(),
entityName
,
entityType
);
if
(
entityVertex
==
null
)
{
return
null
;
...
...
@@ -143,52 +143,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
public
List
<
String
>
getEntityList
(
String
entityType
)
{
LOG
.
info
(
"Retrieving entity list for type={}"
,
entityType
);
return
Collections
.
emptyList
();
}
public
static
void
main
(
String
[]
args
)
throws
Exception
{
TitanGraphService
titanGraphService
=
new
TitanGraphService
();
titanGraphService
.
start
();
Services
.
get
().
register
(
titanGraphService
);
GraphBackedMetadataRepositoryService
service
=
new
GraphBackedMetadataRepositoryService
();
try
{
service
.
start
();
String
guid
=
UUID
.
randomUUID
().
toString
();
final
TransactionalGraph
graph
=
service
.
getTransactionalGraph
();
System
.
out
.
println
(
"graph = "
+
graph
);
System
.
out
.
println
(
"graph.getVertices() = "
+
graph
.
getVertices
());
Vertex
entityVertex
=
null
;
try
{
graph
.
rollback
();
entityVertex
=
graph
.
addVertex
(
null
);
entityVertex
.
setProperty
(
"guid"
,
guid
);
entityVertex
.
setProperty
(
"entityName"
,
"entityName"
);
entityVertex
.
setProperty
(
"entityType"
,
"entityType"
);
}
catch
(
Exception
e
)
{
graph
.
rollback
();
e
.
printStackTrace
();
}
finally
{
graph
.
commit
();
}
System
.
out
.
println
(
"vertex = "
+
GraphUtils
.
vertexString
(
entityVertex
));
GraphQuery
query
=
graph
.
query
()
.
has
(
"entityName"
,
"entityName"
)
.
has
(
"entityType"
,
"entityType"
);
Iterator
<
Vertex
>
results
=
query
.
vertices
().
iterator
();
if
(
results
.
hasNext
())
{
Vertex
vertexFromQuery
=
results
.
next
();
System
.
out
.
println
(
"vertex = "
+
GraphUtils
.
vertexString
(
vertexFromQuery
));
}
}
finally
{
service
.
stop
();
titanGraphService
.
stop
();
}
}
}
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java
View file @
8936008d
...
...
@@ -18,13 +18,9 @@
package
org
.
apache
.
hadoop
.
metadata
.
services
;
import
com.thinkaurelius.titan.core.Cardinality
;
import
com.thinkaurelius.titan.core.PropertyKey
;
import
com.thinkaurelius.titan.core.TitanFactory
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.schema.TitanManagement
;
import
com.tinkerpop.blueprints.Edge
;
import
com.tinkerpop.blueprints.Element
;
import
com.tinkerpop.blueprints.Graph
;
import
com.tinkerpop.blueprints.KeyIndexableGraph
;
import
com.tinkerpop.blueprints.TransactionalGraph
;
...
...
@@ -50,10 +46,8 @@ public class TitanGraphService implements GraphService {
/**
* Constant for the configuration property that indicates the prefix.
*/
private
static
final
String
METADATA_PREFIX
=
"metadata.titanGraph."
;
private
static
final
String
METADATA_INDEX_KEY
=
"index.name"
;
private
static
final
String
METADATA_PREFIX
=
"metadata.graph."
;
private
Configuration
graphConfig
;
private
TitanGraph
titanGraph
;
private
Set
<
String
>
vertexIndexedKeys
;
private
Set
<
String
>
edgeIndexedKeys
;
...
...
@@ -75,11 +69,10 @@ public class TitanGraphService implements GraphService {
*/
@Override
public
void
start
()
throws
Exception
{
// graphConfig = getConfiguration();
Configuration
graphConfig
=
getConfiguration
();
titanGraph
=
initializeGraphDB
(
graphConfig
);
titanGraph
=
initializeGraphDB
();
// createIndicesForVertexKeys();
createIndicesForVertexKeys
();
// todo - create Edge Cardinality Constraints
LOG
.
info
(
"Initialized titanGraph db: {}"
,
titanGraph
);
...
...
@@ -90,29 +83,17 @@ public class TitanGraphService implements GraphService {
LOG
.
info
(
"Init edge property keys: {}"
,
edgeIndexedKeys
);
}
protected
TitanGraph
initializeGraphDB
()
{
LOG
.
info
(
"Initializing titanGraph db"
);
// todo: externalize this
Configuration
graphConfig
=
new
PropertiesConfiguration
();
graphConfig
.
setProperty
(
"storage.backend"
,
"berkeleyje"
);
graphConfig
.
setProperty
(
"storage.directory"
,
"target/data/graphdb"
);
return
TitanFactory
.
open
(
graphConfig
);
}
private
static
Configuration
getConfiguration
()
throws
ConfigurationException
{
PropertiesConfiguration
configProperties
=
new
PropertiesConfiguration
(
"application.properties"
);
PropertiesConfiguration
configProperties
=
new
PropertiesConfiguration
(
"application.properties"
);
Configuration
graphConfig
=
new
PropertiesConfiguration
();
final
Iterator
<
String
>
iterator
=
configProperties
.
getKeys
();
while
(
iterator
.
hasNext
())
{
String
key
=
iterator
.
next
();
System
.
out
.
println
(
"key = "
+
key
);
if
(
key
.
startsWith
(
METADATA_PREFIX
))
{
String
value
=
(
String
)
configProperties
.
getProperty
(
key
);
key
=
key
.
substring
(
METADATA_PREFIX
.
length
());
System
.
out
.
println
(
"**** key = "
+
key
+
", value = "
+
value
);
graphConfig
.
setProperty
(
key
,
value
);
}
}
...
...
@@ -120,49 +101,19 @@ public class TitanGraphService implements GraphService {
return
graphConfig
;
}
/**
* This unfortunately requires a handle to Titan implementation since
* com.tinkerpop.blueprints.KeyIndexableGraph#createKeyIndex does not create an index.
*/
protected
TitanGraph
initializeGraphDB
(
Configuration
graphConfig
)
{
LOG
.
info
(
"Initializing titanGraph db"
);
return
TitanFactory
.
open
(
graphConfig
);
}
protected
void
createIndicesForVertexKeys
()
{
if
(!
((
KeyIndexableGraph
)
titanGraph
)
.
getIndexedKeys
(
Vertex
.
class
).
isEmpty
())
{
if
(!
titanGraph
.
getIndexedKeys
(
Vertex
.
class
).
isEmpty
())
{
LOG
.
info
(
"Indexes already exist for titanGraph"
);
return
;
}
LOG
.
info
(
"Indexes does not exist, Creating indexes for titanGraph"
);
// todo - externalize this
String
indexName
=
graphConfig
.
getString
(
METADATA_INDEX_KEY
);
PropertyKey
guid
=
createPropertyKey
(
"guid"
,
String
.
class
,
Cardinality
.
SINGLE
);
createIndex
(
indexName
,
guid
,
Vertex
.
class
,
true
);
getTitanGraph
().
commit
();
}
private
PropertyKey
createPropertyKey
(
String
propertyKeyName
,
Class
<
String
>
dataType
,
Cardinality
cardinality
)
{
PropertyKey
propertyKey
=
getTitanGraph
().
getManagementSystem
()
.
makePropertyKey
(
propertyKeyName
)
.
dataType
(
dataType
)
.
cardinality
(
cardinality
)
.
make
();
LOG
.
info
(
"Created property key {}"
,
propertyKey
);
return
propertyKey
;
}
private
void
createIndex
(
String
indexName
,
PropertyKey
propertyKey
,
Class
<?
extends
Element
>
clazz
,
boolean
isUnique
)
{
TitanManagement
managementSystem
=
getTitanGraph
().
getManagementSystem
();
managementSystem
.
buildPropertyIndex
(
propertyKey
,
indexName
);
TitanManagement
.
IndexBuilder
indexBuilder
=
managementSystem
.
buildIndex
(
indexName
,
clazz
)
.
addKey
(
propertyKey
);
if
(
isUnique
)
{
indexBuilder
.
unique
();
}
indexBuilder
.
buildCompositeIndex
();
// todo - add index for vertex and edge property keys
}
/**
...
...
webapp/src/main/resources/application.properties
View file @
8936008d
...
...
@@ -16,23 +16,21 @@
# limitations under the License.
#
application.services
=
org.apache.hadoop.metadata.services.TitanGraphService,
\
org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
application.services
=
org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
#metadata.graph.schema.default=none
# Graph implementation
metadata.graph.blueprints.graph
=
com.thinkaurelius.titan.core.TitanFactory
#
metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
# Graph Storage
metadata.graph.storage.backend
=
berkeleyje
metadata.graph.storage.directory
=
${user.dir}/target/data/graphdb
metadata.graph.storage.directory
=
target/data/berkeley
# Graph Search Index
#metadata.graph.index.name=search
#metadata.graph.index.search.backend=elasticsearch
#metadata.graph.index.search.directory=${user.dir}/target/data/searchindex
#metadata.graph.index.search.elasticsearch.client-only=false
#metadata.graph.index.search.elasticsearch.local-mode=true
metadata.graph.index.search.backend
=
elasticsearch
metadata.graph.index.search.directory
=
target/data/es
metadata.graph.index.search.elasticsearch.client-only
=
false
metadata.graph.index.search.elasticsearch.local-mode
=
true
metadata.enableTLS
=
false
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment