Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
8936008d
Commit
8936008d
authored
Dec 11, 2014
by
Venkatesh Seetharam
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ISSUE-3 Externalize graph configuration. Contributed by Venkatesh Seetharam
parent
5273b41f
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
31 additions
and
127 deletions
+31
-127
ServiceInitializer.java
...rg/apache/hadoop/metadata/service/ServiceInitializer.java
+3
-5
Services.java
...ain/java/org/apache/hadoop/metadata/service/Services.java
+4
-2
GraphBackedMetadataRepositoryService.java
...tadata/services/GraphBackedMetadataRepositoryService.java
+3
-48
TitanGraphService.java
...rg/apache/hadoop/metadata/services/TitanGraphService.java
+13
-62
application.properties
webapp/src/main/resources/application.properties
+8
-10
No files found.
common/src/main/java/org/apache/hadoop/metadata/service/ServiceInitializer.java
View file @
8936008d
...
@@ -36,15 +36,13 @@ public class ServiceInitializer {
...
@@ -36,15 +36,13 @@ public class ServiceInitializer {
public
void
initialize
()
throws
MetadataException
{
public
void
initialize
()
throws
MetadataException
{
String
serviceClassNames
;
String
serviceClassNames
;
try
{
try
{
PropertiesConfiguration
configuration
=
new
PropertiesConfiguration
(
"application.properties"
);
PropertiesConfiguration
configuration
=
new
PropertiesConfiguration
(
"application.properties"
);
serviceClassNames
=
configuration
.
getString
(
"application.services"
);
serviceClassNames
=
configuration
.
getString
(
"application.services"
);
}
catch
(
ConfigurationException
e
)
{
}
catch
(
ConfigurationException
e
)
{
throw
new
Metadata
Exception
(
"unable to get server properties"
);
throw
new
Runtime
Exception
(
"unable to get server properties"
);
}
}
serviceClassNames
=
"org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService"
;
for
(
String
serviceClassName
:
serviceClassNames
.
split
(
","
))
{
for
(
String
serviceClassName
:
serviceClassNames
.
split
(
","
))
{
serviceClassName
=
serviceClassName
.
trim
();
serviceClassName
=
serviceClassName
.
trim
();
if
(
serviceClassName
.
isEmpty
())
{
if
(
serviceClassName
.
isEmpty
())
{
...
...
common/src/main/java/org/apache/hadoop/metadata/service/Services.java
View file @
8936008d
...
@@ -59,7 +59,8 @@ public final class Services implements Iterable<Service> {
...
@@ -59,7 +59,8 @@ public final class Services implements Iterable<Service> {
if
(
services
.
containsKey
(
serviceName
))
{
if
(
services
.
containsKey
(
serviceName
))
{
return
(
T
)
services
.
get
(
serviceName
);
return
(
T
)
services
.
get
(
serviceName
);
}
else
{
}
else
{
throw
new
NoSuchElementException
(
"Service "
+
serviceName
+
" not registered with registry"
);
throw
new
NoSuchElementException
(
"Service "
+
serviceName
+
" not registered with registry"
);
}
}
}
}
...
@@ -79,7 +80,8 @@ public final class Services implements Iterable<Service> {
...
@@ -79,7 +80,8 @@ public final class Services implements Iterable<Service> {
String
serviceClassName
;
String
serviceClassName
;
try
{
try
{
PropertiesConfiguration
configuration
=
new
PropertiesConfiguration
(
"application.properties"
);
PropertiesConfiguration
configuration
=
new
PropertiesConfiguration
(
"application.properties"
);
serviceClassName
=
configuration
.
getString
(
serviceName
+
".impl"
);
serviceClassName
=
configuration
.
getString
(
serviceName
+
".impl"
);
}
catch
(
ConfigurationException
e
)
{
}
catch
(
ConfigurationException
e
)
{
throw
new
MetadataException
(
"unable to get server properties"
);
throw
new
MetadataException
(
"unable to get server properties"
);
...
...
repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryService.java
View file @
8936008d
...
@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.services;
...
@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.services;
import
com.google.common.base.Preconditions
;
import
com.google.common.base.Preconditions
;
import
com.tinkerpop.blueprints.Graph
;
import
com.tinkerpop.blueprints.Graph
;
import
com.tinkerpop.blueprints.GraphQuery
;
import
com.tinkerpop.blueprints.TransactionalGraph
;
import
com.tinkerpop.blueprints.TransactionalGraph
;
import
com.tinkerpop.blueprints.Vertex
;
import
com.tinkerpop.blueprints.Vertex
;
import
org.apache.hadoop.metadata.service.Services
;
import
org.apache.hadoop.metadata.service.Services
;
...
@@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory;
...
@@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory;
import
java.io.IOException
;
import
java.io.IOException
;
import
java.util.Collections
;
import
java.util.Collections
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
java.util.UUID
;
import
java.util.UUID
;
...
@@ -102,6 +100,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
...
@@ -102,6 +100,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
@Override
public
String
submitEntity
(
String
entity
,
String
entityType
)
{
public
String
submitEntity
(
String
entity
,
String
entityType
)
{
LOG
.
info
(
"adding entity={} type={}"
,
entity
,
entityType
);
Map
<
String
,
String
>
properties
=
(
Map
<
String
,
String
>)
JSONValue
.
parse
(
entity
);
Map
<
String
,
String
>
properties
=
(
Map
<
String
,
String
>)
JSONValue
.
parse
(
entity
);
final
String
entityName
=
properties
.
get
(
"entityName"
);
final
String
entityName
=
properties
.
get
(
"entityName"
);
...
@@ -132,6 +131,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
...
@@ -132,6 +131,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
@Override
public
String
getEntityDefinition
(
String
entityName
,
String
entityType
)
{
public
String
getEntityDefinition
(
String
entityName
,
String
entityType
)
{
LOG
.
info
(
"Retrieving entity name={} type={}"
,
entityName
,
entityType
);
Vertex
entityVertex
=
GraphUtils
.
findVertex
(
getBlueprintsGraph
(),
entityName
,
entityType
);
Vertex
entityVertex
=
GraphUtils
.
findVertex
(
getBlueprintsGraph
(),
entityName
,
entityType
);
if
(
entityVertex
==
null
)
{
if
(
entityVertex
==
null
)
{
return
null
;
return
null
;
...
@@ -143,52 +143,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
...
@@ -143,52 +143,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
@Override
public
List
<
String
>
getEntityList
(
String
entityType
)
{
public
List
<
String
>
getEntityList
(
String
entityType
)
{
LOG
.
info
(
"Retrieving entity list for type={}"
,
entityType
);
return
Collections
.
emptyList
();
return
Collections
.
emptyList
();
}
}
public
static
void
main
(
String
[]
args
)
throws
Exception
{
TitanGraphService
titanGraphService
=
new
TitanGraphService
();
titanGraphService
.
start
();
Services
.
get
().
register
(
titanGraphService
);
GraphBackedMetadataRepositoryService
service
=
new
GraphBackedMetadataRepositoryService
();
try
{
service
.
start
();
String
guid
=
UUID
.
randomUUID
().
toString
();
final
TransactionalGraph
graph
=
service
.
getTransactionalGraph
();
System
.
out
.
println
(
"graph = "
+
graph
);
System
.
out
.
println
(
"graph.getVertices() = "
+
graph
.
getVertices
());
Vertex
entityVertex
=
null
;
try
{
graph
.
rollback
();
entityVertex
=
graph
.
addVertex
(
null
);
entityVertex
.
setProperty
(
"guid"
,
guid
);
entityVertex
.
setProperty
(
"entityName"
,
"entityName"
);
entityVertex
.
setProperty
(
"entityType"
,
"entityType"
);
}
catch
(
Exception
e
)
{
graph
.
rollback
();
e
.
printStackTrace
();
}
finally
{
graph
.
commit
();
}
System
.
out
.
println
(
"vertex = "
+
GraphUtils
.
vertexString
(
entityVertex
));
GraphQuery
query
=
graph
.
query
()
.
has
(
"entityName"
,
"entityName"
)
.
has
(
"entityType"
,
"entityType"
);
Iterator
<
Vertex
>
results
=
query
.
vertices
().
iterator
();
if
(
results
.
hasNext
())
{
Vertex
vertexFromQuery
=
results
.
next
();
System
.
out
.
println
(
"vertex = "
+
GraphUtils
.
vertexString
(
vertexFromQuery
));
}
}
finally
{
service
.
stop
();
titanGraphService
.
stop
();
}
}
}
}
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java
View file @
8936008d
...
@@ -18,13 +18,9 @@
...
@@ -18,13 +18,9 @@
package
org
.
apache
.
hadoop
.
metadata
.
services
;
package
org
.
apache
.
hadoop
.
metadata
.
services
;
import
com.thinkaurelius.titan.core.Cardinality
;
import
com.thinkaurelius.titan.core.PropertyKey
;
import
com.thinkaurelius.titan.core.TitanFactory
;
import
com.thinkaurelius.titan.core.TitanFactory
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.schema.TitanManagement
;
import
com.tinkerpop.blueprints.Edge
;
import
com.tinkerpop.blueprints.Edge
;
import
com.tinkerpop.blueprints.Element
;
import
com.tinkerpop.blueprints.Graph
;
import
com.tinkerpop.blueprints.Graph
;
import
com.tinkerpop.blueprints.KeyIndexableGraph
;
import
com.tinkerpop.blueprints.KeyIndexableGraph
;
import
com.tinkerpop.blueprints.TransactionalGraph
;
import
com.tinkerpop.blueprints.TransactionalGraph
;
...
@@ -50,10 +46,8 @@ public class TitanGraphService implements GraphService {
...
@@ -50,10 +46,8 @@ public class TitanGraphService implements GraphService {
/**
/**
* Constant for the configuration property that indicates the prefix.
* Constant for the configuration property that indicates the prefix.
*/
*/
private
static
final
String
METADATA_PREFIX
=
"metadata.titanGraph."
;
private
static
final
String
METADATA_PREFIX
=
"metadata.graph."
;
private
static
final
String
METADATA_INDEX_KEY
=
"index.name"
;
private
Configuration
graphConfig
;
private
TitanGraph
titanGraph
;
private
TitanGraph
titanGraph
;
private
Set
<
String
>
vertexIndexedKeys
;
private
Set
<
String
>
vertexIndexedKeys
;
private
Set
<
String
>
edgeIndexedKeys
;
private
Set
<
String
>
edgeIndexedKeys
;
...
@@ -75,11 +69,10 @@ public class TitanGraphService implements GraphService {
...
@@ -75,11 +69,10 @@ public class TitanGraphService implements GraphService {
*/
*/
@Override
@Override
public
void
start
()
throws
Exception
{
public
void
start
()
throws
Exception
{
// graphConfig = getConfiguration();
Configuration
graphConfig
=
getConfiguration
();
titanGraph
=
initializeGraphDB
(
graphConfig
);
titanGraph
=
initializeGraphDB
();
createIndicesForVertexKeys
();
// createIndicesForVertexKeys();
// todo - create Edge Cardinality Constraints
// todo - create Edge Cardinality Constraints
LOG
.
info
(
"Initialized titanGraph db: {}"
,
titanGraph
);
LOG
.
info
(
"Initialized titanGraph db: {}"
,
titanGraph
);
...
@@ -90,29 +83,17 @@ public class TitanGraphService implements GraphService {
...
@@ -90,29 +83,17 @@ public class TitanGraphService implements GraphService {
LOG
.
info
(
"Init edge property keys: {}"
,
edgeIndexedKeys
);
LOG
.
info
(
"Init edge property keys: {}"
,
edgeIndexedKeys
);
}
}
protected
TitanGraph
initializeGraphDB
()
{
LOG
.
info
(
"Initializing titanGraph db"
);
// todo: externalize this
Configuration
graphConfig
=
new
PropertiesConfiguration
();
graphConfig
.
setProperty
(
"storage.backend"
,
"berkeleyje"
);
graphConfig
.
setProperty
(
"storage.directory"
,
"target/data/graphdb"
);
return
TitanFactory
.
open
(
graphConfig
);
}
private
static
Configuration
getConfiguration
()
throws
ConfigurationException
{
private
static
Configuration
getConfiguration
()
throws
ConfigurationException
{
PropertiesConfiguration
configProperties
=
new
PropertiesConfiguration
(
"application.properties"
);
PropertiesConfiguration
configProperties
=
new
PropertiesConfiguration
(
"application.properties"
);
Configuration
graphConfig
=
new
PropertiesConfiguration
();
Configuration
graphConfig
=
new
PropertiesConfiguration
();
final
Iterator
<
String
>
iterator
=
configProperties
.
getKeys
();
final
Iterator
<
String
>
iterator
=
configProperties
.
getKeys
();
while
(
iterator
.
hasNext
())
{
while
(
iterator
.
hasNext
())
{
String
key
=
iterator
.
next
();
String
key
=
iterator
.
next
();
System
.
out
.
println
(
"key = "
+
key
);
if
(
key
.
startsWith
(
METADATA_PREFIX
))
{
if
(
key
.
startsWith
(
METADATA_PREFIX
))
{
String
value
=
(
String
)
configProperties
.
getProperty
(
key
);
String
value
=
(
String
)
configProperties
.
getProperty
(
key
);
key
=
key
.
substring
(
METADATA_PREFIX
.
length
());
key
=
key
.
substring
(
METADATA_PREFIX
.
length
());
System
.
out
.
println
(
"**** key = "
+
key
+
", value = "
+
value
);
graphConfig
.
setProperty
(
key
,
value
);
graphConfig
.
setProperty
(
key
,
value
);
}
}
}
}
...
@@ -120,49 +101,19 @@ public class TitanGraphService implements GraphService {
...
@@ -120,49 +101,19 @@ public class TitanGraphService implements GraphService {
return
graphConfig
;
return
graphConfig
;
}
}
/**
protected
TitanGraph
initializeGraphDB
(
Configuration
graphConfig
)
{
* This unfortunately requires a handle to Titan implementation since
LOG
.
info
(
"Initializing titanGraph db"
);
* com.tinkerpop.blueprints.KeyIndexableGraph#createKeyIndex does not create an index.
return
TitanFactory
.
open
(
graphConfig
);
*/
}
protected
void
createIndicesForVertexKeys
()
{
protected
void
createIndicesForVertexKeys
()
{
if
(!
((
KeyIndexableGraph
)
titanGraph
)
.
getIndexedKeys
(
Vertex
.
class
).
isEmpty
())
{
if
(!
titanGraph
.
getIndexedKeys
(
Vertex
.
class
).
isEmpty
())
{
LOG
.
info
(
"Indexes already exist for titanGraph"
);
LOG
.
info
(
"Indexes already exist for titanGraph"
);
return
;
return
;
}
}
LOG
.
info
(
"Indexes does not exist, Creating indexes for titanGraph"
);
LOG
.
info
(
"Indexes does not exist, Creating indexes for titanGraph"
);
// todo - externalize this
// todo - add index for vertex and edge property keys
String
indexName
=
graphConfig
.
getString
(
METADATA_INDEX_KEY
);
PropertyKey
guid
=
createPropertyKey
(
"guid"
,
String
.
class
,
Cardinality
.
SINGLE
);
createIndex
(
indexName
,
guid
,
Vertex
.
class
,
true
);
getTitanGraph
().
commit
();
}
private
PropertyKey
createPropertyKey
(
String
propertyKeyName
,
Class
<
String
>
dataType
,
Cardinality
cardinality
)
{
PropertyKey
propertyKey
=
getTitanGraph
().
getManagementSystem
()
.
makePropertyKey
(
propertyKeyName
)
.
dataType
(
dataType
)
.
cardinality
(
cardinality
)
.
make
();
LOG
.
info
(
"Created property key {}"
,
propertyKey
);
return
propertyKey
;
}
private
void
createIndex
(
String
indexName
,
PropertyKey
propertyKey
,
Class
<?
extends
Element
>
clazz
,
boolean
isUnique
)
{
TitanManagement
managementSystem
=
getTitanGraph
().
getManagementSystem
();
managementSystem
.
buildPropertyIndex
(
propertyKey
,
indexName
);
TitanManagement
.
IndexBuilder
indexBuilder
=
managementSystem
.
buildIndex
(
indexName
,
clazz
)
.
addKey
(
propertyKey
);
if
(
isUnique
)
{
indexBuilder
.
unique
();
}
indexBuilder
.
buildCompositeIndex
();
}
}
/**
/**
...
...
webapp/src/main/resources/application.properties
View file @
8936008d
...
@@ -16,23 +16,21 @@
...
@@ -16,23 +16,21 @@
# limitations under the License.
# limitations under the License.
#
#
application.services
=
org.apache.hadoop.metadata.services.TitanGraphService,
\
org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
application.services
=
org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
#metadata.graph.schema.default=none
# Graph implementation
# Graph implementation
metadata.graph.blueprints.graph
=
com.thinkaurelius.titan.core.TitanFactory
#
metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
# Graph Storage
# Graph Storage
metadata.graph.storage.backend
=
berkeleyje
metadata.graph.storage.backend
=
berkeleyje
metadata.graph.storage.directory
=
${user.dir}/target/data/graphdb
metadata.graph.storage.directory
=
target/data/berkeley
# Graph Search Index
# Graph Search Index
#metadata.graph.index.name=search
metadata.graph.index.search.backend
=
elasticsearch
#metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory
=
target/data/es
#metadata.graph.index.search.directory=${user.dir}/target/data/searchindex
metadata.graph.index.search.elasticsearch.client-only
=
false
#metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode
=
true
#metadata.graph.index.search.elasticsearch.local-mode=true
metadata.enableTLS
=
false
metadata.enableTLS
=
false
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment