Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
919120f6
Commit
919120f6
authored
9 years ago
by
Suma Shivaprasad
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-352 Improve write performance on type and entity creation with Hbase(sumasai)
parent
91ad0218
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
30 changed files
with
338 additions
and
141 deletions
+338
-141
pom.xml
distro/pom.xml
+10
-0
atlas_start.py
distro/src/bin/atlas_start.py
+1
-0
standalone-package.xml
distro/src/main/assemblies/standalone-package.xml
+1
-1
TestMetadata.py
distro/src/test/python/scripts/TestMetadata.py
+2
-6
pom.xml
pom.xml
+7
-0
release-log.txt
release-log.txt
+1
-0
pom.xml
repository/pom.xml
+5
-46
RepositoryMetadataModule.java
.../main/java/org/apache/atlas/RepositoryMetadataModule.java
+1
-0
GraphRepoMapperScaleTest.java
...ache/atlas/repository/graph/GraphRepoMapperScaleTest.java
+22
-36
pom.xml
titan/pom.xml
+106
-0
AdminMask.java
.../com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
+0
-0
ConnectionMask.java
...thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
+0
-0
HBaseAdmin0_98.java
...thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
+0
-0
HBaseAdmin1_0.java
.../thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
+0
-0
HBaseCompat.java
...om/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
+0
-1
HBaseCompat0_98.java
...hinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
+0
-0
HBaseCompat1_0.java
...thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
+0
-1
HBaseCompat1_1.java
...thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
+0
-0
HBaseCompatLoader.java
...nkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
+0
-0
HBaseKeyColumnValueStore.java
...ius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
+31
-2
HBaseStoreManager.java
...nkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
+25
-24
HBaseTransaction.java
...inkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
+43
-1
HConnection0_98.java
...hinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
+0
-0
HConnection1_0.java
...thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
+0
-0
HTable0_98.java
...com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
+0
-0
HTable1_0.java
.../com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
+0
-1
TableMask.java
.../com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
+0
-0
LocalLockMediator.java
...aurelius/titan/diskstorage/locking/LocalLockMediator.java
+0
-0
Solr5Index.java
.../com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
+23
-22
LocalLockMediatorTest.java
...lius/titan/diskstorage/locking/LocalLockMediatorTest.java
+60
-0
No files found.
distro/pom.xml
View file @
919120f6
...
...
@@ -89,6 +89,16 @@
</profiles>
<build>
<outputDirectory>
target/bin
</outputDirectory>
<resources>
<resource>
<directory>
src/bin
</directory>
<filtering>
true
</filtering>
<includes>
<include>
**/*.py
</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>
org.codehaus.mojo
</groupId>
...
...
This diff is collapsed.
Click to expand it.
distro/src/bin/atlas_start.py
View file @
919120f6
...
...
@@ -58,6 +58,7 @@ def main():
p
=
os
.
pathsep
metadata_classpath
=
confdir
+
p
\
+
os
.
path
.
join
(
web_app_dir
,
"atlas"
,
"WEB-INF"
,
"classes"
)
+
p
\
+
os
.
path
.
join
(
web_app_dir
,
"atlas"
,
"WEB-INF"
,
"lib"
,
"atlas-titan-${project.version}.jar"
)
+
p
\
+
os
.
path
.
join
(
web_app_dir
,
"atlas"
,
"WEB-INF"
,
"lib"
,
"*"
)
+
p
\
+
os
.
path
.
join
(
metadata_home
,
"libext"
,
"*"
)
if
os
.
path
.
exists
(
hbase_conf_dir
):
...
...
This diff is collapsed.
Click to expand it.
distro/src/main/assemblies/standalone-package.xml
View file @
919120f6
...
...
@@ -48,7 +48,7 @@
</fileSet>
<fileSet>
<directory>
src
/bin
</directory>
<directory>
target
/bin
</directory>
<outputDirectory>
bin
</outputDirectory>
<fileMode>
0755
</fileMode>
<directoryMode>
0755
</directoryMode>
...
...
This diff is collapsed.
Click to expand it.
distro/src/test/python/scripts/TestMetadata.py
View file @
919120f6
...
...
@@ -56,20 +56,16 @@ class TestMetadata(unittest.TestCase):
java_mock
.
assert_called_with
(
'org.apache.atlas.Atlas'
,
[
'-app'
,
'metadata_home
\\
server
\\
webapp
\\
atlas'
],
'metadata_home
\\
conf;metadata_home
\\
server
\\
webapp
\\
atlas
\\
WEB-INF
\\
classes;metadata_home
\\
server
\\
webapp
\\
atlas
\\
WEB-INF
\\
lib
\\
*;metadata_home
\\
libext
\\
*;metadata_home
\\
hbase
\\
conf'
,
'metadata_home
\\
conf;metadata_home
\\
server
\\
webapp
\\
atlas
\\
WEB-INF
\\
classes;metadata_home
\\
server
\\
webapp
\\
atlas
\\
WEB-INF
\\
lib
\\
atlas-titan-${project.version}.jar;metadata_home
\\
server
\\
webapp
\\
atlas
\\
WEB-INF
\\
lib
\\
*;metadata_home
\\
libext
\\
*;metadata_home
\\
hbase
\\
conf'
,
[
'-Datlas.log.dir=metadata_home
\\
logs'
,
'-Datlas.log.file=application.log'
,
'-Datlas.home=metadata_home'
,
'-Datlas.conf=metadata_home
\\
conf'
,
'-Xmx1024m'
,
'-XX:MaxPermSize=512m'
,
'-Dlog4j.configuration=atlas-log4j.xml'
],
'metadata_home
\\
logs'
)
else
:
java_mock
.
assert_called_with
(
'org.apache.atlas.Atlas'
,
[
'-app'
,
'metadata_home/server/webapp/atlas'
],
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib/*:metadata_home/libext/*:metadata_home/hbase/conf'
,
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib/
atlas-titan-${project.version}.jar:metadata_home/server/webapp/atlas/WEB-INF/lib/
*:metadata_home/libext/*:metadata_home/hbase/conf'
,
[
'-Datlas.log.dir=metadata_home/logs'
,
'-Datlas.log.file=application.log'
,
'-Datlas.home=metadata_home'
,
'-Datlas.conf=metadata_home/conf'
,
'-Xmx1024m'
,
'-XX:MaxPermSize=512m'
,
'-Dlog4j.configuration=atlas-log4j.xml'
],
'metadata_home/logs'
)
pass
def
test_jar_java_lookups_fail
(
self
):
...
...
This diff is collapsed.
Click to expand it.
pom.xml
View file @
919120f6
...
...
@@ -409,6 +409,7 @@
<module>
typesystem
</module>
<module>
notification
</module>
<module>
client
</module>
<module>
titan
</module>
<module>
repository
</module>
<module>
dashboard
</module>
<module>
webapp
</module>
...
...
@@ -925,6 +926,12 @@
<dependency>
<groupId>
org.apache.atlas
</groupId>
<artifactId>
atlas-titan
</artifactId>
<version>
${project.version}
</version>
</dependency>
<dependency>
<groupId>
org.apache.atlas
</groupId>
<artifactId>
atlas-repository
</artifactId>
<version>
${project.version}
</version>
</dependency>
...
...
This diff is collapsed.
Click to expand it.
release-log.txt
View file @
919120f6
...
...
@@ -9,6 +9,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
ATLAS-352 Improve write performance on type and entity creation with Hbase (sumasai)
ATLAS-350 Document jaas config details for atlas (tbeerbower via shwethags)
ATLAS-344 Document HBase permissions for secure cluster (tbeerbower via shwethags)
ATLAS-335 Kerberized cluster: Atlas fails to come up with hbase as backend (sumasai via shwethags)
...
...
This diff is collapsed.
Click to expand it.
repository/pom.xml
View file @
919120f6
...
...
@@ -50,6 +50,11 @@
</dependency>
<dependency>
<groupId>
org.apache.atlas
</groupId>
<artifactId>
atlas-titan
</artifactId>
</dependency>
<dependency>
<groupId>
joda-time
</groupId>
<artifactId>
joda-time
</artifactId>
</dependency>
...
...
@@ -85,52 +90,6 @@
</dependency>
<dependency>
<groupId>
com.thinkaurelius.titan
</groupId>
<artifactId>
titan-core
</artifactId>
</dependency>
<dependency>
<groupId>
com.thinkaurelius.titan
</groupId>
<artifactId>
titan-es
</artifactId>
</dependency>
<dependency>
<groupId>
com.vividsolutions
</groupId>
<artifactId>
jts
</artifactId>
</dependency>
<dependency>
<groupId>
org.apache.solr
</groupId>
<artifactId>
solr-core
</artifactId>
</dependency>
<dependency>
<groupId>
org.apache.solr
</groupId>
<artifactId>
solr-solrj
</artifactId>
</dependency>
<dependency>
<groupId>
com.thinkaurelius.titan
</groupId>
<artifactId>
titan-berkeleyje
</artifactId>
</dependency>
<!-- Commenting out since titan-hbase classes are shaded for 1.x support -->
<!--<dependency>-->
<!--<groupId>com.thinkaurelius.titan</groupId>-->
<!--<artifactId>titan-hbase</artifactId>-->
<!--</dependency>-->
<dependency>
<groupId>
org.apache.hbase
</groupId>
<artifactId>
hbase-client
</artifactId>
</dependency>
<dependency>
<groupId>
com.thinkaurelius.titan
</groupId>
<artifactId>
titan-lucene
</artifactId>
</dependency>
<dependency>
<groupId>
com.tinkerpop.gremlin
</groupId>
<artifactId>
gremlin-java
</artifactId>
</dependency>
...
...
This diff is collapsed.
Click to expand it.
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
View file @
919120f6
...
...
@@ -27,6 +27,7 @@ import org.aopalliance.intercept.MethodInterceptor;
import
org.apache.atlas.discovery.DiscoveryService
;
import
org.apache.atlas.discovery.HiveLineageService
;
import
org.apache.atlas.discovery.LineageService
;
import
org.apache.atlas.discovery.SearchIndexer
;
import
org.apache.atlas.discovery.graph.GraphBackedDiscoveryService
;
import
org.apache.atlas.listener.TypesChangeListener
;
import
org.apache.atlas.repository.MetadataRepository
;
...
...
This diff is collapsed.
Click to expand it.
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
View file @
919120f6
...
...
@@ -18,9 +18,11 @@
package
org
.
apache
.
atlas
.
repository
.
graph
;
import
com.google.inject.Inject
;
import
com.thinkaurelius.titan.core.TitanFactory
;
import
com.thinkaurelius.titan.core.TitanGraph
;
import
com.thinkaurelius.titan.core.TitanIndexQuery
;
import
com.thinkaurelius.titan.core.util.TitanCleanup
;
import
com.thinkaurelius.titan.diskstorage.BackendException
;
import
com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration
;
import
com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration
;
...
...
@@ -30,6 +32,7 @@ import com.tinkerpop.blueprints.GraphQuery;
import
com.tinkerpop.blueprints.Predicate
;
import
com.tinkerpop.blueprints.Vertex
;
import
org.apache.atlas.GraphTransaction
;
import
org.apache.atlas.RepositoryMetadataModule
;
import
org.apache.atlas.TestUtils
;
import
org.apache.atlas.repository.Constants
;
import
org.apache.atlas.typesystem.ITypedReferenceableInstance
;
...
...
@@ -43,6 +46,7 @@ import org.apache.commons.io.FileUtils;
import
org.testng.Assert
;
import
org.testng.annotations.AfterClass
;
import
org.testng.annotations.BeforeClass
;
import
org.testng.annotations.Guice
;
import
org.testng.annotations.Test
;
import
java.io.File
;
...
...
@@ -53,6 +57,7 @@ import java.util.Date;
import
java.util.Random
;
@Test
@Guice
(
modules
=
RepositoryMetadataModule
.
class
)
public
class
GraphRepoMapperScaleTest
{
private
static
final
String
DATABASE_NAME
=
"foo"
;
...
...
@@ -61,50 +66,21 @@ public class GraphRepoMapperScaleTest {
private
static
final
String
INDEX_DIR
=
System
.
getProperty
(
"java.io.tmpdir"
,
"/tmp"
)
+
"/atlas-test"
+
new
Random
().
nextLong
();
private
GraphProvider
<
TitanGraph
>
graphProvider
=
new
GraphProvider
<
TitanGraph
>()
{
private
TitanGraph
graph
=
null
;
//Ensure separate directory for graph provider to avoid issues with index merging
@Override
public
TitanGraph
get
()
{
try
{
if
(
graph
==
null
)
{
synchronized
(
GraphRepoMapperScaleTest
.
class
)
{
if
(
graph
==
null
)
{
ReadConfiguration
config
=
new
CommonsConfiguration
()
{{
set
(
"storage.backend"
,
"inmemory"
);
set
(
"index.search.directory"
,
INDEX_DIR
);
set
(
"index.search.backend"
,
"elasticsearch"
);
set
(
"index.search.elasticsearch.local-mode"
,
"true"
);
set
(
"index.search.elasticsearch.client-only"
,
"false"
);
}};
GraphDatabaseConfiguration
graphconfig
=
new
GraphDatabaseConfiguration
(
config
);
graphconfig
.
getBackend
().
clearStorage
();
graph
=
TitanFactory
.
open
(
config
);
}
}
}
}
catch
(
BackendException
e
)
{
e
.
printStackTrace
();
}
return
graph
;
}
};
@Inject
GraphProvider
<
TitanGraph
>
graphProvider
;
@Inject
private
GraphBackedMetadataRepository
repositoryService
;
private
GraphBackedSearchIndexer
searchIndexer
;
private
TypeSystem
typeSystem
=
TypeSystem
.
getInstance
();
private
String
dbGUID
;
@BeforeClass
@GraphTransaction
public
void
setUp
()
throws
Exception
{
//Make sure we can cleanup the index directory
repositoryService
=
new
GraphBackedMetadataRepository
(
graphProvider
);
searchIndexer
=
new
GraphBackedSearchIndexer
(
graphProvider
);
Collection
<
IDataType
>
typesAdded
=
TestUtils
.
createHiveTypes
(
typeSystem
);
searchIndexer
.
onAdd
(
typesAdded
);
...
...
@@ -112,11 +88,17 @@ public class GraphRepoMapperScaleTest {
@AfterClass
public
void
tearDown
()
throws
Exception
{
TypeSystem
.
getInstance
().
reset
();
try
{
//TODO - Fix failure during shutdown while using BDB
graphProvider
.
get
().
shutdown
();
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
try
{
FileUtils
.
deleteDirectory
(
new
File
(
INDEX_DIR
));
}
catch
(
IOException
io
e
)
{
System
.
err
.
println
(
"Failed to cleanup index directory"
);
TitanCleanup
.
clear
(
graphProvider
.
get
(
));
}
catch
(
Exception
e
)
{
e
.
printStackTrace
(
);
}
}
...
...
@@ -142,6 +124,10 @@ public class GraphRepoMapperScaleTest {
@Test
(
dependsOnMethods
=
"testSubmitEntity"
)
public
void
testSearchIndex
()
throws
Exception
{
//Elasticsearch requires some time before index is updated
Thread
.
sleep
(
5000
);
searchWithOutIndex
(
Constants
.
GUID_PROPERTY_KEY
,
dbGUID
);
searchWithOutIndex
(
Constants
.
ENTITY_TYPE_PROPERTY_KEY
,
"column_type"
);
searchWithOutIndex
(
Constants
.
ENTITY_TYPE_PROPERTY_KEY
,
TestUtils
.
TABLE_TYPE
);
...
...
This diff is collapsed.
Click to expand it.
titan/pom.xml
0 → 100644
View file @
919120f6
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project
xmlns=
"http://maven.apache.org/POM/4.0.0"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation=
"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
>
<parent>
<artifactId>
apache-atlas
</artifactId>
<groupId>
org.apache.atlas
</groupId>
<version>
0.6-incubating-SNAPSHOT
</version>
</parent>
<modelVersion>
4.0.0
</modelVersion>
<artifactId>
atlas-titan
</artifactId>
<description>
Apache Atlas Titan Overrides
</description>
<name>
Apache Atlas Titan
</name>
<packaging>
jar
</packaging>
<dependencies>
<dependency>
<groupId>
com.thinkaurelius.titan
</groupId>
<artifactId>
titan-core
</artifactId>
</dependency>
<dependency>
<groupId>
org.apache.hbase
</groupId>
<artifactId>
hbase-client
</artifactId>
</dependency>
<dependency>
<groupId>
com.vividsolutions
</groupId>
<artifactId>
jts
</artifactId>
</dependency>
<dependency>
<groupId>
org.apache.solr
</groupId>
<artifactId>
solr-core
</artifactId>
</dependency>
<dependency>
<groupId>
org.apache.solr
</groupId>
<artifactId>
solr-solrj
</artifactId>
</dependency>
<dependency>
<groupId>
com.thinkaurelius.titan
</groupId>
<artifactId>
titan-es
</artifactId>
</dependency>
<dependency>
<groupId>
com.thinkaurelius.titan
</groupId>
<artifactId>
titan-berkeleyje
</artifactId>
</dependency>
<dependency>
<groupId>
com.thinkaurelius.titan
</groupId>
<artifactId>
titan-lucene
</artifactId>
</dependency>
<dependency>
<groupId>
org.testng
</groupId>
<artifactId>
testng
</artifactId>
</dependency>
<dependency>
<groupId>
org.mockito
</groupId>
<artifactId>
mockito-all
</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-jar-plugin
</artifactId>
<version>
2.4
</version>
<configuration>
<excludes>
<exclude>
**/log4j.xml
</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>
net.alchim31.maven
</groupId>
<artifactId>
scala-maven-plugin
</artifactId>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
View file @
919120f6
...
...
@@ -19,7 +19,6 @@ import java.io.IOException;
import
org.apache.hadoop.conf.Configuration
;
import
org.apache.hadoop.hbase.HColumnDescriptor
;
import
org.apache.hadoop.hbase.HTableDescriptor
;
import
org.apache.hadoop.hbase.ZooKeeperConnectionException
;
import
org.apache.hadoop.hbase.client.Delete
;
public
interface
HBaseCompat
{
...
...
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
View file @
919120f6
...
...
@@ -22,7 +22,6 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import
org.apache.hadoop.hbase.TableName
;
import
org.apache.hadoop.hbase.client.ConnectionFactory
;
import
org.apache.hadoop.hbase.client.Delete
;
import
org.apache.hadoop.hbase.client.HConnectionManager
;
import
org.apache.hadoop.hbase.io.compress.Compression
;
public
class
HBaseCompat1_0
implements
HBaseCompat
{
...
...
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
View file @
919120f6
...
...
@@ -18,12 +18,19 @@ import com.google.common.base.Predicate;
import
com.google.common.collect.ImmutableMap
;
import
com.google.common.collect.Iterables
;
import
com.google.common.collect.Iterators
;
import
com.thinkaurelius.titan.core.attribute.Duration
;
import
com.thinkaurelius.titan.diskstorage.*
;
import
com.thinkaurelius.titan.diskstorage.keycolumnvalue.*
;
import
com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator
;
import
com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException
;
import
com.thinkaurelius.titan.diskstorage.util.KeyColumn
;
import
com.thinkaurelius.titan.diskstorage.util.RecordIterator
;
import
com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer
;
import
com.thinkaurelius.titan.diskstorage.util.StaticArrayEntry
;
import
com.thinkaurelius.titan.diskstorage.util.StaticArrayEntryList
;
import
com.thinkaurelius.titan.diskstorage.util.time.Timepoint
;
import
com.thinkaurelius.titan.diskstorage.util.time.Timestamps
;
import
com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration
;
import
com.thinkaurelius.titan.util.system.IOUtils
;
import
org.apache.hadoop.hbase.client.*
;
...
...
@@ -31,6 +38,7 @@ import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
import
org.apache.hadoop.hbase.filter.ColumnRangeFilter
;
import
org.apache.hadoop.hbase.filter.Filter
;
import
org.apache.hadoop.hbase.filter.FilterList
;
import
org.apache.hadoop.hbase.util.Bytes
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -39,6 +47,10 @@ import javax.annotation.Nullable;
import
java.io.Closeable
;
import
java.io.IOException
;
import
java.util.*
;
import
java.util.Map
;
import
java.util.concurrent.TimeUnit
;
import
static
com
.
thinkaurelius
.
titan
.
diskstorage
.
EntryMetaData
.*;
/**
* Here are some areas that might need work:
...
...
@@ -71,7 +83,11 @@ public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
private
final
ConnectionMask
cnx
;
HBaseKeyColumnValueStore
(
HBaseStoreManager
storeManager
,
ConnectionMask
cnx
,
String
tableName
,
String
columnFamily
,
String
storeName
)
{
private
LocalLockMediator
<
StoreTransaction
>
localLockMediator
;
private
Duration
lockExpiryTime
;
HBaseKeyColumnValueStore
(
HBaseStoreManager
storeManager
,
ConnectionMask
cnx
,
String
tableName
,
String
columnFamily
,
String
storeName
,
LocalLockMediator
<
StoreTransaction
>
llm
)
{
this
.
storeManager
=
storeManager
;
this
.
cnx
=
cnx
;
this
.
tableName
=
tableName
;
...
...
@@ -79,6 +95,8 @@ public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
this
.
storeName
=
storeName
;
this
.
columnFamilyBytes
=
columnFamily
.
getBytes
();
this
.
entryGetter
=
new
HBaseGetter
(
storeManager
.
getMetaDataSchema
(
storeName
));
this
.
localLockMediator
=
llm
;
this
.
lockExpiryTime
=
storeManager
.
getStorageConfig
().
get
(
GraphDatabaseConfiguration
.
LOCK_EXPIRE
);
}
@Override
...
...
@@ -107,7 +125,15 @@ public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
StaticBuffer
column
,
StaticBuffer
expectedValue
,
StoreTransaction
txh
)
throws
BackendException
{
throw
new
UnsupportedOperationException
();
KeyColumn
lockID
=
new
KeyColumn
(
key
,
column
);
logger
.
debug
(
"Attempting to acquireLock on {} "
,
lockID
);
final
Timepoint
lockStartTime
=
Timestamps
.
NANO
.
getTime
(
System
.
nanoTime
(),
TimeUnit
.
NANOSECONDS
);
boolean
locked
=
localLockMediator
.
lock
(
lockID
,
txh
,
lockStartTime
.
add
(
lockExpiryTime
));
if
(!
locked
)
{
throw
new
PermanentLockingException
(
"Could not lock the keyColumn "
+
lockID
+
" on CF {} "
+
Bytes
.
toString
(
columnFamilyBytes
));
}
((
HBaseTransaction
)
txh
).
updateLocks
(
lockID
,
expectedValue
);
}
@Override
...
...
@@ -167,7 +193,9 @@ public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
try
{
table
=
cnx
.
getTable
(
tableName
);
logger
.
debug
(
"Get requests {} {} "
,
Bytes
.
toString
(
columnFamilyBytes
),
requests
.
size
());
results
=
table
.
get
(
requests
);
logger
.
debug
(
"Get requests finished {} {} "
,
Bytes
.
toString
(
columnFamilyBytes
),
requests
.
size
());
}
finally
{
IOUtils
.
closeQuietly
(
table
);
}
...
...
@@ -231,6 +259,7 @@ public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
TableMask
table
=
null
;
logger
.
debug
(
"Scan for row keys {} {} "
,
Bytes
.
toString
(
startKey
),
Bytes
.
toString
(
endKey
));
try
{
table
=
cnx
.
getTable
(
tableName
);
return
new
RowIterator
(
table
,
table
.
getScanner
(
scan
.
setFilter
(
filters
)),
columnFamilyBytes
);
...
...
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
View file @
919120f6
...
...
@@ -14,14 +14,6 @@
*/
package
com
.
thinkaurelius
.
titan
.
diskstorage
.
hbase
;
import
static
com
.
thinkaurelius
.
titan
.
diskstorage
.
Backend
.
EDGESTORE_NAME
;
import
static
com
.
thinkaurelius
.
titan
.
diskstorage
.
Backend
.
ID_STORE_NAME
;
import
static
com
.
thinkaurelius
.
titan
.
diskstorage
.
Backend
.
INDEXSTORE_NAME
;
import
static
com
.
thinkaurelius
.
titan
.
diskstorage
.
Backend
.
LOCK_STORE_SUFFIX
;
import
static
com
.
thinkaurelius
.
titan
.
diskstorage
.
Backend
.
SYSTEM_MGMT_LOG_NAME
;
import
static
com
.
thinkaurelius
.
titan
.
diskstorage
.
Backend
.
SYSTEM_TX_LOG_NAME
;
import
static
com
.
thinkaurelius
.
titan
.
graphdb
.
configuration
.
GraphDatabaseConfiguration
.
SYSTEM_PROPERTIES_STORE_NAME
;
import
java.io.IOException
;
import
java.nio.ByteBuffer
;
import
java.util.ArrayList
;
...
...
@@ -34,8 +26,11 @@ import java.util.NavigableMap;
import
java.util.concurrent.ConcurrentHashMap
;
import
java.util.concurrent.ConcurrentMap
;
import
com.thinkaurelius.titan.diskstorage.Backend
;
import
com.thinkaurelius.titan.diskstorage.configuration.ConfigElement
;
import
com.thinkaurelius.titan.diskstorage.keycolumnvalue.CustomizeStoreKCVSManager
;
import
com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator
;
import
com.thinkaurelius.titan.diskstorage.locking.LocalLockMediators
;
import
com.thinkaurelius.titan.diskstorage.util.time.Timestamps
;
import
org.apache.hadoop.hbase.HBaseConfiguration
;
import
org.apache.hadoop.hbase.HColumnDescriptor
;
...
...
@@ -51,6 +46,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import
org.apache.hadoop.hbase.client.HTable
;
import
org.apache.hadoop.hbase.client.Put
;
import
org.apache.hadoop.hbase.client.Row
;
import
org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
;
import
org.apache.hadoop.hbase.util.Pair
;
import
org.apache.hadoop.hbase.util.VersionInfo
;
import
org.slf4j.Logger
;
...
...
@@ -236,15 +232,15 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
private
static
final
BiMap
<
String
,
String
>
SHORT_CF_NAME_MAP
=
ImmutableBiMap
.<
String
,
String
>
builder
()
.
put
(
INDEXSTORE_NAME
,
"g"
)
.
put
(
INDEXSTORE_NAME
+
LOCK_STORE_SUFFIX
,
"h"
)
.
put
(
ID_STORE_NAME
,
"i"
)
.
put
(
EDGESTORE_NAME
,
"e"
)
.
put
(
EDGESTORE_NAME
+
LOCK_STORE_SUFFIX
,
"f"
)
.
put
(
SYSTEM_PROPERTIES_STORE_NAME
,
"s"
)
.
put
(
SYSTEM_PROPERTIES_STORE_NAME
+
LOCK_STORE_SUFFIX
,
"t"
)
.
put
(
SYSTEM_MGMT_LOG_NAME
,
"m"
)
.
put
(
SYSTEM_TX_LOG_NAME
,
"l"
)
.
put
(
Backend
.
INDEXSTORE_NAME
,
"g"
)
.
put
(
Backend
.
INDEXSTORE_NAME
+
Backend
.
LOCK_STORE_SUFFIX
,
"h"
)
.
put
(
Backend
.
ID_STORE_NAME
,
"i"
)
.
put
(
Backend
.
EDGESTORE_NAME
,
"e"
)
.
put
(
Backend
.
EDGESTORE_NAME
+
Backend
.
LOCK_STORE_SUFFIX
,
"f"
)
.
put
(
GraphDatabaseConfiguration
.
SYSTEM_PROPERTIES_STORE_NAME
,
"s"
)
.
put
(
GraphDatabaseConfiguration
.
SYSTEM_PROPERTIES_STORE_NAME
+
Backend
.
LOCK_STORE_SUFFIX
,
"t"
)
.
put
(
Backend
.
SYSTEM_MGMT_LOG_NAME
,
"m"
)
.
put
(
Backend
.
SYSTEM_TX_LOG_NAME
,
"l"
)
.
build
();
private
static
final
StaticBuffer
FOUR_ZERO_BYTES
=
BufferUtil
.
zeroBuffer
(
4
);
...
...
@@ -275,6 +271,8 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
// Mutable instance state
private
final
ConcurrentMap
<
String
,
HBaseKeyColumnValueStore
>
openStores
;
private
LocalLockMediator
<
StoreTransaction
>
llm
;
public
HBaseStoreManager
(
com
.
thinkaurelius
.
titan
.
diskstorage
.
configuration
.
Configuration
config
)
throws
BackendException
{
super
(
config
,
PORT_DEFAULT
);
...
...
@@ -390,6 +388,7 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
.
orderedScan
(
true
).
unorderedScan
(
true
).
batchMutation
(
true
)
.
multiQuery
(
true
).
distributed
(
true
).
keyOrdered
(
true
).
storeTTL
(
true
)
.
timestamps
(
true
).
preferredTimestamps
(
PREFERRED_TIMESTAMPS
)
.
locking
(
true
)
.
keyConsistent
(
c
);
try
{
...
...
@@ -403,6 +402,7 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
@Override
public
void
mutateMany
(
Map
<
String
,
Map
<
StaticBuffer
,
KCVMutation
>>
mutations
,
StoreTransaction
txh
)
throws
BackendException
{
logger
.
debug
(
"Enter mutateMany"
);
final
MaskedTimestamp
commitTime
=
new
MaskedTimestamp
(
txh
);
// In case of an addition and deletion with identical timestamps, the
// deletion tombstone wins.
...
...
@@ -429,7 +429,9 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
try
{
table
=
cnx
.
getTable
(
tableName
);
logger
.
debug
(
"mutateMany : batch mutate started size {} "
,
batch
.
size
());
table
.
batch
(
batch
,
new
Object
[
batch
.
size
()]);
logger
.
debug
(
"mutateMany : batch mutate finished {} "
,
batch
.
size
());
}
finally
{
IOUtils
.
closeQuietly
(
table
);
}
...
...
@@ -456,7 +458,9 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
if
(
store
==
null
)
{
final
String
cfName
=
shortCfNames
?
shortenCfName
(
longName
)
:
longName
;
HBaseKeyColumnValueStore
newStore
=
new
HBaseKeyColumnValueStore
(
this
,
cnx
,
tableName
,
cfName
,
longName
);
final
String
llmPrefix
=
getName
();
llm
=
LocalLockMediators
.
INSTANCE
.<
StoreTransaction
>
get
(
llmPrefix
,
times
);
HBaseKeyColumnValueStore
newStore
=
new
HBaseKeyColumnValueStore
(
this
,
cnx
,
tableName
,
cfName
,
longName
,
llm
);
store
=
openStores
.
putIfAbsent
(
longName
,
newStore
);
// nothing bad happens if we loose to other thread
...
...
@@ -475,7 +479,7 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
@Override
public
StoreTransaction
beginTransaction
(
final
BaseTransactionConfig
config
)
throws
BackendException
{
return
new
HBaseTransaction
(
config
);
return
new
HBaseTransaction
(
config
,
llm
);
}
@Override
...
...
@@ -804,12 +808,7 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
adm
.
addColumn
(
tableName
,
cdesc
);
try
{
logger
.
debug
(
"Added HBase ColumnFamily {}, waiting for 1 sec. to propogate."
,
columnFamily
);
Thread
.
sleep
(
1000L
);
}
catch
(
InterruptedException
ie
)
{
throw
new
TemporaryBackendException
(
ie
);
}
adm
.
enableTable
(
tableName
);
}
catch
(
TableNotFoundException
ee
)
{
...
...
@@ -832,6 +831,8 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
if
(
ttlInSeconds
>
0
)
cdesc
.
setTimeToLive
(
ttlInSeconds
);
cdesc
.
setDataBlockEncoding
(
DataBlockEncoding
.
FAST_DIFF
);
}
/**
...
...
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
View file @
919120f6
...
...
@@ -14,8 +14,18 @@
*/
package
com
.
thinkaurelius
.
titan
.
diskstorage
.
hbase
;
import
com.thinkaurelius.titan.diskstorage.BackendException
;
import
com.thinkaurelius.titan.diskstorage.BaseTransactionConfig
;
import
com.thinkaurelius.titan.diskstorage.StaticBuffer
;
import
com.thinkaurelius.titan.diskstorage.common.AbstractStoreTransaction
;
import
com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction
;
import
com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator
;
import
com.thinkaurelius.titan.diskstorage.util.KeyColumn
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.util.LinkedHashSet
;
import
java.util.Set
;
/**
* This class overrides and adds nothing compared with
...
...
@@ -27,7 +37,39 @@ import com.thinkaurelius.titan.diskstorage.common.AbstractStoreTransaction;
*/
public
class
HBaseTransaction
extends
AbstractStoreTransaction
{
public
HBaseTransaction
(
final
BaseTransactionConfig
config
)
{
private
static
final
Logger
log
=
LoggerFactory
.
getLogger
(
HBaseTransaction
.
class
);
LocalLockMediator
<
StoreTransaction
>
llm
;
Set
<
KeyColumn
>
keyColumnLocks
=
new
LinkedHashSet
<>();
public
HBaseTransaction
(
final
BaseTransactionConfig
config
,
LocalLockMediator
<
StoreTransaction
>
llm
)
{
super
(
config
);
this
.
llm
=
llm
;
}
@Override
public
synchronized
void
rollback
()
throws
BackendException
{
super
.
rollback
();
log
.
debug
(
"Rolled back transaction"
);
deleteAllLocks
();
}
@Override
public
synchronized
void
commit
()
throws
BackendException
{
super
.
commit
();
log
.
debug
(
"Committed transaction"
);
deleteAllLocks
();
}
public
void
updateLocks
(
KeyColumn
lockID
,
StaticBuffer
expectedValue
)
{
keyColumnLocks
.
add
(
lockID
);
}
private
void
deleteAllLocks
()
{
for
(
KeyColumn
kc
:
keyColumnLocks
)
{
log
.
debug
(
"Removed lock {} "
,
kc
);
llm
.
unlock
(
kc
,
this
);
}
}
}
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
View file @
919120f6
...
...
@@ -18,7 +18,6 @@ import java.io.IOException;
import
java.util.List
;
import
org.apache.hadoop.hbase.client.Get
;
import
org.apache.hadoop.hbase.client.HTableInterface
;
import
org.apache.hadoop.hbase.client.Result
;
import
org.apache.hadoop.hbase.client.ResultScanner
;
import
org.apache.hadoop.hbase.client.Row
;
...
...
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
View file @
919120f6
File moved
This diff is collapsed.
Click to expand it.
titan/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java
0 → 100644
View file @
919120f6
This diff is collapsed.
Click to expand it.
repository
/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
→
titan
/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
View file @
919120f6
...
...
@@ -41,6 +41,7 @@ import com.thinkaurelius.titan.diskstorage.indexing.IndexQuery;
import
com.thinkaurelius.titan.diskstorage.indexing.KeyInformation
;
import
com.thinkaurelius.titan.diskstorage.indexing.RawQuery
;
import
com.thinkaurelius.titan.diskstorage.util.DefaultTransaction
;
import
com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration
;
import
com.thinkaurelius.titan.graphdb.configuration.PreInitializeConfigOptions
;
import
com.thinkaurelius.titan.graphdb.database.serialize.AttributeUtil
;
import
com.thinkaurelius.titan.graphdb.database.serialize.attribute.AbstractDecimal
;
...
...
@@ -89,8 +90,8 @@ import java.util.Map;
import
java.util.TimeZone
;
import
java.util.UUID
;
import
static
com
.
thinkaurelius
.
titan
.
graphdb
.
configuration
.
GraphDatabaseConfiguration
.
INDEX_MAX_RESULT_SET_SIZE
;
import
static
com
.
thinkaurelius
.
titan
.
graphdb
.
configuration
.
GraphDatabaseConfiguration
.
INDEX_NS
;
import
static
com
.
thinkaurelius
.
titan
.
core
.
attribute
.
Cmp
.*
;
import
static
com
.
thinkaurelius
.
titan
.
core
.
schema
.
Mapping
.*
;
/**
* NOTE: Copied from titan for supporting sol5. Do not change
...
...
@@ -115,7 +116,7 @@ public class Solr5Index implements IndexProvider {
}
public
static
final
ConfigNamespace
SOLR_NS
=
new
ConfigNamespace
(
INDEX_NS
,
"solr"
,
"Solr index configuration"
);
new
ConfigNamespace
(
GraphDatabaseConfiguration
.
INDEX_NS
,
"solr"
,
"Solr index configuration"
);
public
static
final
ConfigOption
<
String
>
SOLR_MODE
=
new
ConfigOption
<
String
>(
SOLR_NS
,
"mode"
,
"The operation mode for Solr which is either via HTTP (`http`) or using SolrCloud (`cloud`)"
,
...
...
@@ -182,7 +183,7 @@ public class Solr5Index implements IndexProvider {
private
static
final
IndexFeatures
SOLR_FEATURES
=
new
IndexFeatures
.
Builder
().
supportsDocumentTTL
()
.
setDefaultStringMapping
(
Mapping
.
TEXT
).
supportedStringMappings
(
Mapping
.
TEXT
,
Mapping
.
STRING
).
build
();
.
setDefaultStringMapping
(
TEXT
).
supportedStringMappings
(
TEXT
,
STRING
).
build
();
private
final
SolrClient
solrClient
;
private
final
Configuration
configuration
;
...
...
@@ -200,7 +201,7 @@ public class Solr5Index implements IndexProvider {
mode
=
Mode
.
parse
(
config
.
get
(
SOLR_MODE
));
dynFields
=
config
.
get
(
DYNAMIC_FIELDS
);
keyFieldIds
=
parseKeyFieldsForCollections
(
config
);
maxResults
=
config
.
get
(
INDEX_MAX_RESULT_SET_SIZE
);
maxResults
=
config
.
get
(
GraphDatabaseConfiguration
.
INDEX_MAX_RESULT_SET_SIZE
);
ttlField
=
config
.
get
(
TTL_FIELD
);
waitSearcher
=
config
.
get
(
WAIT_SEARCHER
);
...
...
@@ -556,10 +557,10 @@ public class Solr5Index implements IndexProvider {
}
}
else
if
(
value
instanceof
String
)
{
Mapping
map
=
getStringMapping
(
informations
.
get
(
key
));
assert
map
==
Mapping
.
TEXT
||
map
==
Mapping
.
STRING
;
if
(
map
==
Mapping
.
TEXT
&&
!
titanPredicate
.
toString
().
startsWith
(
"CONTAINS"
))
assert
map
==
TEXT
||
map
==
STRING
;
if
(
map
==
TEXT
&&
!
titanPredicate
.
toString
().
startsWith
(
"CONTAINS"
))
throw
new
IllegalArgumentException
(
"Text mapped string values only support CONTAINS queries and not: "
+
titanPredicate
);
if
(
map
==
Mapping
.
STRING
&&
titanPredicate
.
toString
().
startsWith
(
"CONTAINS"
))
if
(
map
==
STRING
&&
titanPredicate
.
toString
().
startsWith
(
"CONTAINS"
))
throw
new
IllegalArgumentException
(
"String mapped string values do not support CONTAINS queries: "
+
titanPredicate
);
//Special case
...
...
@@ -587,9 +588,9 @@ public class Solr5Index implements IndexProvider {
return
(
key
+
":"
+
escapeValue
(
value
)
+
"*"
);
}
else
if
(
titanPredicate
==
Text
.
REGEX
||
titanPredicate
==
Text
.
CONTAINS_REGEX
)
{
return
(
key
+
":/"
+
value
+
"/"
);
}
else
if
(
titanPredicate
==
Cmp
.
EQUAL
)
{
}
else
if
(
titanPredicate
==
EQUAL
)
{
return
(
key
+
":\""
+
escapeValue
(
value
)
+
"\""
);
}
else
if
(
titanPredicate
==
Cmp
.
NOT_EQUAL
)
{
}
else
if
(
titanPredicate
==
NOT_EQUAL
)
{
return
(
"-"
+
key
+
":\""
+
escapeValue
(
value
)
+
"\""
);
}
else
{
throw
new
IllegalArgumentException
(
"Relation is not supported for string value: "
+
titanPredicate
);
...
...
@@ -651,9 +652,9 @@ public class Solr5Index implements IndexProvider {
throw
new
IllegalArgumentException
(
"Boolean types only support EQUAL or NOT_EQUAL"
);
}
}
else
if
(
value
instanceof
UUID
)
{
if
(
titanPredicate
==
Cmp
.
EQUAL
)
{
if
(
titanPredicate
==
EQUAL
)
{
return
(
key
+
":\""
+
escapeValue
(
value
)
+
"\""
);
}
else
if
(
titanPredicate
==
Cmp
.
NOT_EQUAL
)
{
}
else
if
(
titanPredicate
==
NOT_EQUAL
)
{
return
(
"-"
+
key
+
":\""
+
escapeValue
(
value
)
+
"\""
);
}
else
{
throw
new
IllegalArgumentException
(
"Relation is not supported for uuid value: "
+
titanPredicate
);
...
...
@@ -779,8 +780,8 @@ public class Solr5Index implements IndexProvider {
@Override
public
boolean
supports
(
KeyInformation
information
,
TitanPredicate
titanPredicate
)
{
Class
<?>
dataType
=
information
.
getDataType
();
Mapping
mapping
=
Mapping
.
getMapping
(
information
);
if
(
mapping
!=
Mapping
.
DEFAULT
&&
!
AttributeUtil
.
isString
(
dataType
))
return
false
;
Mapping
mapping
=
getMapping
(
information
);
if
(
mapping
!=
DEFAULT
&&
!
AttributeUtil
.
isString
(
dataType
))
return
false
;
if
(
Number
.
class
.
isAssignableFrom
(
dataType
))
{
return
titanPredicate
instanceof
Cmp
;
...
...
@@ -792,16 +793,16 @@ public class Solr5Index implements IndexProvider {
case
TEXT:
return
titanPredicate
==
Text
.
CONTAINS
||
titanPredicate
==
Text
.
CONTAINS_PREFIX
||
titanPredicate
==
Text
.
CONTAINS_REGEX
;
case
STRING:
return
titanPredicate
==
Cmp
.
EQUAL
||
titanPredicate
==
Cmp
.
NOT_EQUAL
||
titanPredicate
==
Text
.
REGEX
||
titanPredicate
==
Text
.
PREFIX
;
return
titanPredicate
==
EQUAL
||
titanPredicate
==
NOT_EQUAL
||
titanPredicate
==
Text
.
REGEX
||
titanPredicate
==
Text
.
PREFIX
;
// case TEXTSTRING:
// return (titanPredicate instanceof Text) || titanPredicate == Cmp.EQUAL || titanPredicate==Cmp.NOT_EQUAL;
}
}
else
if
(
dataType
==
Date
.
class
)
{
if
(
titanPredicate
instanceof
Cmp
)
return
true
;
}
else
if
(
dataType
==
Boolean
.
class
)
{
return
titanPredicate
==
Cmp
.
EQUAL
||
titanPredicate
==
Cmp
.
NOT_EQUAL
;
return
titanPredicate
==
EQUAL
||
titanPredicate
==
NOT_EQUAL
;
}
else
if
(
dataType
==
UUID
.
class
)
{
return
titanPredicate
==
Cmp
.
EQUAL
||
titanPredicate
==
Cmp
.
NOT_EQUAL
;
return
titanPredicate
==
EQUAL
||
titanPredicate
==
NOT_EQUAL
;
}
return
false
;
}
...
...
@@ -809,11 +810,11 @@ public class Solr5Index implements IndexProvider {
@Override
public
boolean
supports
(
KeyInformation
information
)
{
Class
<?>
dataType
=
information
.
getDataType
();
Mapping
mapping
=
Mapping
.
getMapping
(
information
);
Mapping
mapping
=
getMapping
(
information
);
if
(
Number
.
class
.
isAssignableFrom
(
dataType
)
||
dataType
==
Geoshape
.
class
||
dataType
==
Date
.
class
||
dataType
==
Boolean
.
class
||
dataType
==
UUID
.
class
)
{
if
(
mapping
==
Mapping
.
DEFAULT
)
return
true
;
if
(
mapping
==
DEFAULT
)
return
true
;
}
else
if
(
AttributeUtil
.
isString
(
dataType
))
{
if
(
mapping
==
Mapping
.
DEFAULT
||
mapping
==
Mapping
.
TEXT
||
mapping
==
Mapping
.
STRING
)
return
true
;
if
(
mapping
==
DEFAULT
||
mapping
==
TEXT
||
mapping
==
STRING
)
return
true
;
}
return
false
;
}
...
...
@@ -861,8 +862,8 @@ public class Solr5Index implements IndexProvider {
private
static
Mapping
getStringMapping
(
KeyInformation
information
)
{
assert
AttributeUtil
.
isString
(
information
.
getDataType
());
Mapping
map
=
Mapping
.
getMapping
(
information
);
if
(
map
==
Mapping
.
DEFAULT
)
map
=
Mapping
.
TEXT
;
Mapping
map
=
getMapping
(
information
);
if
(
map
==
DEFAULT
)
map
=
TEXT
;
return
map
;
}
...
...
This diff is collapsed.
Click to expand it.
titan/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java
0 → 100644
View file @
919120f6
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
com
.
thinkaurelius
.
titan
.
diskstorage
.
locking
;
import
com.thinkaurelius.titan.diskstorage.hbase.HBaseTransaction
;
import
com.thinkaurelius.titan.diskstorage.util.time.TimestampProvider
;
import
com.thinkaurelius.titan.diskstorage.util.time.Timestamps
;
import
com.thinkaurelius.titan.diskstorage.StaticBuffer
;
import
com.thinkaurelius.titan.diskstorage.util.KeyColumn
;
import
com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer
;
import
org.mockito.Mockito
;
import
org.testng.Assert
;
import
org.testng.annotations.Test
;
import
java.util.concurrent.TimeUnit
;
public
class
LocalLockMediatorTest
{
private
static
final
String
LOCK_NAMESPACE
=
"test"
;
private
static
final
StaticBuffer
LOCK_ROW
=
StaticArrayBuffer
.
of
(
new
byte
[]{
1
});
private
static
final
StaticBuffer
LOCK_COL
=
StaticArrayBuffer
.
of
(
new
byte
[]{
1
});
private
static
final
KeyColumn
kc
=
new
KeyColumn
(
LOCK_ROW
,
LOCK_COL
);
private
static
final
HBaseTransaction
mockTx1
=
Mockito
.
mock
(
HBaseTransaction
.
class
);
private
static
final
HBaseTransaction
mockTx2
=
Mockito
.
mock
(
HBaseTransaction
.
class
);
@Test
public
void
testLock
()
throws
InterruptedException
{
TimestampProvider
times
=
Timestamps
.
MICRO
;
LocalLockMediator
<
HBaseTransaction
>
llm
=
new
LocalLockMediator
<
HBaseTransaction
>(
LOCK_NAMESPACE
,
times
);
//Expire immediately
Assert
.
assertTrue
(
llm
.
lock
(
kc
,
mockTx1
,
times
.
getTime
(
0
,
TimeUnit
.
NANOSECONDS
)));
Assert
.
assertTrue
(
llm
.
lock
(
kc
,
mockTx2
,
times
.
getTime
(
Long
.
MAX_VALUE
,
TimeUnit
.
NANOSECONDS
)));
llm
=
new
LocalLockMediator
<
HBaseTransaction
>(
LOCK_NAMESPACE
,
times
);
//Expire later
Assert
.
assertTrue
(
llm
.
lock
(
kc
,
mockTx1
,
times
.
getTime
(
Long
.
MAX_VALUE
,
TimeUnit
.
NANOSECONDS
)));
//So second lock should fail on same keyCol
Assert
.
assertFalse
(
llm
.
lock
(
kc
,
mockTx2
,
times
.
getTime
(
Long
.
MAX_VALUE
,
TimeUnit
.
NANOSECONDS
)));
//Unlock
Assert
.
assertTrue
(
llm
.
unlock
(
kc
,
mockTx1
));
//Now locking should succeed
Assert
.
assertTrue
(
llm
.
lock
(
kc
,
mockTx2
,
times
.
getTime
(
Long
.
MAX_VALUE
,
TimeUnit
.
NANOSECONDS
)));
}
}
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment