Skip to content

A
atlas
Commit effb7537 by Sarath Subramanian
Options

ATLAS-2521: Remove Titan 0.5.4 support from Atlas

parent 8b65aed0
Showing with 31 additions and 8698 deletions
addons/falcon-bridge-shim/pom.xml
......@@ -52,10 +52,6 @@
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
......
addons/falcon-bridge/pom.xml
......@@ -67,10 +67,6 @@
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
......
addons/falcon-bridge/src/test/resources/atlas-application.properties
......@@ -39,7 +39,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
......
addons/falcon-bridge/src/test/resources/atlas-log4j.xml
......@@ -88,7 +88,7 @@
<appender-ref ref="FILE"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
</logger>
......
addons/hbase-bridge/src/test/resources/atlas-application.properties
......@@ -39,7 +39,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
......
addons/hbase-bridge/src/test/resources/atlas-log4j.xml
......@@ -88,7 +88,7 @@
<appender-ref ref="FILE"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
</logger>
......
addons/hive-bridge/src/main/resources/atlas-hive-import-log4j.xml
......@@ -43,11 +43,6 @@
<appender-ref ref="FILE"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
</logger>
<!-- to avoid logs - The configuration log.flush.interval.messages = 1 was supplied but isn't a known config -->
<logger name="org.apache.kafka.common.config.AbstractConfig" additivity="false">
<level value="error"/>
......
addons/hive-bridge/src/test/resources/atlas-application.properties
......@@ -39,7 +39,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
......
addons/hive-bridge/src/test/resources/atlas-log4j.xml
......@@ -88,7 +88,7 @@
<appender-ref ref="FILE"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
</logger>
......
addons/sqoop-bridge/src/test/resources/atlas-application.properties
......@@ -39,7 +39,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
......
addons/sqoop-bridge/src/test/resources/atlas-log4j.xml
......@@ -88,7 +88,7 @@
<appender-ref ref="FILE"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
</logger>
......
addons/storm-bridge/src/test/resources/atlas-application.properties
......@@ -41,7 +41,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
......
addons/storm-bridge/src/test/resources/atlas-log4j.xml
......@@ -88,7 +88,7 @@
<appender-ref ref="FILE"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
</logger>
......
authorization/src/test/resources/atlas-application.properties
......@@ -57,7 +57,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
......
build-tools/src/main/resources/checkstyle-suppressions.xml
......@@ -23,7 +23,4 @@
<suppressions>
<suppress checks="JavadocType" files="[/\\]src[/\\]test[/\\]java[/\\]"/>
<!-- skip checks on customized titan 0.5.4 files -->
<suppress checks="[a-zA-Z0-9]*" files="[/\\]com[/\\]thinkaurelius[/\\]titan[/\\]"/>
</suppressions>
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -47,7 +47,6 @@ public final class Constants {
/**
* Full-text for the entity for enabling full-text search.
*/
//weird issue in TitanDB if __ added to this property key. Not adding it for now
public static final String ENTITY_TEXT_PROPERTY_KEY = "entityText";
/**
......
common/src/test/resources/atlas-log4j.xml
......@@ -48,9 +48,9 @@
</logger>
-->
<logger name="com.thinkaurelius.titan" additivity="false">
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="console"/>
<appender-ref ref="FILE"/>
</logger>
<logger name="org.springframework" additivity="false">
......
distro/pom.xml
......@@ -35,7 +35,7 @@
<graph.storage.backend>hbase</graph.storage.backend>
<graph.storage.properties>#Hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=
atlas.graph.storage.hbase.regions-per-server=1
atlas.graph.storage.lock.wait-time=10000
......@@ -173,7 +173,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
<properties>
<graph.storage.properties>#Hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=localhost
atlas.graph.storage.hbase.regions-per-server=1
atlas.graph.storage.lock.wait-time=10000
......
distro/src/conf/atlas-env.sh
......@@ -49,7 +49,7 @@
# Where pid files are stored. Defatult is logs directory under the base install location
#export ATLAS_PID_DIR=
# where the atlas titan db data is stored. Defatult is logs/data directory under the base install location
# where the atlas janusgraph db data is stored. Defatult is logs/data directory under the base install location
#export ATLAS_DATA_DIR=
# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
......
distro/src/conf/atlas-log4j.xml
......@@ -88,7 +88,7 @@
<appender-ref ref="FILE"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
</logger>
......
distro/src/conf/solr/schema.xml
......@@ -519,7 +519,6 @@
-->
<fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml" />
<!--Titan specific-->
<fieldType name="uuid"
class="solr.UUIDField"
indexed="true" />
......
distro/src/conf/solr/solrconfig.xml
......@@ -606,7 +606,6 @@
</admin>
<!--Titan specific-->
<updateRequestProcessorChain default="true">
<processor class="solr.TimestampUpdateProcessorFactory">
<str name="fieldName">timestamp</str>
......
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/tinkerpop/query/TinkerpopGraphQuery.java
......@@ -38,8 +38,7 @@ import java.util.Set;
/**
*
* Abstract implementation of AtlasGraphQuery that is used by Titan 0.5.4,
* Titan 1.0.0 and JanusGraph
* Abstract implementation of AtlasGraphQuery that is used by JanusGraph
* <p>
* Represents a graph query as an OrConditions which consists of
* 1 or more AndConditions. The query is executed by converting
......
graphdb/janus/readme.txt
......@@ -35,8 +35,7 @@ mvn install [-P dist] -DGRAPH-PROVIDER=janus
Some tests in the repository and webapp projects are skipped when running with the janus provider, due to hard
dependencies on Gremlin2. These components need to be updated. Please refer to "known issues" section below.
This will build Atlas and run all of the tests against Janus. Such a build MUST be used with JanusGraph and
CANNOT be used with any other graph provider, e.g. Titan 0.5.4 or Titan 1.0.0.
This will build Atlas and run all of the tests against Janus.
2) Configure the Atlas runtime to use JanusGraph by setting the atlas.graphdb.backend property in
ATLAS_HOME/conf/atlas-application.properties, as follows:
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
......@@ -50,7 +50,7 @@ import java.util.ArrayList;
import java.util.Map;
/**
* Default implementation for Graph Provider that doles out Titan Graph.
* Default implementation for Graph Provider that doles out JanusGraph.
*/
public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex, AtlasJanusEdge> {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphDatabase.class);
......
graphdb/pom.xml
......@@ -36,7 +36,6 @@
<module>api</module>
<module>common</module>
<module>graphdb-impls</module>
<module>titan0</module>
<module>janus</module>
</modules>
......
graphdb/readme.txt
......@@ -9,17 +9,12 @@ If GRAPH-PROVIDER is not set, the default graph backend is adopted. This is curr
In order to build with a specific (non-default) graph backend set the GRAPH-PROVDER system variable.
If GRAPH-PROVIDER is set to titan0, the build will contain Titan 0.5.4
If GRAPH-PROVIDER is set to janus, the build will contain JanusGraph 0.2.0 (i.e. the default above)
For example, to build Atlas with the janus graph-provider:
mvn install [-P dist] -DGRAPH-PROVIDER=janus
Titan 0.5.4 supports Gremlin2 only, whereas JanusGraph support Gremlin3 only (and NOT Gremlin2).
Gremlin2 and Gremlin3 are not compatible. The gremlin used by Atlas is translated into either Gremlin2 or
Gremlin3 depending on which graph backend is used in the build. This is implemented in GremlinExpressionFactory.
JanusGraph support Gremlin3 only (and NOT Gremlin2).
REQUIREMENTS
------------
......
graphdb/titan0/pom.xml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>atlas-graphdb</artifactId>
<groupId>org.apache.atlas</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>atlas-graphdb-titan0</artifactId>
<description>Apache Atlas Titan 0.5.4 Graph DB Impl</description>
<name>Apache Atlas Titan 0.5.4 Graph DB Impl</name>
<packaging>jar</packaging>
<properties>
<tinkerpop.version>2.6.0</tinkerpop.version>
<titan.version>0.5.4</titan.version>
<checkstyle.failOnViolation>false</checkstyle.failOnViolation>
<guava.version>14.0</guava.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-api</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId>
<version>${titan.version}</version>
<exclusions>
<!-- rexster does not work with servlet-api -->
<exclusion>
<groupId>com.tinkerpop.rexster</groupId>
<artifactId>rexster-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.tinkerpop.rexster</groupId>
<artifactId>rexster-server</artifactId>
</exclusion>
<!-- asm 4.0 does not work with jersey asm 3.1 -->
<exclusion>
<groupId>com.tinkerpop</groupId>
<artifactId>frames</artifactId>
</exclusion>
<exclusion>
<groupId>com.esotericsoftware.reflectasm</groupId>
<artifactId>reflectasm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion> <!-- GPL license imported from ganglia -->
<groupId>org.acplt</groupId>
<artifactId>oncrpc</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.tinkerpop.blueprints</groupId>
<artifactId>blueprints-core</artifactId>
<version>${tinkerpop.version}</version>
</dependency>
<dependency>
<groupId>com.tinkerpop.gremlin</groupId>
<artifactId>gremlin-java</artifactId>
<version>${tinkerpop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-hbase-client-shaded</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-es</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-hbase</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-solr</artifactId>
<version>${titan.version}</version>
<!-- this conflicts with the servlet api version that Atlas needs -->
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-common</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
<version>${solr.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.eclipse.jetty.orbit</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.restlet.jee</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.ow2.asm</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<!--
Create 'uber' jar that contains all of the dependencies (except those whose scope is provided)
Only Titan 0.5.4 and its dependencies are included. The other dependencies are bundled in the war file.
-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<excludes>
<!-- these are bundled with Atlas -->
<exclude>org.slf4j:*</exclude>
<exclude>com.fasterxml.jackson.core:*</exclude>
<exclude>com.fasterxml.jackson.dataformat:*</exclude>
<exclude>com.fasterxml.jackson.databind:*</exclude>
<exclude>com.fasterxml.jackson.datatype:*</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>com.thinkaurelius.titan:titan-core</artifact>
<!-- force use of our custom titan class implementations -->
<excludes>
<exclude>com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.class</exclude>
<exclude>com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.class</exclude>
</excludes>
</filter>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>org/codehaus/jackson/**</exclude>
<exclude>com/fasterxml/jackson/**</exclude>
</excludes>
</filter>
</filters>
<relocations>
<!-- guava has incompatibilities across its versions. Titan requires different version of guava than the version that atlas needs.
So, shading the guava reference in titan -->
<relocation>
<pattern>com.google</pattern>
<shadedPattern>atlas.shaded.titan.guava</shadedPattern>
</relocation>
</relocations>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<configuration>
<excludes>
<exclude>**/log4j.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.version}</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.Closeable;
import java.io.IOException;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
/**
* This interface hides ABI/API breaking changes that HBase has made to its Admin/HBaseAdmin over the course
* of development from 0.94 to 1.0 and beyond.
*/
public interface AdminMask extends Closeable
{
void clearTable(String tableName, long timestamp) throws IOException;
HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException;
boolean tableExists(String tableName) throws IOException;
void createTable(HTableDescriptor desc) throws IOException;
void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException;
/**
* Estimate the number of regionservers in the HBase cluster.
*
* This is usually implemented by calling
* {@link HBaseAdmin#getClusterStatus()} and then
* {@link ClusterStatus#getServers()} and finally {@code size()} on the
* returned server list.
*
* @return the number of servers in the cluster or -1 if it could not be determined
*/
int getEstimatedRegionServerCount();
void disableTable(String tableName) throws IOException;
void enableTable(String tableName) throws IOException;
boolean isTableDisabled(String tableName) throws IOException;
void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException;
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.Closeable;
import java.io.IOException;
/**
* This interface hides ABI/API breaking changes that HBase has made to its (H)Connection class over the course
* of development from 0.94 to 1.0 and beyond.
*/
public interface ConnectionMask extends Closeable
{
TableMask getTable(String name) throws IOException;
AdminMask getAdmin() throws IOException;
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.thinkaurelius.titan.util.system.IOUtils;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
public class HBaseAdmin0_98 implements AdminMask
{
private static final Logger log = LoggerFactory.getLogger(HBaseAdmin0_98.class);
private final HBaseAdmin adm;
public HBaseAdmin0_98(HBaseAdmin adm)
{
this.adm = adm;
}
@Override
public void clearTable(String tableName, long timestamp) throws IOException
{
if (!adm.tableExists(tableName)) {
log.debug("clearStorage() called before table {} was created, skipping.", tableName);
return;
}
// Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than
// disabling and deleting tables.
HTable table = null;
try {
table = new HTable(adm.getConfiguration(), tableName);
Scan scan = new Scan();
scan.setBatch(100);
scan.setCacheBlocks(false);
scan.setCaching(2000);
scan.setTimeRange(0, Long.MAX_VALUE);
scan.setMaxVersions(1);
ResultScanner scanner = null;
try {
scanner = table.getScanner(scan);
for (Result res : scanner) {
Delete d = new Delete(res.getRow());
d.setTimestamp(timestamp);
table.delete(d);
}
} finally {
IOUtils.closeQuietly(scanner);
}
} finally {
IOUtils.closeQuietly(table);
}
}
@Override
public HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException
{
return adm.getTableDescriptor(tableName.getBytes());
}
@Override
public boolean tableExists(String tableName) throws IOException
{
return adm.tableExists(tableName);
}
@Override
public void createTable(HTableDescriptor desc) throws IOException
{
adm.createTable(desc);
}
@Override
public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException
{
adm.createTable(desc, startKey, endKey, numRegions);
}
@Override
public int getEstimatedRegionServerCount()
{
int serverCount = -1;
try {
serverCount = adm.getClusterStatus().getServers().size();
log.debug("Read {} servers from HBase ClusterStatus", serverCount);
} catch (IOException e) {
log.debug("Unable to retrieve HBase cluster status", e);
}
return serverCount;
}
@Override
public void disableTable(String tableName) throws IOException
{
adm.disableTable(tableName);
}
@Override
public void enableTable(String tableName) throws IOException
{
adm.enableTable(tableName);
}
@Override
public boolean isTableDisabled(String tableName) throws IOException
{
return adm.isTableDisabled(tableName);
}
@Override
public void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException
{
adm.addColumn(tableName, columnDescriptor);
}
@Override
public void close() throws IOException
{
adm.close();
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.HBaseAdmin;
public class HBaseAdmin1_0 implements AdminMask
{
private static final Logger log = LoggerFactory.getLogger(HBaseAdmin1_0.class);
private final Admin adm;
public HBaseAdmin1_0(HBaseAdmin adm)
{
this.adm = adm;
}
@Override
public void clearTable(String tableString, long timestamp) throws IOException
{
TableName tableName = TableName.valueOf(tableString);
if (!adm.tableExists(tableName)) {
log.debug("Attempted to clear table {} before it exists (noop)", tableString);
return;
}
if (!adm.isTableDisabled(tableName))
adm.disableTable(tableName);
if (!adm.isTableDisabled(tableName))
throw new RuntimeException("Unable to disable table " + tableName);
// This API call appears to both truncate and reenable the table.
log.info("Truncating table {}", tableName);
adm.truncateTable(tableName, true /* preserve splits */);
try {
adm.enableTable(tableName);
} catch (TableNotDisabledException e) {
// This triggers seemingly every time in testing with 1.0.2.
log.debug("Table automatically reenabled by truncation: {}", tableName, e);
}
}
@Override
public HTableDescriptor getTableDescriptor(String tableString) throws TableNotFoundException, IOException
{
return adm.getTableDescriptor(TableName.valueOf(tableString));
}
@Override
public boolean tableExists(String tableString) throws IOException
{
return adm.tableExists(TableName.valueOf(tableString));
}
@Override
public void createTable(HTableDescriptor desc) throws IOException
{
adm.createTable(desc);
}
@Override
public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException
{
adm.createTable(desc, startKey, endKey, numRegions);
}
@Override
public int getEstimatedRegionServerCount()
{
int serverCount = -1;
try {
serverCount = adm.getClusterStatus().getServers().size();
log.debug("Read {} servers from HBase ClusterStatus", serverCount);
} catch (IOException e) {
log.debug("Unable to retrieve HBase cluster status", e);
}
return serverCount;
}
@Override
public void disableTable(String tableString) throws IOException
{
adm.disableTable(TableName.valueOf(tableString));
}
@Override
public void enableTable(String tableString) throws IOException
{
adm.enableTable(TableName.valueOf(tableString));
}
@Override
public boolean isTableDisabled(String tableString) throws IOException
{
return adm.isTableDisabled(TableName.valueOf(tableString));
}
@Override
public void addColumn(String tableString, HColumnDescriptor columnDescriptor) throws IOException
{
adm.addColumn(TableName.valueOf(tableString), columnDescriptor);
}
@Override
public void close() throws IOException
{
adm.close();
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.Delete;
public interface HBaseCompat {
/**
* Configure the compression scheme {@code algo} on a column family
* descriptor {@code cd}. The {@code algo} parameter is a string value
* corresponding to one of the values of HBase's Compression enum. The
* Compression enum has moved between packages as HBase has evolved, which
* is why this method has a String argument in the signature instead of the
* enum itself.
*
* @param cd
* column family to configure
* @param algo
* compression type to use
*/
void setCompression(HColumnDescriptor cd, String algo);
/**
* Create and return a HTableDescriptor instance with the given name. The
* constructors on this method have remained stable over HBase development
* so far, but the old HTableDescriptor(String) constructor & byte[] friends
* are now marked deprecated and may eventually be removed in favor of the
* HTableDescriptor(TableName) constructor. That constructor (and the
* TableName type) only exists in newer HBase versions. Hence this method.
*
* @param tableName
* HBase table name
* @return a new table descriptor instance
*/
HTableDescriptor newTableDescriptor(String tableName);
ConnectionMask createConnection(Configuration conf) throws IOException;
void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc);
void setTimestamp(Delete d, long timestamp);
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.io.compress.Compression;
public class HBaseCompat0_98 implements HBaseCompat {
@Override
public void setCompression(HColumnDescriptor cd, String algo) {
cd.setCompressionType(Compression.Algorithm.valueOf(algo));
}
@Override
public HTableDescriptor newTableDescriptor(String tableName) {
TableName tn = TableName.valueOf(tableName);
return new HTableDescriptor(tn);
}
@Override
public ConnectionMask createConnection(Configuration conf) throws IOException
{
return new HConnection0_98(HConnectionManager.createConnection(conf));
}
@Override
public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
{
tdesc.addFamily(cdesc);
}
@Override
public void setTimestamp(Delete d, long timestamp)
{
d.setTimestamp(timestamp);
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.io.compress.Compression;
public class HBaseCompat1_0 implements HBaseCompat {
@Override
public void setCompression(HColumnDescriptor cd, String algo) {
cd.setCompressionType(Compression.Algorithm.valueOf(algo));
}
@Override
public HTableDescriptor newTableDescriptor(String tableName) {
TableName tn = TableName.valueOf(tableName);
return new HTableDescriptor(tn);
}
@Override
public ConnectionMask createConnection(Configuration conf) throws IOException
{
return new HConnection1_0(ConnectionFactory.createConnection(conf));
}
@Override
public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
{
tdesc.addFamily(cdesc);
}
@Override
public void setTimestamp(Delete d, long timestamp)
{
d.setTimestamp(timestamp);
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.io.compress.Compression;
import java.io.IOException;
public class HBaseCompat1_1 implements HBaseCompat {
@Override
public void setCompression(HColumnDescriptor cd, String algo) {
cd.setCompressionType(Compression.Algorithm.valueOf(algo));
}
@Override
public HTableDescriptor newTableDescriptor(String tableName) {
TableName tn = TableName.valueOf(tableName);
return new HTableDescriptor(tn);
}
@Override
public ConnectionMask createConnection(Configuration conf) throws IOException
{
return new HConnection1_0(ConnectionFactory.createConnection(conf));
}
@Override
public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
{
tdesc.addFamily(cdesc);
}
@Override
public void setTimestamp(Delete d, long timestamp)
{
d.setTimestamp(timestamp);
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.util.Arrays;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HBaseCompatLoader {
private static final Logger log = LoggerFactory.getLogger(HBaseCompatLoader.class);
private static final String DEFAULT_HBASE_COMPAT_VERSION = "1.1";
private static final String DEFAULT_HBASE_CLASS_NAME = "com.thinkaurelius.titan.diskstorage.hbase.HBaseCompat1_1";
private static HBaseCompat cachedCompat;
public synchronized static HBaseCompat getCompat(String classOverride) {
if (null != cachedCompat) {
log.debug("Returning cached HBase compatibility layer: {}", cachedCompat);
return cachedCompat;
}
HBaseCompat compat;
String className = null;
String classNameSource = null;
if (null != classOverride) {
className = classOverride;
classNameSource = "from explicit configuration";
} else {
String hbaseVersion = VersionInfo.getVersion();
for (String supportedVersion : Arrays.asList("0.94", "0.96", "0.98", "1.0", "1.1")) {
if (hbaseVersion.startsWith(supportedVersion + ".")) {
className = "com.thinkaurelius.titan.diskstorage.hbase.HBaseCompat" + supportedVersion.replaceAll("\\.", "_");
classNameSource = "supporting runtime HBase version " + hbaseVersion;
break;
}
}
if (null == className) {
log.info("The HBase version {} is not explicitly supported by Titan. " +
"Loading Titan's compatibility layer for its most recent supported HBase version ({})",
hbaseVersion, DEFAULT_HBASE_COMPAT_VERSION);
className = DEFAULT_HBASE_CLASS_NAME;
classNameSource = " by default";
}
}
final String errTemplate = " when instantiating HBase compatibility class " + className;
try {
compat = (HBaseCompat)Class.forName(className).newInstance();
log.info("Instantiated HBase compatibility layer {}: {}", classNameSource, compat.getClass().getCanonicalName());
} catch (IllegalAccessException | ClassNotFoundException | InstantiationException e) {
throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
}
return cachedCompat = compat;
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.thinkaurelius.titan.core.attribute.Duration;
import com.thinkaurelius.titan.diskstorage.*;
import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*;
import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
import com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException;
import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
import com.thinkaurelius.titan.diskstorage.util.RecordIterator;
import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
import com.thinkaurelius.titan.diskstorage.util.StaticArrayEntry;
import com.thinkaurelius.titan.diskstorage.util.StaticArrayEntryList;
import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.thinkaurelius.titan.util.system.IOUtils;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.Closeable;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
/**
* Here are some areas that might need work:
* <p/>
* - batching? (consider HTable#batch, HTable#setAutoFlush(false)
* - tuning HTable#setWriteBufferSize (?)
* - writing a server-side filter to replace ColumnCountGetFilter, which drops
* all columns on the row where it reaches its limit. This requires getSlice,
* currently, to impose its limit on the client side. That obviously won't
* scale.
* - RowMutations for combining Puts+Deletes (need a newer HBase than 0.92 for this)
* - (maybe) fiddle with HTable#setRegionCachePrefetch and/or #prewarmRegionCache
* <p/>
* There may be other problem areas. These are just the ones of which I'm aware.
*/
public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
private static final Logger logger = LoggerFactory.getLogger(HBaseKeyColumnValueStore.class);
private final String tableName;
private final HBaseStoreManager storeManager;
// When using shortened CF names, columnFamily is the shortname and storeName is the longname
// When not using shortened CF names, they are the same
//private final String columnFamily;
private final String storeName;
// This is columnFamily.getBytes()
private final byte[] columnFamilyBytes;
private final HBaseGetter entryGetter;
private final ConnectionMask cnx;
private LocalLockMediator<StoreTransaction> localLockMediator;
private final Duration lockExpiryTimeMs;
private final Duration lockMaxWaitTimeMs;
private final Integer lockMaxRetries;
HBaseKeyColumnValueStore(HBaseStoreManager storeManager, ConnectionMask cnx, String tableName, String columnFamily, String storeName, LocalLockMediator<StoreTransaction> llm) {
this.storeManager = storeManager;
this.cnx = cnx;
this.tableName = tableName;
//this.columnFamily = columnFamily;
this.storeName = storeName;
this.columnFamilyBytes = columnFamily.getBytes();
this.entryGetter = new HBaseGetter(storeManager.getMetaDataSchema(storeName));
this.localLockMediator = llm;
Configuration storageConfig = storeManager.getStorageConfig();
this.lockExpiryTimeMs = storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE);
this.lockMaxWaitTimeMs = storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT);
this.lockMaxRetries = storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY);
}
@Override
public void close() throws BackendException {
}
@Override
public EntryList getSlice(KeySliceQuery query, StoreTransaction txh) throws BackendException {
Map<StaticBuffer, EntryList> result = getHelper(Arrays.asList(query.getKey()), getFilter(query));
return Iterables.getOnlyElement(result.values(), EntryList.EMPTY_LIST);
}
@Override
public Map<StaticBuffer,EntryList> getSlice(List<StaticBuffer> keys, SliceQuery query, StoreTransaction txh) throws BackendException {
return getHelper(keys, getFilter(query));
}
@Override
public void mutate(StaticBuffer key, List<Entry> additions, List<StaticBuffer> deletions, StoreTransaction txh) throws BackendException {
Map<StaticBuffer, KCVMutation> mutations = ImmutableMap.of(key, new KCVMutation(additions, deletions));
mutateMany(mutations, txh);
}
@Override
public void acquireLock(StaticBuffer key,
StaticBuffer column,
StaticBuffer expectedValue,
StoreTransaction txh) throws BackendException {
KeyColumn lockID = new KeyColumn(key, column);
logger.debug("Attempting to acquireLock on {} ", lockID);
int trialCount = 0;
boolean locked;
while (trialCount < lockMaxRetries) {
final Timepoint lockStartTime = Timestamps.MILLI.getTime(System.currentTimeMillis(), TimeUnit.MILLISECONDS);
locked = localLockMediator.lock(lockID, txh, lockStartTime.add(lockExpiryTimeMs));
trialCount++;
if (!locked) {
handleLockFailure(txh, lockID, trialCount);
} else {
logger.debug("Acquired lock on {}, {}", lockID, txh);
break;
}
}
((HBaseTransaction) txh).updateLocks(lockID, expectedValue);
}
void handleLockFailure(StoreTransaction txh, KeyColumn lockID, int trialCount) throws PermanentLockingException {
if (trialCount < lockMaxRetries) {
try {
Thread.sleep(lockMaxWaitTimeMs.getLength(TimeUnit.MILLISECONDS));
} catch (InterruptedException e) {
throw new PermanentLockingException(
"Interrupted while waiting for acquiring lock for transaction "
+ txh + " lockID " + lockID + " on retry " + trialCount, e);
}
} else {
throw new PermanentLockingException("Could not lock the keyColumn " +
lockID + " on CF {} " + Bytes.toString(columnFamilyBytes));
}
}
@Override
public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) throws BackendException {
return executeKeySliceQuery(query.getKeyStart().as(StaticBuffer.ARRAY_FACTORY),
query.getKeyEnd().as(StaticBuffer.ARRAY_FACTORY),
new FilterList(FilterList.Operator.MUST_PASS_ALL),
query);
}
@Override
public String getName() {
return storeName;
}
@Override
public KeyIterator getKeys(SliceQuery query, StoreTransaction txh) throws BackendException {
return executeKeySliceQuery(new FilterList(FilterList.Operator.MUST_PASS_ALL), query);
}
public static Filter getFilter(SliceQuery query) {
byte[] colStartBytes = query.getSliceEnd().length() > 0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null;
byte[] colEndBytes = query.getSliceEnd().length() > 0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null;
Filter filter = new ColumnRangeFilter(colStartBytes, true, colEndBytes, false);
if (query.hasLimit()) {
filter = new FilterList(FilterList.Operator.MUST_PASS_ALL,
filter,
new ColumnPaginationFilter(query.getLimit(), 0));
}
logger.debug("Generated HBase Filter {}", filter);
return filter;
}
private Map<StaticBuffer,EntryList> getHelper(List<StaticBuffer> keys, Filter getFilter) throws BackendException {
List<Get> requests = new ArrayList<>(keys.size());
{
for (StaticBuffer key : keys) {
Get g = new Get(key.as(StaticBuffer.ARRAY_FACTORY)).addFamily(columnFamilyBytes).setFilter(getFilter);
try {
g.setTimeRange(0, Long.MAX_VALUE);
} catch (IOException e) {
throw new PermanentBackendException(e);
}
requests.add(g);
}
}
Map<StaticBuffer,EntryList> resultMap = new HashMap<>(keys.size());
try {
TableMask table = null;
Result[] results = null;
try {
table = cnx.getTable(tableName);
logger.debug("Get requests {} {} ", Bytes.toString(columnFamilyBytes), requests.size());
results = table.get(requests);
logger.debug("Get requests finished {} {} ", Bytes.toString(columnFamilyBytes), requests.size());
} finally {
IOUtils.closeQuietly(table);
}
if (results == null)
return KCVSUtil.emptyResults(keys);
assert results.length==keys.size();
for (int i = 0; i < results.length; i++) {
Result result = results[i];
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> f = result.getMap();
if (f == null) { // no result for this key
resultMap.put(keys.get(i), EntryList.EMPTY_LIST);
continue;
}
// actual key with <timestamp, value>
NavigableMap<byte[], NavigableMap<Long, byte[]>> r = f.get(columnFamilyBytes);
resultMap.put(keys.get(i), (r == null)
? EntryList.EMPTY_LIST
: StaticArrayEntryList.ofBytes(r.entrySet(), entryGetter));
}
return resultMap;
} catch (IOException e) {
throw new TemporaryBackendException(e);
}
}
private void mutateMany(Map<StaticBuffer, KCVMutation> mutations, StoreTransaction txh) throws BackendException {
storeManager.mutateMany(ImmutableMap.of(storeName, mutations), txh);
}
private KeyIterator executeKeySliceQuery(FilterList filters, @Nullable SliceQuery columnSlice) throws BackendException {
return executeKeySliceQuery(null, null, filters, columnSlice);
}
private KeyIterator executeKeySliceQuery(@Nullable byte[] startKey,
@Nullable byte[] endKey,
FilterList filters,
@Nullable SliceQuery columnSlice) throws BackendException {
Scan scan = new Scan().addFamily(columnFamilyBytes);
try {
scan.setTimeRange(0, Long.MAX_VALUE);
} catch (IOException e) {
throw new PermanentBackendException(e);
}
if (startKey != null)
scan.setStartRow(startKey);
if (endKey != null)
scan.setStopRow(endKey);
if (columnSlice != null) {
filters.addFilter(getFilter(columnSlice));
}
TableMask table = null;
logger.debug("Scan for row keys {} {} ", Bytes.toString(startKey), Bytes.toString(endKey));
try {
table = cnx.getTable(tableName);
return new RowIterator(table, table.getScanner(scan.setFilter(filters)), columnFamilyBytes);
} catch (IOException e) {
IOUtils.closeQuietly(table);
throw new PermanentBackendException(e);
}
}
private class RowIterator implements KeyIterator {
private final Closeable table;
private final Iterator<Result> rows;
private final byte[] columnFamilyBytes;
private Result currentRow;
private boolean isClosed;
public RowIterator(Closeable table, ResultScanner rows, byte[] columnFamilyBytes) {
this.table = table;
this.columnFamilyBytes = Arrays.copyOf(columnFamilyBytes, columnFamilyBytes.length);
this.rows = Iterators.filter(rows.iterator(), new Predicate<Result>() {
@Override
public boolean apply(@Nullable Result result) {
if (result == null)
return false;
try {
StaticBuffer id = StaticArrayBuffer.of(result.getRow());
id.getLong(0);
} catch (NumberFormatException e) {
return false;
}
return true;
}
});
}
@Override
public RecordIterator<Entry> getEntries() {
ensureOpen();
return new RecordIterator<Entry>() {
private final NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> currentMap = currentRow.getMap();
private final Iterator<Map.Entry<byte[], NavigableMap<Long, byte[]>>> kv = currentMap == null ? null : currentMap.get(columnFamilyBytes).entrySet().iterator();
@Override
public boolean hasNext() {
ensureOpen();
return kv != null && kv.hasNext();
}
@Override
public Entry next() {
ensureOpen();
return kv == null ? null : StaticArrayEntry.ofBytes(kv.next(), entryGetter);
}
@Override
public void close() {
isClosed = true;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
@Override
public boolean hasNext() {
ensureOpen();
return rows.hasNext();
}
@Override
public StaticBuffer next() {
ensureOpen();
currentRow = rows.next();
return StaticArrayBuffer.of(currentRow.getRow());
}
@Override
public void close() {
IOUtils.closeQuietly(table);
isClosed = true;
logger.debug("RowIterator closed table {}", table);
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
private void ensureOpen() {
if (isClosed)
throw new IllegalStateException("Iterator has been closed.");
}
}
private static class HBaseGetter implements StaticArrayEntry.GetColVal<Map.Entry<byte[], NavigableMap<Long, byte[]>>, byte[]> {
private final EntryMetaData[] schema;
private HBaseGetter(EntryMetaData[] schema) {
this.schema = schema;
}
@Override
public byte[] getColumn(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
return element.getKey();
}
@Override
public byte[] getValue(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
return element.getValue().lastEntry().getValue();
}
@Override
public EntryMetaData[] getMetaSchema(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
return schema;
}
@Override
public Object getMetaData(Map.Entry<byte[], NavigableMap<Long, byte[]>> element, EntryMetaData meta) {
switch(meta) {
case TIMESTAMP:
return element.getValue().lastEntry().getKey();
default:
throw new UnsupportedOperationException("Unsupported meta data: " + meta);
}
}
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import com.thinkaurelius.titan.diskstorage.Backend;
import com.thinkaurelius.titan.diskstorage.configuration.ConfigElement;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.CustomizeStoreKCVSManager;
import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediators;
import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Row;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.thinkaurelius.titan.core.TitanException;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
import com.thinkaurelius.titan.diskstorage.Entry;
import com.thinkaurelius.titan.diskstorage.PermanentBackendException;
import com.thinkaurelius.titan.diskstorage.StaticBuffer;
import com.thinkaurelius.titan.diskstorage.TemporaryBackendException;
import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager;
import com.thinkaurelius.titan.diskstorage.configuration.ConfigNamespace;
import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption;
import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KCVMutation;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StandardStoreFeatures;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreFeatures;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction;
import com.thinkaurelius.titan.diskstorage.util.BufferUtil;
import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.thinkaurelius.titan.graphdb.configuration.PreInitializeConfigOptions;
import com.thinkaurelius.titan.util.system.IOUtils;
import com.thinkaurelius.titan.util.system.NetworkUtil;
/**
* Storage Manager for HBase
*
* @author Dan LaRocque <dalaro@hopcount.org>
*/
@PreInitializeConfigOptions
public class HBaseStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager, CustomizeStoreKCVSManager {
private static final Logger logger = LoggerFactory.getLogger(HBaseStoreManager.class);
public static final ConfigNamespace HBASE_NS =
new ConfigNamespace(GraphDatabaseConfiguration.STORAGE_NS, "hbase", "HBase storage options");
public static final ConfigOption<Boolean> SHORT_CF_NAMES =
new ConfigOption<>(HBASE_NS, "short-cf-names",
"Whether to shorten the names of Titan's column families to one-character mnemonics " +
"to conserve storage space", ConfigOption.Type.FIXED, true);
public static final String COMPRESSION_DEFAULT = "-DEFAULT-";
public static final ConfigOption<String> COMPRESSION =
new ConfigOption<>(HBASE_NS, "compression-algorithm",
"An HBase Compression.Algorithm enum string which will be applied to newly created column families. " +
"The compression algorithm must be installed and available on the HBase cluster. Titan cannot install " +
"and configure new compression algorithms on the HBase cluster by itself.",
ConfigOption.Type.MASKABLE, "GZ");
public static final ConfigOption<Boolean> SKIP_SCHEMA_CHECK =
new ConfigOption<>(HBASE_NS, "skip-schema-check",
"Assume that Titan's HBase table and column families already exist. " +
"When this is true, Titan will not check for the existence of its table/CFs, " +
"nor will it attempt to create them under any circumstances. This is useful " +
"when running Titan without HBase admin privileges.",
ConfigOption.Type.MASKABLE, false);
public static final ConfigOption<String> HBASE_TABLE =
new ConfigOption<>(HBASE_NS, "table",
"The name of the table Titan will use. When " + ConfigElement.getPath(SKIP_SCHEMA_CHECK) +
" is false, Titan will automatically create this table if it does not already exist.",
ConfigOption.Type.LOCAL, "titan");
/**
* Related bug fixed in 0.98.0, 0.94.7, 0.95.0:
*
* https://issues.apache.org/jira/browse/HBASE-8170
*/
public static final int MIN_REGION_COUNT = 3;
/**
* The total number of HBase regions to create with Titan's table. This
* setting only effects table creation; this normally happens just once when
* Titan connects to an HBase backend for the first time.
*/
public static final ConfigOption<Integer> REGION_COUNT =
new ConfigOption<>(HBASE_NS, "region-count",
"The number of initial regions set when creating Titan's HBase table",
ConfigOption.Type.MASKABLE, Integer.class, new Predicate<Integer>() {
@Override
public boolean apply(Integer input) {
return null != input && MIN_REGION_COUNT <= input;
}
}
);
/**
* This setting is used only when {@link #REGION_COUNT} is unset.
* <p/>
* If Titan's HBase table does not exist, then it will be created with total
* region count = (number of servers reported by ClusterStatus) * (this
* value).
* <p/>
* The Apache HBase manual suggests an order-of-magnitude range of potential
* values for this setting:
*
* <ul>
* <li>
* <a href="https://hbase.apache.org/book/important_configurations.html#disable.splitting">2.5.2.7. Managed Splitting</a>:
* <blockquote>
* What's the optimal number of pre-split regions to create? Mileage will
* vary depending upon your application. You could start low with 10
* pre-split regions / server and watch as data grows over time. It's
* better to err on the side of too little regions and rolling split later.
* </blockquote>
* </li>
* <li>
* <a href="https://hbase.apache.org/book/regions.arch.html">9.7 Regions</a>:
* <blockquote>
* In general, HBase is designed to run with a small (20-200) number of
* relatively large (5-20Gb) regions per server... Typically you want to
* keep your region count low on HBase for numerous reasons. Usually
* right around 100 regions per RegionServer has yielded the best results.
* </blockquote>
* </li>
* </ul>
*
* These considerations may differ for other HBase implementations (e.g. MapR).
*/
public static final ConfigOption<Integer> REGIONS_PER_SERVER =
new ConfigOption<>(HBASE_NS, "regions-per-server",
"The number of regions per regionserver to set when creating Titan's HBase table",
ConfigOption.Type.MASKABLE, Integer.class);
/**
* If this key is present in either the JVM system properties or the process
* environment (checked in the listed order, first hit wins), then its value
* must be the full package and class name of an implementation of
* {@link HBaseCompat} that has a no-arg public constructor.
* <p>
* When this <b>is not</b> set, Titan attempts to automatically detect the
* HBase runtime version by calling {@link VersionInfo#getVersion()}. Titan
* then checks the returned version string against a hard-coded list of
* supported version prefixes and instantiates the associated compat layer
* if a match is found.
* <p>
* When this <b>is</b> set, Titan will not call
* {@code VersionInfo.getVersion()} or read its hard-coded list of supported
* version prefixes. Titan will instead attempt to instantiate the class
* specified (via the no-arg constructor which must exist) and then attempt
* to cast it to HBaseCompat and use it as such. Titan will assume the
* supplied implementation is compatible with the runtime HBase version and
* make no attempt to verify that assumption.
* <p>
* Setting this key incorrectly could cause runtime exceptions at best or
* silent data corruption at worst. This setting is intended for users
* running exotic HBase implementations that don't support VersionInfo or
* implementations which return values from {@code VersionInfo.getVersion()}
* that are inconsistent with Apache's versioning convention. It may also be
* useful to users who want to run against a new release of HBase that Titan
* doesn't yet officially support.
*
*/
public static final ConfigOption<String> COMPAT_CLASS =
new ConfigOption<>(HBASE_NS, "compat-class",
"The package and class name of the HBaseCompat implementation. HBaseCompat masks version-specific HBase API differences. " +
"When this option is unset, Titan calls HBase's VersionInfo.getVersion() and loads the matching compat class " +
"at runtime. Setting this option forces Titan to instead reflectively load and instantiate the specified class.",
ConfigOption.Type.MASKABLE, String.class);
public static final int PORT_DEFAULT = 9160;
public static final Timestamps PREFERRED_TIMESTAMPS = Timestamps.MILLI;
public static final ConfigNamespace HBASE_CONFIGURATION_NAMESPACE =
new ConfigNamespace(HBASE_NS, "ext", "Overrides for hbase-{site,default}.xml options", true);
private static final BiMap<String, String> SHORT_CF_NAME_MAP =
ImmutableBiMap.<String, String>builder()
.put(Backend.INDEXSTORE_NAME, "g")
.put(Backend.INDEXSTORE_NAME + Backend.LOCK_STORE_SUFFIX, "h")
.put(Backend.ID_STORE_NAME, "i")
.put(Backend.EDGESTORE_NAME, "e")
.put(Backend.EDGESTORE_NAME + Backend.LOCK_STORE_SUFFIX, "f")
.put(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME, "s")
.put(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME + Backend.LOCK_STORE_SUFFIX, "t")
.put(Backend.SYSTEM_MGMT_LOG_NAME, "m")
.put(Backend.SYSTEM_TX_LOG_NAME, "l")
.build();
private static final StaticBuffer FOUR_ZERO_BYTES = BufferUtil.zeroBuffer(4);
static {
// Verify that shortCfNameMap is injective
// Should be guaranteed by Guava BiMap, but it doesn't hurt to check
Preconditions.checkArgument(null != SHORT_CF_NAME_MAP);
Collection<String> shorts = SHORT_CF_NAME_MAP.values();
Preconditions.checkArgument(Sets.newHashSet(shorts).size() == shorts.size());
}
// Immutable instance fields
private final String tableName;
private final String compression;
private final int regionCount;
private final int regionsPerServer;
private final ConnectionMask cnx;
private final org.apache.hadoop.conf.Configuration hconf;
private final boolean shortCfNames;
private final boolean skipSchemaCheck;
private final String compatClass;
private final HBaseCompat compat;
private static final ConcurrentHashMap<HBaseStoreManager, Throwable> openManagers =
new ConcurrentHashMap<>();
// Mutable instance state
private final ConcurrentMap<String, HBaseKeyColumnValueStore> openStores;
private LocalLockMediator<StoreTransaction> llm;
public HBaseStoreManager(com.thinkaurelius.titan.diskstorage.configuration.Configuration config) throws BackendException {
super(config, PORT_DEFAULT);
checkConfigDeprecation(config);
this.tableName = config.get(HBASE_TABLE);
this.compression = config.get(COMPRESSION);
this.regionCount = config.has(REGION_COUNT) ? config.get(REGION_COUNT) : -1;
this.regionsPerServer = config.has(REGIONS_PER_SERVER) ? config.get(REGIONS_PER_SERVER) : -1;
this.skipSchemaCheck = config.get(SKIP_SCHEMA_CHECK);
this.compatClass = config.has(COMPAT_CLASS) ? config.get(COMPAT_CLASS) : null;
this.compat = HBaseCompatLoader.getCompat(compatClass);
/*
* Specifying both region count options is permitted but may be
* indicative of a misunderstanding, so issue a warning.
*/
if (config.has(REGIONS_PER_SERVER) && config.has(REGION_COUNT)) {
logger.warn("Both {} and {} are set in Titan's configuration, but "
+ "the former takes precedence and the latter will be ignored.",
REGION_COUNT, REGIONS_PER_SERVER);
}
/* This static factory calls HBaseConfiguration.addHbaseResources(),
* which in turn applies the contents of hbase-default.xml and then
* applies the contents of hbase-site.xml.
*/
this.hconf = HBaseConfiguration.create();
// Copy a subset of our commons config into a Hadoop config
int keysLoaded=0;
Map<String,Object> configSub = config.getSubset(HBASE_CONFIGURATION_NAMESPACE);
for (Map.Entry<String,Object> entry : configSub.entrySet()) {
logger.info("HBase configuration: setting {}={}", entry.getKey(), entry.getValue());
if (entry.getValue()==null) continue;
hconf.set(entry.getKey(), entry.getValue().toString());
keysLoaded++;
}
// Special case for STORAGE_HOSTS
if (config.has(GraphDatabaseConfiguration.STORAGE_HOSTS)) {
String zkQuorumKey = "hbase.zookeeper.quorum";
String csHostList = Joiner.on(",").join(config.get(GraphDatabaseConfiguration.STORAGE_HOSTS));
hconf.set(zkQuorumKey, csHostList);
logger.info("Copied host list from {} to {}: {}", GraphDatabaseConfiguration.STORAGE_HOSTS, zkQuorumKey, csHostList);
}
logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded);
this.shortCfNames = config.get(SHORT_CF_NAMES);
try {
//this.cnx = HConnectionManager.createConnection(hconf);
this.cnx = compat.createConnection(hconf);
} catch (IOException e) {
throw new PermanentBackendException(e);
}
if (logger.isTraceEnabled()) {
openManagers.put(this, new Throwable("Manager Opened"));
dumpOpenManagers();
}
logger.debug("Dumping HBase config key=value pairs");
for (Map.Entry<String, String> entry : hconf) {
logger.debug("[HBaseConfig] " + entry.getKey() + "=" + entry.getValue());
}
logger.debug("End of HBase config key=value pairs");
openStores = new ConcurrentHashMap<>();
}
@Override
public Deployment getDeployment() {
return Deployment.REMOTE;
/* If just one of the regions for titan table is in the localhost,
* this method returns Deployment.LOCAL - which does not sound right.
*
List<KeyRange> local;
try {
local = getLocalKeyPartition();
return null != local && !local.isEmpty() ? Deployment.LOCAL : Deployment.REMOTE;
} catch (BackendException e) {
// propagating StorageException might be a better approach
throw new RuntimeException(e);
}
*
*/
}
@Override
public String toString() {
return "hbase[" + tableName + "@" + super.toString() + "]";
}
public void dumpOpenManagers() {
int estimatedSize = openManagers.size();
logger.trace("---- Begin open HBase store manager list ({} managers) ----", estimatedSize);
for (HBaseStoreManager m : openManagers.keySet()) {
logger.trace("Manager {} opened at:", m, openManagers.get(m));
}
logger.trace("---- End open HBase store manager list ({} managers) ----", estimatedSize);
}
@Override
public void close() {
openStores.clear();
if (logger.isTraceEnabled())
openManagers.remove(this);
IOUtils.closeQuietly(cnx);
}
@Override
public StoreFeatures getFeatures() {
Configuration c = GraphDatabaseConfiguration.buildConfiguration();
StandardStoreFeatures.Builder fb = new StandardStoreFeatures.Builder()
.orderedScan(true).unorderedScan(true).batchMutation(true)
.multiQuery(true).distributed(true).keyOrdered(true).storeTTL(true)
.timestamps(true).preferredTimestamps(PREFERRED_TIMESTAMPS)
.locking(true)
.keyConsistent(c);
try {
fb.localKeyPartition(getDeployment() == Deployment.LOCAL);
} catch (Exception e) {
logger.warn("Unexpected exception during getDeployment()", e);
}
return fb.build();
}
@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
logger.debug("Enter mutateMany");
final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
// In case of an addition and deletion with identical timestamps, the
// deletion tombstone wins.
// http://hbase.apache.org/book/versions.html#d244e4250
Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey =
convertToCommands(
mutations,
commitTime.getAdditionTime(times.getUnit()),
commitTime.getDeletionTime(times.getUnit()));
List<Row> batch = new ArrayList<>(commandsPerKey.size()); // actual batch operation
// convert sorted commands into representation required for 'batch' operation
for (Pair<Put, Delete> commands : commandsPerKey.values()) {
if (commands.getFirst() != null)
batch.add(commands.getFirst());
if (commands.getSecond() != null)
batch.add(commands.getSecond());
}
try {
TableMask table = null;
try {
table = cnx.getTable(tableName);
logger.debug("mutateMany : batch mutate started size {} ", batch.size());
table.batch(batch, new Object[batch.size()]);
logger.debug("mutateMany : batch mutate finished {} ", batch.size());
} finally {
IOUtils.closeQuietly(table);
}
} catch (IOException | InterruptedException e) {
throw new TemporaryBackendException(e);
}
sleepAfterWrite(txh, commitTime);
}
@Override
public KeyColumnValueStore openDatabase(String longName) throws BackendException {
return openDatabase(longName, -1);
}
@Override
public KeyColumnValueStore openDatabase(final String longName, int ttlInSeconds) throws BackendException {
HBaseKeyColumnValueStore store = openStores.get(longName);
if (store == null) {
final String cfName = shortCfNames ? shortenCfName(longName) : longName;
final String llmPrefix = getName();
llm = LocalLockMediators.INSTANCE.get(llmPrefix, times);
HBaseKeyColumnValueStore newStore = new HBaseKeyColumnValueStore(this, cnx, tableName, cfName, longName, llm);
store = openStores.putIfAbsent(longName, newStore); // nothing bad happens if we loose to other thread
if (store == null) {
if (!skipSchemaCheck)
ensureColumnFamilyExists(tableName, cfName, ttlInSeconds);
store = newStore;
}
logger.info("Loaded 1.x Hbase Client Store Manager");
}
return store;
}
@Override
public StoreTransaction beginTransaction(final BaseTransactionConfig config) throws BackendException {
return new HBaseTransaction(config, llm);
}
@Override
public String getName() {
return tableName;
}
/**
* Deletes the specified table with all its columns.
* ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss.
*/
@Override
public void clearStorage() throws BackendException {
try (AdminMask adm = getAdminInterface()) {
adm.clearTable(tableName, times.getTime().getNativeTimestamp());
} catch (IOException e)
{
throw new TemporaryBackendException(e);
}
}
@Override
public List<KeyRange> getLocalKeyPartition() throws BackendException {
List<KeyRange> result = new LinkedList<>();
TableMask table = null;
try {
ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
table = cnx.getTable(tableName);
HTable hTable = (HTable)table.getTableObject();
Map<KeyRange, ServerName> normed =
normalizeKeyBounds(hTable.getRegionLocations());
for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) {
if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) {
result.add(e.getKey());
logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue());
} else {
logger.debug("Discarding remote {}", e.getValue());
}
}
} catch (MasterNotRunningException e) {
logger.warn("Unexpected MasterNotRunningException", e);
} catch (ZooKeeperConnectionException e) {
logger.warn("Unexpected ZooKeeperConnectionException", e);
} catch (IOException e) {
logger.warn("Unexpected IOException", e);
} finally {
IOUtils.closeQuietly(table);
}
return result;
}
/**
* Given a map produced by {@link HTable#getRegionLocations()}, transform
* each key from an {@link HRegionInfo} to a {@link KeyRange} expressing the
* region's start and end key bounds using Titan-partitioning-friendly
* conventions (start inclusive, end exclusive, zero bytes appended where
* necessary to make all keys at least 4 bytes long).
* <p/>
* This method iterates over the entries in its map parameter and performs
* the following conditional conversions on its keys. "Require" below means
* either a {@link Preconditions} invocation or an assertion. HRegionInfo
* sometimes returns start and end keys of zero length; this method replaces
* zero length keys with null before doing any of the checks described
* below. The parameter map and the values it contains are only read and
* never modified.
*
* <ul>
* <li>If an entry's HRegionInfo has null start and end keys, then first
* require that the parameter map is a singleton, and then return a
* single-entry map whose {@code KeyRange} has start and end buffers that
* are both four bytes of zeros.</li>
* <li>If the entry has a null end key (but non-null start key), put an
* equivalent entry in the result map with a start key identical to the
* input, except that zeros are appended to values less than 4 bytes long,
* and an end key that is four bytes of zeros.
* <li>If the entry has a null start key (but non-null end key), put an
* equivalent entry in the result map where the start key is four bytes of
* zeros, and the end key has zeros appended, if necessary, to make it at
* least 4 bytes long, after which one is added to the padded value in
* unsigned 32-bit arithmetic with overflow allowed.</li>
* <li>Any entry which matches none of the above criteria results in an
* equivalent entry in the returned map, except that zeros are appended to
* both keys to make each at least 4 bytes long, and the end key is then
* incremented as described in the last bullet point.</li>
* </ul>
*
* After iterating over the parameter map, this method checks that it either
* saw no entries with null keys, one entry with a null start key and a
* different entry with a null end key, or one entry with both start and end
* keys null. If any null keys are observed besides these three cases, the
* method will die with a precondition failure.
*
* @param raw
* A map of HRegionInfo and ServerName from HBase
* @return Titan-friendly expression of each region's rowkey boundaries
*/
private Map<KeyRange, ServerName> normalizeKeyBounds(NavigableMap<HRegionInfo, ServerName> raw) {
Map.Entry<HRegionInfo, ServerName> nullStart = null;
Map.Entry<HRegionInfo, ServerName> nullEnd = null;
ImmutableMap.Builder<KeyRange, ServerName> b = ImmutableMap.builder();
for (Map.Entry<HRegionInfo, ServerName> e : raw.entrySet()) {
HRegionInfo regionInfo = e.getKey();
byte startKey[] = regionInfo.getStartKey();
byte endKey[] = regionInfo.getEndKey();
if (0 == startKey.length) {
startKey = null;
logger.trace("Converted zero-length HBase startKey byte array to null");
}
if (0 == endKey.length) {
endKey = null;
logger.trace("Converted zero-length HBase endKey byte array to null");
}
if (null == startKey && null == endKey) {
Preconditions.checkState(1 == raw.size());
logger.debug("HBase table {} has a single region {}", tableName, regionInfo);
// Choose arbitrary shared value = startKey = endKey
return b.put(new KeyRange(FOUR_ZERO_BYTES, FOUR_ZERO_BYTES), e.getValue()).build();
} else if (null == startKey) {
logger.debug("Found HRegionInfo with null startKey on server {}: {}", e.getValue(), regionInfo);
Preconditions.checkState(null == nullStart);
nullStart = e;
// I thought endBuf would be inclusive from the HBase javadoc, but in practice it is exclusive
StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
// Replace null start key with zeroes
b.put(new KeyRange(FOUR_ZERO_BYTES, endBuf), e.getValue());
} else if (null == endKey) {
logger.debug("Found HRegionInfo with null endKey on server {}: {}", e.getValue(), regionInfo);
Preconditions.checkState(null == nullEnd);
nullEnd = e;
// Replace null end key with zeroes
b.put(new KeyRange(StaticArrayBuffer.of(zeroExtend(startKey)), FOUR_ZERO_BYTES), e.getValue());
} else {
Preconditions.checkState(null != startKey);
Preconditions.checkState(null != endKey);
// Convert HBase's inclusive end keys into exclusive Titan end keys
StaticBuffer startBuf = StaticArrayBuffer.of(zeroExtend(startKey));
StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
KeyRange kr = new KeyRange(startBuf, endBuf);
b.put(kr, e.getValue());
logger.debug("Found HRegionInfo with non-null end and start keys on server {}: {}", e.getValue(), regionInfo);
}
}
// Require either no null key bounds or a pair of them
Preconditions.checkState((null == nullStart) == (null == nullEnd));
// Check that every key in the result is at least 4 bytes long
Map<KeyRange, ServerName> result = b.build();
for (KeyRange kr : result.keySet()) {
Preconditions.checkState(4 <= kr.getStart().length());
Preconditions.checkState(4 <= kr.getEnd().length());
}
return result;
}
/**
* If the parameter is shorter than 4 bytes, then create and return a new 4
* byte array with the input array's bytes followed by zero bytes. Otherwise
* return the parameter.
*
* @param dataToPad non-null but possibly zero-length byte array
* @return either the parameter or a new array
*/
private final byte[] zeroExtend(byte[] dataToPad) {
assert null != dataToPad;
final int targetLength = 4;
if (targetLength <= dataToPad.length)
return dataToPad;
byte padded[] = new byte[targetLength];
System.arraycopy(dataToPad, 0, padded, 0, dataToPad.length);
for (int i = dataToPad.length; i < padded.length; i++)
padded[i] = (byte)0;
return padded;
}
public static String shortenCfName(String longName) throws PermanentBackendException {
final String s;
if (SHORT_CF_NAME_MAP.containsKey(longName)) {
s = SHORT_CF_NAME_MAP.get(longName);
Preconditions.checkNotNull(s);
logger.debug("Substituted default CF name \"{}\" with short form \"{}\" to reduce HBase KeyValue size", longName, s);
} else {
if (SHORT_CF_NAME_MAP.containsValue(longName)) {
String fmt = "Must use CF long-form name \"%s\" instead of the short-form name \"%s\" when configured with %s=true";
String msg = String.format(fmt, SHORT_CF_NAME_MAP.inverse().get(longName), longName, SHORT_CF_NAMES.getName());
throw new PermanentBackendException(msg);
}
s = longName;
logger.debug("Kept default CF name \"{}\" because it has no associated short form", s);
}
return s;
}
private HTableDescriptor ensureTableExists(String tableName, String initialCFName, int ttlInSeconds) throws BackendException {
AdminMask adm = null;
HTableDescriptor desc;
try { // Create our table, if necessary
adm = getAdminInterface();
/*
* Some HBase versions/impls respond badly to attempts to create a
* table without at least one CF. See #661. Creating a CF along with
* the table avoids HBase carping.
*/
if (adm.tableExists(tableName)) {
desc = adm.getTableDescriptor(tableName);
} else {
desc = createTable(tableName, initialCFName, ttlInSeconds, adm);
}
} catch (IOException e) {
throw new TemporaryBackendException(e);
} finally {
IOUtils.closeQuietly(adm);
}
return desc;
}
private HTableDescriptor createTable(String tableName, String cfName, int ttlInSeconds, AdminMask adm) throws IOException {
HTableDescriptor desc = compat.newTableDescriptor(tableName);
HColumnDescriptor cdesc = new HColumnDescriptor(cfName);
setCFOptions(cdesc, ttlInSeconds);
compat.addColumnFamilyToTableDescriptor(desc, cdesc);
int count; // total regions to create
String src;
if (MIN_REGION_COUNT <= (count = regionCount)) {
src = "region count configuration";
} else if (0 < regionsPerServer &&
MIN_REGION_COUNT <= (count = regionsPerServer * adm.getEstimatedRegionServerCount())) {
src = "ClusterStatus server count";
} else {
count = -1;
src = "default";
}
if (MIN_REGION_COUNT < count) {
adm.createTable(desc, getStartKey(count), getEndKey(count), count);
logger.debug("Created table {} with region count {} from {}", tableName, count, src);
} else {
adm.createTable(desc);
logger.debug("Created table {} with default start key, end key, and region count", tableName);
}
return desc;
}
/**
* This method generates the second argument to
* {@link HBaseAdmin#createTable(HTableDescriptor, byte[], byte[], int)}.
* <p/>
* From the {@code createTable} javadoc:
* "The start key specified will become the end key of the first region of
* the table, and the end key specified will become the start key of the
* last region of the table (the first region has a null start key and
* the last region has a null end key)"
* <p/>
* To summarize, the {@code createTable} argument called "startKey" is
* actually the end key of the first region.
*/
private byte[] getStartKey(int regionCount) {
ByteBuffer regionWidth = ByteBuffer.allocate(4);
regionWidth.putInt((int)(((1L << 32) - 1L) / regionCount)).flip();
return StaticArrayBuffer.of(regionWidth).getBytes(0, 4);
}
/**
* Companion to {@link #getStartKey(int)}. See its javadoc for details.
*/
private byte[] getEndKey(int regionCount) {
ByteBuffer regionWidth = ByteBuffer.allocate(4);
regionWidth.putInt((int)(((1L << 32) - 1L) / regionCount * (regionCount - 1))).flip();
return StaticArrayBuffer.of(regionWidth).getBytes(0, 4);
}
private void ensureColumnFamilyExists(String tableName, String columnFamily, int ttlInSeconds) throws BackendException {
AdminMask adm = null;
try {
adm = getAdminInterface();
HTableDescriptor desc = ensureTableExists(tableName, columnFamily, ttlInSeconds);
Preconditions.checkNotNull(desc);
HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes());
// Create our column family, if necessary
if (cf == null) {
try {
if (!adm.isTableDisabled(tableName)) {
adm.disableTable(tableName);
}
} catch (TableNotEnabledException e) {
logger.debug("Table {} already disabled", tableName);
} catch (IOException e) {
throw new TemporaryBackendException(e);
}
try {
HColumnDescriptor cdesc = new HColumnDescriptor(columnFamily);
setCFOptions(cdesc, ttlInSeconds);
adm.addColumn(tableName, cdesc);
logger.debug("Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
adm.enableTable(tableName);
} catch (TableNotFoundException ee) {
logger.error("TableNotFoundException", ee);
throw new PermanentBackendException(ee);
} catch (org.apache.hadoop.hbase.TableExistsException ee) {
logger.debug("Swallowing exception {}", ee);
} catch (IOException ee) {
throw new TemporaryBackendException(ee);
}
}
} finally {
IOUtils.closeQuietly(adm);
}
}
private void setCFOptions(HColumnDescriptor cdesc, int ttlInSeconds) {
if (null != compression && !compression.equals(COMPRESSION_DEFAULT))
compat.setCompression(cdesc, compression);
if (ttlInSeconds > 0)
cdesc.setTimeToLive(ttlInSeconds);
cdesc.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
}
/**
* Convert Titan internal Mutation representation into HBase native commands.
*
* @param mutations Mutations to convert into HBase commands.
* @param putTimestamp The timestamp to use for Put commands.
* @param delTimestamp The timestamp to use for Delete commands.
* @return Commands sorted by key converted from Titan internal representation.
* @throws com.thinkaurelius.titan.diskstorage.PermanentBackendException
*/
private Map<StaticBuffer, Pair<Put, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations,
final long putTimestamp,
final long delTimestamp) throws PermanentBackendException {
Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey = new HashMap<>();
for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {
String cfString = getCfNameForStoreName(entry.getKey());
byte[] cfName = cfString.getBytes();
for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
KCVMutation mutation = m.getValue();
Pair<Put, Delete> commands = commandsPerKey.get(m.getKey());
if (commands == null) {
commands = new Pair<>();
commandsPerKey.put(m.getKey(), commands);
}
if (mutation.hasDeletions()) {
if (commands.getSecond() == null) {
Delete d = new Delete(key);
compat.setTimestamp(d, delTimestamp);
commands.setSecond(d);
}
for (StaticBuffer b : mutation.getDeletions()) {
commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
}
}
if (mutation.hasAdditions()) {
if (commands.getFirst() == null) {
Put p = new Put(key, putTimestamp);
commands.setFirst(p);
}
for (Entry e : mutation.getAdditions()) {
commands.getFirst().add(cfName,
e.getColumnAs(StaticBuffer.ARRAY_FACTORY),
putTimestamp,
e.getValueAs(StaticBuffer.ARRAY_FACTORY));
}
}
}
}
return commandsPerKey;
}
private String getCfNameForStoreName(String storeName) throws PermanentBackendException {
return shortCfNames ? shortenCfName(storeName) : storeName;
}
private void checkConfigDeprecation(com.thinkaurelius.titan.diskstorage.configuration.Configuration config) {
if (config.has(GraphDatabaseConfiguration.STORAGE_PORT)) {
logger.warn("The configuration property {} is ignored for HBase. Set hbase.zookeeper.property.clientPort in hbase-site.xml or {}.hbase.zookeeper.property.clientPort in Titan's configuration file.",
ConfigElement.getPath(GraphDatabaseConfiguration.STORAGE_PORT), ConfigElement.getPath(HBASE_CONFIGURATION_NAMESPACE));
}
}
private AdminMask getAdminInterface() {
try {
return cnx.getAdmin();
} catch (IOException e) {
throw new TitanException(e);
}
}
/**
* Similar to {@link Function}, except that the {@code apply} method is allowed
* to throw {@link BackendException}.
*/
private interface BackendFunction<F, T> {
T apply(F input) throws BackendException;
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
import com.thinkaurelius.titan.diskstorage.StaticBuffer;
import com.thinkaurelius.titan.diskstorage.common.AbstractStoreTransaction;
import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction;
import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.LinkedHashSet;
import java.util.Set;
/**
* This class overrides and adds nothing compared with
* {@link com.thinkaurelius.titan.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction}; however, it creates a transaction type specific
* to HBase, which lets us check for user errors like passing a Cassandra
* transaction into a HBase method.
*
* @author Dan LaRocque <dalaro@hopcount.org>
*/
public class HBaseTransaction extends AbstractStoreTransaction {
private static final Logger log = LoggerFactory.getLogger(HBaseTransaction.class);
LocalLockMediator<StoreTransaction> llm;
Set<KeyColumn> keyColumnLocks = new LinkedHashSet<>();
public HBaseTransaction(final BaseTransactionConfig config, LocalLockMediator<StoreTransaction> llm) {
super(config);
this.llm = llm;
}
@Override
public synchronized void rollback() throws BackendException {
super.rollback();
log.debug("Rolled back transaction");
deleteAllLocks();
}
@Override
public synchronized void commit() throws BackendException {
super.commit();
log.debug("Committed transaction");
deleteAllLocks();
}
public void updateLocks(KeyColumn lockID, StaticBuffer expectedValue) {
keyColumnLocks.add(lockID);
}
private void deleteAllLocks() {
for(KeyColumn kc : keyColumnLocks) {
log.debug("Removed lock {} ", kc);
llm.unlock(kc, this);
}
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
public class HConnection0_98 implements ConnectionMask
{
private final HConnection cnx;
public HConnection0_98(HConnection cnx)
{
this.cnx = cnx;
}
@Override
public TableMask getTable(String name) throws IOException
{
return new HTable0_98(cnx.getTable(name));
}
@Override
public AdminMask getAdmin() throws IOException
{
return new HBaseAdmin0_98(new HBaseAdmin(cnx));
}
@Override
public void close() throws IOException
{
cnx.close();
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.HBaseAdmin;
public class HConnection1_0 implements ConnectionMask
{
private final Connection cnx;
public HConnection1_0(Connection cnx)
{
this.cnx = cnx;
}
@Override
public TableMask getTable(String name) throws IOException
{
return new HTable1_0(cnx.getTable(TableName.valueOf(name)));
}
@Override
public AdminMask getAdmin() throws IOException
{
return new HBaseAdmin1_0(new HBaseAdmin(cnx));
}
@Override
public void close() throws IOException
{
cnx.close();
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Row;
import org.apache.hadoop.hbase.client.Scan;
public class HTable0_98 implements TableMask
{
private final HTableInterface table;
public HTable0_98(HTableInterface table)
{
this.table = table;
}
@Override
public ResultScanner getScanner(Scan filter) throws IOException
{
return table.getScanner(filter);
}
@Override
public Result[] get(List<Get> gets) throws IOException
{
return table.get(gets);
}
@Override
public void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException
{
table.batch(writes, results);
table.flushCommits();
}
@Override
public void close() throws IOException
{
table.close();
}
@Override
public Object getTableObject() {
return table;
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Row;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
public class HTable1_0 implements TableMask
{
private final Table table;
public HTable1_0(Table table)
{
this.table = table;
}
@Override
public ResultScanner getScanner(Scan filter) throws IOException
{
return table.getScanner(filter);
}
@Override
public Result[] get(List<Get> gets) throws IOException
{
return table.get(gets);
}
@Override
public void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException
{
table.batch(writes, results);
/* table.flushCommits(); not needed anymore */
}
@Override
public void close() throws IOException
{
table.close();
}
@Override
public Object getTableObject() {
return table;
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Row;
import org.apache.hadoop.hbase.client.Scan;
/**
* This interface hides ABI/API breaking changes that HBase has made to its Table/HTableInterface over the course
* of development from 0.94 to 1.0 and beyond.
*/
public interface TableMask extends Closeable
{
ResultScanner getScanner(Scan filter) throws IOException;
Result[] get(List<Get> gets) throws IOException;
void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException;
Object getTableObject();
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.locking;
import com.google.common.base.Preconditions;
import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
import com.thinkaurelius.titan.diskstorage.util.time.TimestampProvider;
import com.thinkaurelius.titan.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction;
import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.DelayQueue;
import java.util.concurrent.Delayed;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
/**
* This class resolves lock contention between two transactions on the same JVM.
* <p/>
* This is not just an optimization to reduce network traffic. Locks written by
* Titan to a distributed key-value store contain an identifier, the "Rid",
* which is unique only to the process level. The Rid can't tell which
* transaction in a process holds any given lock. This class prevents two
* transactions in a single process from concurrently writing the same lock to a
* distributed key-value store.
*
* @author Dan LaRocque <dalaro@hopcount.org>
*/
public class LocalLockMediator<T> {
private static final Logger log = LoggerFactory
.getLogger(LocalLockMediator.class);
/**
* Namespace for which this mediator is responsible
*
* @see LocalLockMediatorProvider
*/
private final String name;
private final TimestampProvider times;
private DelayQueue<ExpirableKeyColumn> expiryQueue = new DelayQueue<>();
private ExecutorService lockCleanerService = Executors.newFixedThreadPool(1, new ThreadFactory() {
@Override
public Thread newThread(Runnable runnable) {
Thread thread = Executors.defaultThreadFactory().newThread(runnable);
thread.setDaemon(true);
return thread;
}
});
/**
* Maps a ({@code key}, {@code column}) pair to the local transaction
* holding a lock on that pair. Values in this map may have already expired
* according to {@link AuditRecord#expires}, in which case the lock should
* be considered invalid.
*/
private final ConcurrentHashMap<KeyColumn, AuditRecord<T>> locks = new ConcurrentHashMap<>();
public LocalLockMediator(String name, TimestampProvider times) {
this.name = name;
this.times = times;
Preconditions.checkNotNull(name);
Preconditions.checkNotNull(times);
lockCleanerService.submit(new LockCleaner());
}
/**
* Acquire the lock specified by {@code kc}.
* <p/>
* <p/>
* For any particular key-column, whatever value of {@code requestor} is
* passed to this method must also be passed to the associated later call to
* {@link #unlock(KeyColumn, ExpectedValueCheckingTransaction)}.
* <p/>
* If some requestor {@code r} calls this method on a KeyColumn {@code k}
* and this method returns true, then subsequent calls to this method by
* {@code r} on {@code l} merely attempt to update the {@code expiresAt}
* timestamp. This differs from typical lock reentrance: multiple successful
* calls to this method do not require an equal number of calls to
* {@code #unlock()}. One {@code #unlock()} call is enough, no matter how
* many times a {@code requestor} called {@code lock} beforehand. Note that
* updating the timestamp may fail, in which case the lock is considered to
* have expired and the calling context should assume it no longer holds the
* lock specified by {@code kc}.
* <p/>
* The number of nanoseconds elapsed since the UNIX Epoch is not readily
* available within the JVM. When reckoning expiration times, this method
* uses the approximation implemented by
* {@link com.thinkaurelius.titan.diskstorage.util.NanoTime#getApproxNSSinceEpoch(false)}.
* <p/>
* The current implementation of this method returns true when given an
* {@code expiresAt} argument in the past. Future implementations may return
* false instead.
*
* @param kc lock identifier
* @param requestor the object locking {@code kc}
* @param expires instant at which this lock will automatically expire
* @return true if the lock is acquired, false if it was not acquired
*/
public boolean lock(KeyColumn kc, T requestor, Timepoint expires) {
assert null != kc;
assert null != requestor;
AuditRecord<T> audit = new AuditRecord<>(requestor, expires);
AuditRecord<T> inmap = locks.putIfAbsent(kc, audit);
boolean success = false;
if (null == inmap) {
// Uncontended lock succeeded
if (log.isTraceEnabled()) {
log.trace("New local lock created: {} namespace={} txn={}",
kc, name, requestor);
}
success = true;
} else if (inmap.equals(audit)) {
// requestor has already locked kc; update expiresAt
success = locks.replace(kc, inmap, audit);
if (log.isTraceEnabled()) {
if (success) {
log.trace(
"Updated local lock expiration: {} namespace={} txn={} oldexp={} newexp={}",
kc, name, requestor, inmap.expires,
audit.expires);
} else {
log.trace(
"Failed to update local lock expiration: {} namespace={} txn={} oldexp={} newexp={}",
kc, name, requestor, inmap.expires,
audit.expires);
}
}
} else if (0 > inmap.expires.compareTo(times.getTime())) {
// the recorded lock has expired; replace it
success = locks.replace(kc, inmap, audit);
if (log.isTraceEnabled()) {
log.trace(
"Discarding expired lock: {} namespace={} txn={} expired={}",
kc, name, inmap.holder, inmap.expires);
}
} else {
// we lost to a valid lock
if (log.isTraceEnabled()) {
log.trace(
"Local lock failed: {} namespace={} txn={} (already owned by {})",
kc, name, requestor, inmap);
}
}
if (success) {
expiryQueue.add(new ExpirableKeyColumn(kc, expires));
}
return success;
}
/**
* Release the lock specified by {@code kc} and which was previously
* locked by {@code requestor}, if it is possible to release it.
*
* @param kc lock identifier
* @param requestor the object which previously locked {@code kc}
*/
public boolean unlock(KeyColumn kc, T requestor) {
if (!locks.containsKey(kc)) {
log.info("Local unlock failed: no locks found for {}", kc);
return false;
}
AuditRecord<T> unlocker = new AuditRecord<>(requestor, null);
AuditRecord<T> holder = locks.get(kc);
if (!holder.equals(unlocker)) {
log.error("Local unlock of {} by {} failed: it is held by {}",
kc, unlocker, holder);
return false;
}
boolean removed = locks.remove(kc, unlocker);
if (removed) {
expiryQueue.remove(kc);
if (log.isTraceEnabled()) {
log.trace("Local unlock succeeded: {} namespace={} txn={}",
kc, name, requestor);
}
} else {
log.warn("Local unlock warning: lock record for {} disappeared "
+ "during removal; this suggests the lock either expired "
+ "while we were removing it, or that it was erroneously "
+ "unlocked multiple times.", kc);
}
// Even if !removed, we're finished unlocking, so return true
return true;
}
public String toString() {
return "LocalLockMediator [" + name + ", ~" + locks.size()
+ " current locks]";
}
/**
* A record containing the local transaction that holds a lock and the
* lock's expiration time.
*/
private static class AuditRecord<T> {
/**
* The local transaction that holds/held the lock.
*/
private final T holder;
/**
* The expiration time of a the lock.
*/
private final Timepoint expires;
/**
* Cached hashCode.
*/
private int hashCode;
private AuditRecord(T holder, Timepoint expires) {
this.holder = holder;
this.expires = expires;
}
/**
* This implementation depends only on the lock holder and not on the
* lock expiration time.
*/
@Override
public int hashCode() {
if (0 == hashCode)
hashCode = holder.hashCode();
return hashCode;
}
/**
* This implementation depends only on the lock holder and not on the
* lock expiration time.
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
/*
* This warning suppression is harmless because we are only going to
* call other.holder.equals(...), and since equals(...) is part of
* Object, it is guaranteed to be defined no matter the concrete
* type of parameter T.
*/
@SuppressWarnings("rawtypes")
AuditRecord other = (AuditRecord) obj;
if (holder == null) {
if (other.holder != null)
return false;
} else if (!holder.equals(other.holder))
return false;
return true;
}
@Override
public String toString() {
return "AuditRecord [txn=" + holder + ", expires=" + expires + "]";
}
}
private class LockCleaner implements Runnable {
@Override
public void run() {
try {
while (true) {
log.debug("Lock Cleaner service started");
ExpirableKeyColumn lock = expiryQueue.take();
log.debug("Expiring key column " + lock.getKeyColumn());
locks.remove(lock.getKeyColumn());
}
} catch (InterruptedException e) {
log.debug("Received interrupt. Exiting");
}
}
}
private static class ExpirableKeyColumn implements Delayed {
private Timepoint expiryTime;
private KeyColumn kc;
public ExpirableKeyColumn(KeyColumn keyColumn, Timepoint expiryTime) {
this.kc = keyColumn;
this.expiryTime = expiryTime;
}
@Override
public long getDelay(TimeUnit unit) {
return expiryTime.getTimestamp(TimeUnit.NANOSECONDS);
}
@Override
public int compareTo(Delayed o) {
if (this.expiryTime.getTimestamp(TimeUnit.NANOSECONDS) < ((ExpirableKeyColumn) o).expiryTime.getTimestamp(TimeUnit.NANOSECONDS)) {
return -1;
}
if (this.expiryTime.getTimestamp(TimeUnit.NANOSECONDS) > ((ExpirableKeyColumn) o).expiryTime.getTimestamp(TimeUnit.NANOSECONDS)) {
return 1;
}
return 0;
}
public KeyColumn getKeyColumn() {
return kc;
}
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.solr;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import com.thinkaurelius.titan.core.Order;
import com.thinkaurelius.titan.core.TitanElement;
import com.thinkaurelius.titan.core.attribute.Cmp;
import com.thinkaurelius.titan.core.attribute.Geo;
import com.thinkaurelius.titan.core.attribute.Geoshape;
import com.thinkaurelius.titan.core.attribute.Text;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.BaseTransaction;
import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
import com.thinkaurelius.titan.diskstorage.BaseTransactionConfigurable;
import com.thinkaurelius.titan.diskstorage.PermanentBackendException;
import com.thinkaurelius.titan.diskstorage.TemporaryBackendException;
import com.thinkaurelius.titan.diskstorage.configuration.ConfigNamespace;
import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption;
import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
import com.thinkaurelius.titan.diskstorage.indexing.IndexEntry;
import com.thinkaurelius.titan.diskstorage.indexing.IndexFeatures;
import com.thinkaurelius.titan.diskstorage.indexing.IndexMutation;
import com.thinkaurelius.titan.diskstorage.indexing.IndexProvider;
import com.thinkaurelius.titan.diskstorage.indexing.IndexQuery;
import com.thinkaurelius.titan.diskstorage.indexing.KeyInformation;
import com.thinkaurelius.titan.diskstorage.indexing.RawQuery;
import com.thinkaurelius.titan.diskstorage.util.DefaultTransaction;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.thinkaurelius.titan.graphdb.configuration.PreInitializeConfigOptions;
import com.thinkaurelius.titan.graphdb.database.serialize.AttributeUtil;
import com.thinkaurelius.titan.graphdb.database.serialize.attribute.AbstractDecimal;
import com.thinkaurelius.titan.graphdb.query.TitanPredicate;
import com.thinkaurelius.titan.graphdb.query.condition.And;
import com.thinkaurelius.titan.graphdb.query.condition.Condition;
import com.thinkaurelius.titan.graphdb.query.condition.Not;
import com.thinkaurelius.titan.graphdb.query.condition.Or;
import com.thinkaurelius.titan.graphdb.query.condition.PredicateCondition;
import com.thinkaurelius.titan.graphdb.types.ParameterType;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.UUID;
import static com.thinkaurelius.titan.core.attribute.Cmp.*;
import static com.thinkaurelius.titan.core.schema.Mapping.*;
/**
* NOTE: Copied from titan for supporting sol5. Do not change
*/
@PreInitializeConfigOptions
public class Solr5Index implements IndexProvider {
private static final Logger logger = LoggerFactory.getLogger(Solr5Index.class);
private static final String DEFAULT_ID_FIELD = "id";
private enum Mode {
HTTP, CLOUD;
public static Mode parse(String mode) {
for (Mode m : Mode.values()) {
if (m.toString().equalsIgnoreCase(mode)) return m;
}
throw new IllegalArgumentException("Unrecognized mode: "+mode);
}
}
public static final ConfigNamespace SOLR_NS =
new ConfigNamespace(GraphDatabaseConfiguration.INDEX_NS, "solr", "Solr index configuration");
public static final ConfigOption<String> SOLR_MODE = new ConfigOption<>(SOLR_NS, "mode",
"The operation mode for Solr which is either via HTTP (`http`) or using SolrCloud (`cloud`)",
ConfigOption.Type.GLOBAL_OFFLINE, "cloud");
public static final ConfigOption<Boolean> DYNAMIC_FIELDS = new ConfigOption<>(SOLR_NS, "dyn-fields",
"Whether to use dynamic fields (which appends the data type to the field name). If dynamic fields is disabled" +
"the user must map field names and define them explicitly in the schema.",
ConfigOption.Type.GLOBAL_OFFLINE, true);
public static final ConfigOption<String[]> KEY_FIELD_NAMES = new ConfigOption<>(SOLR_NS, "key-field-names",
"Field name that uniquely identifies each document in Solr. Must be specified as a list of `collection=field`.",
ConfigOption.Type.GLOBAL, String[].class);
public static final ConfigOption<String> TTL_FIELD = new ConfigOption<>(SOLR_NS, "ttl_field",
"Name of the TTL field for Solr collections.",
ConfigOption.Type.GLOBAL_OFFLINE, "ttl");
/** SolrCloud Configuration */
public static final ConfigOption<String> ZOOKEEPER_URL = new ConfigOption<>(SOLR_NS, "zookeeper-url",
"URL of the Zookeeper instance coordinating the SolrCloud cluster",
ConfigOption.Type.MASKABLE, "localhost:2181");
public static final ConfigOption<Integer> ZOOKEEPER_CONNECT_TIMEOUT = new ConfigOption<>(SOLR_NS,"zookeeper-connect-timeout",
"SolrCloud Zookeeper connect timeout",
ConfigOption.Type.MASKABLE, 60000);
public static final ConfigOption<Integer> ZOOKEEPER_SESSION_TIMEOUT = new ConfigOption<>(SOLR_NS,"zookeeper-session-timeout",
"SolrCloud Zookeeper session timeout",
ConfigOption.Type.MASKABLE, 60000);
public static final ConfigOption<Integer> NUM_SHARDS = new ConfigOption<>(SOLR_NS,"num-shards",
"Number of shards for a collection. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
ConfigOption.Type.GLOBAL_OFFLINE, 1);
public static final ConfigOption<Integer> MAX_SHARDS_PER_NODE = new ConfigOption<>(SOLR_NS, "max-shards-per-node",
"Maximum number of shards per node. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
ConfigOption.Type.GLOBAL_OFFLINE, 1);
public static final ConfigOption<Integer> REPLICATION_FACTOR = new ConfigOption<>(SOLR_NS, "replication-factor",
"Replication factor for a collection. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
ConfigOption.Type.GLOBAL_OFFLINE, 1);
/** HTTP Configuration */
public static final ConfigOption<String[]> HTTP_URLS = new ConfigOption<>(SOLR_NS, "http-urls",
"List of URLs to use to connect to Solr Servers (LBHttpSolrClient is used), don't add core or collection name to the URL.",
ConfigOption.Type.MASKABLE, new String[]{"http://localhost:8983/solr"});
public static final ConfigOption<Integer> HTTP_CONNECTION_TIMEOUT = new ConfigOption<>(SOLR_NS, "http-connection-timeout",
"Solr HTTP connection timeout.",
ConfigOption.Type.MASKABLE, 5000);
public static final ConfigOption<Boolean> HTTP_ALLOW_COMPRESSION = new ConfigOption<>(SOLR_NS, "http-compression",
"Enable/disable compression on the HTTP connections made to Solr.",
ConfigOption.Type.MASKABLE, false);
public static final ConfigOption<Integer> HTTP_MAX_CONNECTIONS_PER_HOST = new ConfigOption<>(SOLR_NS, "http-max-per-host",
"Maximum number of HTTP connections per Solr host.",
ConfigOption.Type.MASKABLE, 20);
public static final ConfigOption<Integer> HTTP_GLOBAL_MAX_CONNECTIONS = new ConfigOption<>(SOLR_NS, "http-max",
"Maximum number of HTTP connections in total to all Solr servers.",
ConfigOption.Type.MASKABLE, 100);
public static final ConfigOption<Boolean> WAIT_SEARCHER = new ConfigOption<>(SOLR_NS, "wait-searcher",
"When mutating - wait for the index to reflect new mutations before returning. This can have a negative impact on performance.",
ConfigOption.Type.LOCAL, false);
private static final IndexFeatures SOLR_FEATURES = new IndexFeatures.Builder().supportsDocumentTTL()
.setDefaultStringMapping(TEXT).supportedStringMappings(TEXT, STRING).build();
private final SolrClient solrClient;
private final Configuration configuration;
private final Mode mode;
private final boolean dynFields;
private final Map<String, String> keyFieldIds;
private final String ttlField;
private final int maxResults;
private final boolean waitSearcher;
public Solr5Index(final Configuration config) throws BackendException {
Preconditions.checkArgument(config!=null);
configuration = config;
mode = Mode.parse(config.get(SOLR_MODE));
dynFields = config.get(DYNAMIC_FIELDS);
keyFieldIds = parseKeyFieldsForCollections(config);
maxResults = config.get(GraphDatabaseConfiguration.INDEX_MAX_RESULT_SET_SIZE);
ttlField = config.get(TTL_FIELD);
waitSearcher = config.get(WAIT_SEARCHER);
if (mode==Mode.CLOUD) {
HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
String zookeeperUrl = config.get(Solr5Index.ZOOKEEPER_URL);
CloudSolrClient cloudServer = new CloudSolrClient(zookeeperUrl, true);
logger.info("Zookeeper connect timeout : " + config.get(ZOOKEEPER_CONNECT_TIMEOUT));
cloudServer.setZkConnectTimeout(config.get(ZOOKEEPER_CONNECT_TIMEOUT));
logger.info("Zookeeper session timeout : " + config.get(ZOOKEEPER_SESSION_TIMEOUT));
cloudServer.setZkClientTimeout(config.get(ZOOKEEPER_SESSION_TIMEOUT));
cloudServer.connect();
solrClient = cloudServer;
} else if (mode==Mode.HTTP) {
HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
HttpClient clientParams = HttpClientUtil.createClient(new ModifiableSolrParams() {{
add(HttpClientUtil.PROP_ALLOW_COMPRESSION, config.get(HTTP_ALLOW_COMPRESSION).toString());
add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, config.get(HTTP_CONNECTION_TIMEOUT).toString());
add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, config.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
add(HttpClientUtil.PROP_MAX_CONNECTIONS, config.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
}});
solrClient = new LBHttpSolrClient(clientParams, config.get(HTTP_URLS));
} else {
throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
}
}
private Map<String, String> parseKeyFieldsForCollections(Configuration config) throws BackendException {
Map<String, String> keyFieldNames = new HashMap<>();
String[] collectionFieldStatements = config.has(KEY_FIELD_NAMES)?config.get(KEY_FIELD_NAMES):new String[0];
for (String collectionFieldStatement : collectionFieldStatements) {
String[] parts = collectionFieldStatement.trim().split("=");
if (parts.length != 2) {
throw new PermanentBackendException("Unable to parse the collection name / key field name pair. It should be of the format collection=field");
}
String collectionName = parts[0];
String keyFieldName = parts[1];
keyFieldNames.put(collectionName, keyFieldName);
}
return keyFieldNames;
}
private String getKeyFieldId(String collection) {
String field = keyFieldIds.get(collection);
if (field==null) field = DEFAULT_ID_FIELD;
return field;
}
/**
* Unlike the ElasticSearch Index, which is schema free, Solr requires a schema to
* support searching. This means that you will need to modify the solr schema with the
* appropriate field definitions in order to work properly. If you have a running instance
* of Solr and you modify its schema with new fields, don't forget to re-index!
* @param store Index store
* @param key New key to register
* @param information Datatype to register for the key
* @param tx enclosing transaction
* @throws com.thinkaurelius.titan.diskstorage.BackendException
*/
@Override
public void register(String store, String key, KeyInformation information, BaseTransaction tx) throws BackendException {
if (mode==Mode.CLOUD) {
CloudSolrClient client = (CloudSolrClient) solrClient;
try {
createCollectionIfNotExists(client, configuration, store);
} catch (IOException | KeeperException | InterruptedException | SolrServerException e) {
throw new PermanentBackendException(e);
}
}
//Since all data types must be defined in the schema.xml, pre-registering a type does not work
}
@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
logger.debug("Mutating SOLR");
try {
for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
String collectionName = stores.getKey();
String keyIdField = getKeyFieldId(collectionName);
List<String> deleteIds = new ArrayList<>();
Collection<SolrInputDocument> changes = new ArrayList<>();
for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
String docId = entry.getKey();
IndexMutation mutation = entry.getValue();
Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
//Handle any deletions
if (mutation.hasDeletions()) {
if (mutation.isDeleted()) {
logger.trace("Deleting entire document {}", docId);
deleteIds.add(docId);
} else {
HashSet<IndexEntry> fieldDeletions = Sets.newHashSet(mutation.getDeletions());
if (mutation.hasAdditions()) {
for (IndexEntry indexEntry : mutation.getAdditions()) {
fieldDeletions.remove(indexEntry);
}
}
deleteIndividualFieldsFromIndex(collectionName, keyIdField, docId, fieldDeletions);
}
}
if (mutation.hasAdditions()) {
int ttl = mutation.determineTTL();
SolrInputDocument doc = new SolrInputDocument();
doc.setField(keyIdField, docId);
boolean isNewDoc = mutation.isNew();
if (isNewDoc)
logger.trace("Adding new document {}", docId);
for (IndexEntry e : mutation.getAdditions()) {
final Object fieldValue = convertValue(e.value);
doc.setField(e.field, isNewDoc
? fieldValue : new HashMap<String, Object>(1) {{ put("set", fieldValue); }});
}
if (ttl>0) {
Preconditions.checkArgument(isNewDoc,"Solr only supports TTL on new documents [%s]",docId);
doc.setField(ttlField, String.format("+%dSECONDS", ttl));
}
changes.add(doc);
}
}
commitDeletes(collectionName, deleteIds);
commitDocumentChanges(collectionName, changes);
}
} catch (Exception e) {
throw storageException(e);
}
}
private Object convertValue(Object value) throws BackendException {
if (value instanceof Geoshape)
return GeoToWktConverter.convertToWktString((Geoshape) value);
// in order to serialize/deserialize properly Solr will have to have an
// access to Titan source which has Decimal type, so for now we simply convert to
// double and let Solr do the same thing or fail.
if (value instanceof AbstractDecimal)
return ((AbstractDecimal) value).doubleValue();
if (value instanceof UUID)
return value.toString();
return value;
}
@Override
public void restore(Map<String, Map<String, List<IndexEntry>>> documents, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
try {
for (Map.Entry<String, Map<String, List<IndexEntry>>> stores : documents.entrySet()) {
final String collectionName = stores.getKey();
List<String> deleteIds = new ArrayList<>();
List<SolrInputDocument> newDocuments = new ArrayList<>();
for (Map.Entry<String, List<IndexEntry>> entry : stores.getValue().entrySet()) {
final String docID = entry.getKey();
final List<IndexEntry> content = entry.getValue();
if (content == null || content.isEmpty()) {
if (logger.isTraceEnabled())
logger.trace("Deleting document [{}]", docID);
deleteIds.add(docID);
continue;
}
newDocuments.add(new SolrInputDocument() {{
setField(getKeyFieldId(collectionName), docID);
for (IndexEntry addition : content) {
Object fieldValue = addition.value;
setField(addition.field, convertValue(fieldValue));
}
}});
}
commitDeletes(collectionName, deleteIds);
commitDocumentChanges(collectionName, newDocuments);
}
} catch (Exception e) {
throw new TemporaryBackendException("Could not restore Solr index", e);
}
}
private void deleteIndividualFieldsFromIndex(String collectionName, String keyIdField, String docId, HashSet<IndexEntry> fieldDeletions) throws SolrServerException, IOException {
if (fieldDeletions.isEmpty()) return;
Map<String, String> fieldDeletes = new HashMap<String, String>(1) {{ put("set", null); }};
SolrInputDocument doc = new SolrInputDocument();
doc.addField(keyIdField, docId);
StringBuilder sb = new StringBuilder();
for (IndexEntry fieldToDelete : fieldDeletions) {
doc.addField(fieldToDelete.field, fieldDeletes);
sb.append(fieldToDelete).append(",");
}
if (logger.isTraceEnabled())
logger.trace("Deleting individual fields [{}] for document {}", sb.toString(), docId);
UpdateRequest singleDocument = newUpdateRequest();
singleDocument.add(doc);
solrClient.request(singleDocument, collectionName);
}
private void commitDocumentChanges(String collectionName, Collection<SolrInputDocument> documents) throws SolrServerException, IOException {
if (documents.size() == 0) return;
try {
solrClient.request(newUpdateRequest().add(documents), collectionName);
} catch (HttpSolrClient.RemoteSolrException rse) {
logger.error("Unable to save documents to Solr as one of the shape objects stored were not compatible with Solr.", rse);
logger.error("Details in failed document batch: ");
for (SolrInputDocument d : documents) {
Collection<String> fieldNames = d.getFieldNames();
for (String name : fieldNames) {
logger.error(name + ":" + d.getFieldValue(name).toString());
}
}
throw rse;
}
}
private void commitDeletes(String collectionName, List<String> deleteIds) throws SolrServerException, IOException {
if (deleteIds.size() == 0) return;
solrClient.request(newUpdateRequest().deleteById(deleteIds), collectionName);
}
@Override
public List<String> query(IndexQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
List<String> result;
String collection = query.getStore();
String keyIdField = getKeyFieldId(collection);
SolrQuery solrQuery = new SolrQuery("*:*");
String queryFilter = buildQueryFilter(query.getCondition(), informations.get(collection));
solrQuery.addFilterQuery(queryFilter);
if (!query.getOrder().isEmpty()) {
List<IndexQuery.OrderEntry> orders = query.getOrder();
for (IndexQuery.OrderEntry order1 : orders) {
String item = order1.getKey();
SolrQuery.ORDER order = order1.getOrder() == Order.ASC ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc;
solrQuery.addSort(new SolrQuery.SortClause(item, order));
}
}
solrQuery.setStart(0);
if (query.hasLimit()) {
solrQuery.setRows(query.getLimit());
} else {
solrQuery.setRows(maxResults);
}
try {
QueryResponse response = solrClient.query(collection, solrQuery);
if (logger.isDebugEnabled())
logger.debug("Executed query [{}] in {} ms", query.getCondition(), response.getElapsedTime());
int totalHits = response.getResults().size();
if (!query.hasLimit() && totalHits >= maxResults)
logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
result = new ArrayList<>(totalHits);
for (SolrDocument hit : response.getResults()) {
result.add(hit.getFieldValue(keyIdField).toString());
}
} catch (IOException e) {
logger.error("Query did not complete : ", e);
throw new PermanentBackendException(e);
} catch (SolrServerException e) {
logger.error("Unable to query Solr index.", e);
throw new PermanentBackendException(e);
}
return result;
}
@Override
public Iterable<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
List<RawQuery.Result<String>> result;
String collection = query.getStore();
String keyIdField = getKeyFieldId(collection);
SolrQuery solrQuery = new SolrQuery(query.getQuery())
.addField(keyIdField)
.setIncludeScore(true)
.setStart(query.getOffset())
.setRows(query.hasLimit() ? query.getLimit() : maxResults);
try {
QueryResponse response = solrClient.query(collection, solrQuery);
if (logger.isDebugEnabled())
logger.debug("Executed query [{}] in {} ms", query.getQuery(), response.getElapsedTime());
int totalHits = response.getResults().size();
if (!query.hasLimit() && totalHits >= maxResults) {
logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
}
result = new ArrayList<>(totalHits);
for (SolrDocument hit : response.getResults()) {
double score = Double.parseDouble(hit.getFieldValue("score").toString());
result.add(new RawQuery.Result<>(hit.getFieldValue(keyIdField).toString(), score));
}
} catch (IOException e) {
logger.error("Query did not complete : ", e);
throw new PermanentBackendException(e);
} catch (SolrServerException e) {
logger.error("Unable to query Solr index.", e);
throw new PermanentBackendException(e);
}
return result;
}
private static String escapeValue(Object value) {
return ClientUtils.escapeQueryChars(value.toString());
}
public String buildQueryFilter(Condition<TitanElement> condition, KeyInformation.StoreRetriever informations) {
if (condition instanceof PredicateCondition) {
PredicateCondition<String, TitanElement> atom = (PredicateCondition<String, TitanElement>) condition;
Object value = atom.getValue();
String key = atom.getKey();
TitanPredicate titanPredicate = atom.getPredicate();
if (value instanceof Number) {
String queryValue = escapeValue(value);
Preconditions.checkArgument(titanPredicate instanceof Cmp, "Relation not supported on numeric types: " + titanPredicate);
Cmp numRel = (Cmp) titanPredicate;
switch (numRel) {
case EQUAL:
return (key + ":" + queryValue);
case NOT_EQUAL:
return ("-" + key + ":" + queryValue);
case LESS_THAN:
//use right curly to mean up to but not including value
return (key + ":[* TO " + queryValue + "}");
case LESS_THAN_EQUAL:
return (key + ":[* TO " + queryValue + "]");
case GREATER_THAN:
//use left curly to mean greater than but not including value
return (key + ":{" + queryValue + " TO *]");
case GREATER_THAN_EQUAL:
return (key + ":[" + queryValue + " TO *]");
default: throw new IllegalArgumentException("Unexpected relation: " + numRel);
}
} else if (value instanceof String) {
Mapping map = getStringMapping(informations.get(key));
assert map== TEXT || map== STRING;
if (map== TEXT && !titanPredicate.toString().startsWith("CONTAINS"))
throw new IllegalArgumentException("Text mapped string values only support CONTAINS queries and not: " + titanPredicate);
if (map== STRING && titanPredicate.toString().startsWith("CONTAINS"))
throw new IllegalArgumentException("String mapped string values do not support CONTAINS queries: " + titanPredicate);
//Special case
if (titanPredicate == Text.CONTAINS) {
//e.g. - if terms tomorrow and world were supplied, and fq=text:(tomorrow world)
//sample data set would return 2 documents: one where text = Tomorrow is the World,
//and the second where text = Hello World. Hence, we are decomposing the query string
//and building an AND query explicitly because we need AND semantics
value = ((String) value).toLowerCase();
List<String> terms = Text.tokenize((String) value);
if (terms.isEmpty()) {
return "";
} else if (terms.size() == 1) {
return (key + ":(" + escapeValue(terms.get(0)) + ")");
} else {
And<TitanElement> andTerms = new And<>();
for (String term : terms) {
andTerms.add(new PredicateCondition<>(key, titanPredicate, term));
}
return buildQueryFilter(andTerms, informations);
}
}
if (titanPredicate == Text.PREFIX || titanPredicate == Text.CONTAINS_PREFIX) {
return (key + ":" + escapeValue(value) + "*");
} else if (titanPredicate == Text.REGEX || titanPredicate == Text.CONTAINS_REGEX) {
return (key + ":/" + value + "/");
} else if (titanPredicate == EQUAL) {
return (key + ":\"" + escapeValue(value) + "\"");
} else if (titanPredicate == NOT_EQUAL) {
return ("-" + key + ":\"" + escapeValue(value) + "\"");
} else {
throw new IllegalArgumentException("Relation is not supported for string value: " + titanPredicate);
}
} else if (value instanceof Geoshape) {
Geoshape geo = (Geoshape)value;
if (geo.getType() == Geoshape.Type.CIRCLE) {
Geoshape.Point center = geo.getPoint();
return ("{!geofilt sfield=" + key +
" pt=" + center.getLatitude() + "," + center.getLongitude() +
" d=" + geo.getRadius() + "} distErrPct=0"); //distance in kilometers
} else if (geo.getType() == Geoshape.Type.BOX) {
Geoshape.Point southwest = geo.getPoint(0);
Geoshape.Point northeast = geo.getPoint(1);
return (key + ":[" + southwest.getLatitude() + "," + southwest.getLongitude() +
" TO " + northeast.getLatitude() + "," + northeast.getLongitude() + "]");
} else if (geo.getType() == Geoshape.Type.POLYGON) {
List<Geoshape.Point> coordinates = getPolygonPoints(geo);
StringBuilder poly = new StringBuilder(key + ":\"IsWithin(POLYGON((");
for (Geoshape.Point coordinate : coordinates) {
poly.append(coordinate.getLongitude()).append(" ").append(coordinate.getLatitude()).append(", ");
}
//close the polygon with the first coordinate
poly.append(coordinates.get(0).getLongitude()).append(" ").append(coordinates.get(0).getLatitude());
poly.append(")))\" distErrPct=0");
return (poly.toString());
}
} else if (value instanceof Date) {
String queryValue = escapeValue(toIsoDate((Date)value));
Preconditions.checkArgument(titanPredicate instanceof Cmp, "Relation not supported on date types: " + titanPredicate);
Cmp numRel = (Cmp) titanPredicate;
switch (numRel) {
case EQUAL:
return (key + ":" + queryValue);
case NOT_EQUAL:
return ("-" + key + ":" + queryValue);
case LESS_THAN:
//use right curly to mean up to but not including value
return (key + ":[* TO " + queryValue + "}");
case LESS_THAN_EQUAL:
return (key + ":[* TO " + queryValue + "]");
case GREATER_THAN:
//use left curly to mean greater than but not including value
return (key + ":{" + queryValue + " TO *]");
case GREATER_THAN_EQUAL:
return (key + ":[" + queryValue + " TO *]");
default: throw new IllegalArgumentException("Unexpected relation: " + numRel);
}
} else if (value instanceof Boolean) {
Cmp numRel = (Cmp) titanPredicate;
String queryValue = escapeValue(value);
switch (numRel) {
case EQUAL:
return (key + ":" + queryValue);
case NOT_EQUAL:
return ("-" + key + ":" + queryValue);
default:
throw new IllegalArgumentException("Boolean types only support EQUAL or NOT_EQUAL");
}
} else if (value instanceof UUID) {
if (titanPredicate == EQUAL) {
return (key + ":\"" + escapeValue(value) + "\"");
} else if (titanPredicate == NOT_EQUAL) {
return ("-" + key + ":\"" + escapeValue(value) + "\"");
} else {
throw new IllegalArgumentException("Relation is not supported for uuid value: " + titanPredicate);
}
} else throw new IllegalArgumentException("Unsupported type: " + value);
} else if (condition instanceof Not) {
String sub = buildQueryFilter(((Not)condition).getChild(),informations);
if (StringUtils.isNotBlank(sub)) return "-("+sub+")";
else return "";
} else if (condition instanceof And) {
int numChildren = ((And) condition).size();
StringBuilder sb = new StringBuilder();
for (Condition<TitanElement> c : condition.getChildren()) {
String sub = buildQueryFilter(c, informations);
if (StringUtils.isBlank(sub))
continue;
// we don't have to add "+" which means AND iff
// a. it's a NOT query,
// b. expression is a single statement in the AND.
if (!sub.startsWith("-") && numChildren > 1)
sb.append("+");
sb.append(sub).append(" ");
}
return sb.toString();
} else if (condition instanceof Or) {
StringBuilder sb = new StringBuilder();
int element=0;
for (Condition<TitanElement> c : condition.getChildren()) {
String sub = buildQueryFilter(c,informations);
if (StringUtils.isBlank(sub)) continue;
if (element==0) sb.append("(");
else sb.append(" OR ");
sb.append(sub);
element++;
}
if (element>0) sb.append(")");
return sb.toString();
} else {
throw new IllegalArgumentException("Invalid condition: " + condition);
}
return null;
}
private String toIsoDate(Date value) {
TimeZone tz = TimeZone.getTimeZone("UTC");
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
df.setTimeZone(tz);
return df.format(value);
}
private List<Geoshape.Point> getPolygonPoints(Geoshape polygon) {
List<Geoshape.Point> locations = new ArrayList<>();
int index = 0;
boolean hasCoordinates = true;
while (hasCoordinates) {
try {
locations.add(polygon.getPoint(index));
} catch (ArrayIndexOutOfBoundsException ignore) {
//just means we asked for a point past the size of the list
//of known coordinates
hasCoordinates = false;
}
}
return locations;
}
/**
* Solr handles all transactions on the server-side. That means all
* commit, optimize, or rollback applies since the last commit/optimize/rollback.
* Solr documentation recommends best way to update Solr is in one process to avoid
* race conditions.
*
* @return New Transaction Handle
* @throws com.thinkaurelius.titan.diskstorage.BackendException
*/
@Override
public BaseTransactionConfigurable beginTransaction(BaseTransactionConfig config) throws BackendException {
return new DefaultTransaction(config);
}
@Override
public void close() throws BackendException {
logger.trace("Shutting down connection to Solr", solrClient);
try {
solrClient.close();
} catch (IOException e) {
throw new TemporaryBackendException(e);
}
}
@Override
public void clearStorage() throws BackendException {
try {
if (mode!=Mode.CLOUD) throw new UnsupportedOperationException("Operation only supported for SolrCloud");
logger.debug("Clearing storage from Solr: {}", solrClient);
ZkStateReader zkStateReader = ((CloudSolrClient) solrClient).getZkStateReader();
zkStateReader.updateClusterState();
ClusterState clusterState = zkStateReader.getClusterState();
for (String collection : clusterState.getCollections()) {
logger.debug("Clearing collection [{}] in Solr",collection);
UpdateRequest deleteAll = newUpdateRequest();
deleteAll.deleteByQuery("*:*");
solrClient.request(deleteAll, collection);
}
} catch (SolrServerException e) {
logger.error("Unable to clear storage from index due to server error on Solr.", e);
throw new PermanentBackendException(e);
} catch (IOException e) {
logger.error("Unable to clear storage from index due to low-level I/O error.", e);
throw new PermanentBackendException(e);
} catch (Exception e) {
logger.error("Unable to clear storage from index due to general error.", e);
throw new PermanentBackendException(e);
}
}
@Override
public boolean supports(KeyInformation information, TitanPredicate titanPredicate) {
Class<?> dataType = information.getDataType();
Mapping mapping = getMapping(information);
if (mapping!= DEFAULT && !AttributeUtil.isString(dataType)) return false;
if (Number.class.isAssignableFrom(dataType)) {
return titanPredicate instanceof Cmp;
} else if (dataType == Geoshape.class) {
return titanPredicate == Geo.WITHIN;
} else if (AttributeUtil.isString(dataType)) {
switch(mapping) {
case DEFAULT:
case TEXT:
return titanPredicate == Text.CONTAINS || titanPredicate == Text.CONTAINS_PREFIX || titanPredicate == Text.CONTAINS_REGEX;
case STRING:
return titanPredicate == EQUAL || titanPredicate== NOT_EQUAL || titanPredicate==Text.REGEX || titanPredicate==Text.PREFIX;
// case TEXTSTRING:
// return (titanPredicate instanceof Text) || titanPredicate == Cmp.EQUAL || titanPredicate==Cmp.NOT_EQUAL;
}
} else if (dataType == Date.class) {
if (titanPredicate instanceof Cmp) return true;
} else if (dataType == Boolean.class) {
return titanPredicate == EQUAL || titanPredicate == NOT_EQUAL;
} else if (dataType == UUID.class) {
return titanPredicate == EQUAL || titanPredicate== NOT_EQUAL;
}
return false;
}
@Override
public boolean supports(KeyInformation information) {
Class<?> dataType = information.getDataType();
Mapping mapping = getMapping(information);
if (Number.class.isAssignableFrom(dataType) || dataType == Geoshape.class || dataType == Date.class || dataType == Boolean.class || dataType == UUID.class) {
if (mapping== DEFAULT) return true;
} else if (AttributeUtil.isString(dataType)) {
if (mapping== DEFAULT || mapping== TEXT || mapping== STRING) return true;
}
return false;
}
@Override
public String mapKey2Field(String key, KeyInformation keyInfo) {
Preconditions.checkArgument(!StringUtils.containsAny(key, new char[]{' '}),"Invalid key name provided: %s",key);
if (!dynFields) return key;
if (ParameterType.MAPPED_NAME.hasParameter(keyInfo.getParameters())) return key;
String postfix;
Class datatype = keyInfo.getDataType();
if (AttributeUtil.isString(datatype)) {
Mapping map = getStringMapping(keyInfo);
switch (map) {
case TEXT: postfix = "_t"; break;
case STRING: postfix = "_s"; break;
default: throw new IllegalArgumentException("Unsupported string mapping: " + map);
}
} else if (AttributeUtil.isWholeNumber(datatype)) {
if (datatype.equals(Long.class)) postfix = "_l";
else postfix = "_i";
} else if (AttributeUtil.isDecimal(datatype)) {
if (datatype.equals(Float.class)) postfix = "_f";
else postfix = "_d";
} else if (datatype.equals(Geoshape.class)) {
postfix = "_g";
} else if (datatype.equals(Date.class)) {
postfix = "_dt";
} else if (datatype.equals(Boolean.class)) {
postfix = "_b";
} else if (datatype.equals(UUID.class)) {
postfix = "_uuid";
} else throw new IllegalArgumentException("Unsupported data type ["+datatype+"] for field: " + key);
return key+postfix;
}
@Override
public IndexFeatures getFeatures() {
return SOLR_FEATURES;
}
/*
################# UTILITY METHODS #######################
*/
private static Mapping getStringMapping(KeyInformation information) {
assert AttributeUtil.isString(information.getDataType());
Mapping map = getMapping(information);
if (map== DEFAULT) map = TEXT;
return map;
}
private UpdateRequest newUpdateRequest() {
UpdateRequest req = new UpdateRequest();
req.setAction(UpdateRequest.ACTION.COMMIT, true, true);
if (waitSearcher) {
req.setAction(UpdateRequest.ACTION.COMMIT, true, true);
}
return req;
}
private BackendException storageException(Exception solrException) {
return new TemporaryBackendException("Unable to complete query on Solr.", solrException);
}
private static void createCollectionIfNotExists(CloudSolrClient client, Configuration config, String collection)
throws IOException, SolrServerException, KeeperException, InterruptedException {
if (!checkIfCollectionExists(client, collection)) {
Integer numShards = config.get(NUM_SHARDS);
Integer maxShardsPerNode = config.get(MAX_SHARDS_PER_NODE);
Integer replicationFactor = config.get(REPLICATION_FACTOR);
CollectionAdminRequest.Create createRequest = new CollectionAdminRequest.Create();
createRequest.setConfigName(collection);
createRequest.setCollectionName(collection);
createRequest.setNumShards(numShards);
createRequest.setMaxShardsPerNode(maxShardsPerNode);
createRequest.setReplicationFactor(replicationFactor);
CollectionAdminResponse createResponse = createRequest.process(client);
if (createResponse.isSuccess()) {
logger.trace("Collection {} successfully created.", collection);
} else {
throw new SolrServerException(Joiner.on("\n").join(createResponse.getErrorMessages()));
}
}
waitForRecoveriesToFinish(client, collection);
}
/**
* Checks if the collection has already been created in Solr.
*/
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
ZkStateReader zkStateReader = server.getZkStateReader();
zkStateReader.updateClusterState();
ClusterState clusterState = zkStateReader.getClusterState();
return clusterState.getCollectionOrNull(collection) != null;
}
/**
* Wait for all the collection shards to be ready.
*/
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
ZkStateReader zkStateReader = server.getZkStateReader();
try {
boolean cont = true;
while (cont) {
boolean sawLiveRecovering = false;
zkStateReader.updateClusterState();
ClusterState clusterState = zkStateReader.getClusterState();
Map<String, Slice> slices = clusterState.getSlicesMap(collection);
Preconditions.checkNotNull("Could not find collection:" + collection, slices);
for (Map.Entry<String, Slice> entry : slices.entrySet()) {
Map<String, Replica> shards = entry.getValue().getReplicasMap();
for (Map.Entry<String, Replica> shard : shards.entrySet()) {
String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
if ((state.equals(Replica.State.RECOVERING.toString())
|| state.equals(Replica.State.DOWN.toString()))
&& clusterState.liveNodesContain(shard.getValue().getStr(
ZkStateReader.NODE_NAME_PROP))) {
sawLiveRecovering = true;
}
}
}
if (!sawLiveRecovering) {
cont = false;
} else {
Thread.sleep(1000);
}
}
} finally {
logger.info("Exiting solr wait");
}
}
private static class GeoToWktConverter {
/**
* {@link com.thinkaurelius.titan.core.attribute.Geoshape} stores Points in the String format: point[X.0,Y.0].
* Solr needs it to be in Well-Known Text format: POINT(X.0 Y.0)
*/
static String convertToWktString(Geoshape fieldValue) throws BackendException {
if (fieldValue.getType() == Geoshape.Type.POINT) {
Geoshape.Point point = fieldValue.getPoint();
return "POINT(" + point.getLongitude() + " " + point.getLatitude() + ")";
} else {
throw new PermanentBackendException("Cannot index " + fieldValue.getType());
}
}
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/database/idassigner/StandardIDPool.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.graphdb.database.idassigner;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.thinkaurelius.titan.core.TitanException;
import com.thinkaurelius.titan.core.attribute.Duration;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.IDAuthority;
import com.thinkaurelius.titan.diskstorage.IDBlock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* @author Matthias Broecheler (me@matthiasb.com)
*/
public class StandardIDPool implements IDPool {
private static final Logger log =
LoggerFactory.getLogger(StandardIDPool.class);
private static final TimeUnit SCHEDULING_TIME_UNIT =
TimeUnit.MILLISECONDS; // TODO
private static final IDBlock ID_POOL_EXHAUSTION = new IDBlock() {
@Override
public long numIds() {
throw new UnsupportedOperationException();
}
@Override
public long getId(long index) {
throw new UnsupportedOperationException();
}
};
private static final IDBlock UNINITIALIZED_BLOCK = new IDBlock() {
@Override
public long numIds() {
return 0;
}
@Override
public long getId(long index) {
throw new ArrayIndexOutOfBoundsException(0);
}
};
private static final int RENEW_ID_COUNT = 100;
private final IDAuthority idAuthority;
private final long idUpperBound; //exclusive
private final int partition;
private final int idNamespace;
private final Duration renewTimeout;
private final double renewBufferPercentage;
private IDBlock currentBlock;
private long currentIndex;
private long renewBlockIndex;
// private long nextID;
// private long currentMaxID;
// private long renewBufferID;
private volatile IDBlock nextBlock;
private Future<?> idBlockFuture;
private final ThreadPoolExecutor exec;
private volatile boolean initialized;
private volatile boolean closed;
public StandardIDPool(IDAuthority idAuthority, int partition, int idNamespace, long idUpperBound, Duration renewTimeout, double renewBufferPercentage) {
Preconditions.checkArgument(idUpperBound > 0);
this.idAuthority = idAuthority;
Preconditions.checkArgument(partition>=0);
this.partition = partition;
Preconditions.checkArgument(idNamespace>=0);
this.idNamespace = idNamespace;
this.idUpperBound = idUpperBound;
Preconditions.checkArgument(!renewTimeout.isZeroLength(), "Renew-timeout must be positive");
this.renewTimeout = renewTimeout;
Preconditions.checkArgument(renewBufferPercentage>0.0 && renewBufferPercentage<=1.0,"Renew-buffer percentage must be in (0.0,1.0]");
this.renewBufferPercentage = renewBufferPercentage;
currentBlock = UNINITIALIZED_BLOCK;
currentIndex = 0;
renewBlockIndex = 0;
nextBlock = null;
// daemon=true would probably be fine too
exec = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(), new ThreadFactoryBuilder()
.setDaemon(false)
.setNameFormat("TitanID(" + partition + ")("+idNamespace+")[%d]")
.build());
//exec.allowCoreThreadTimeOut(false);
//exec.prestartCoreThread();
idBlockFuture = null;
initialized = false;
closed = false;
}
private void waitForIDRenewer() throws InterruptedException {
long start = swStart();
if (null != idBlockFuture) {
try {
idBlockFuture.get(renewTimeout.getLength(SCHEDULING_TIME_UNIT), SCHEDULING_TIME_UNIT);
} catch (ExecutionException e) {
String msg = String.format("ID block allocation on partition(%d)-namespace(%d) failed with an exception in %s",
partition, idNamespace, swStop(start));
throw new TitanException(msg, e);
} catch (TimeoutException e) {
// Attempt to cancel the renewer
idBlockFuture.cancel(true);
String msg = String.format("ID block allocation on partition(%d)-namespace(%d) timed out in %s",
partition, idNamespace, swStop(start));
throw new TitanException(msg, e);
} catch (CancellationException e) {
String msg = String.format("ID block allocation on partition(%d)-namespace(%d) was cancelled after %s",
partition, idNamespace, swStop(start));
throw new TitanException(msg, e);
} finally {
idBlockFuture = null;
}
// Allow InterruptedException to propagate up the stack
}
}
private long swStop(long start) {
return swStart() - start;
}
private synchronized void nextBlock() throws InterruptedException {
assert currentIndex == currentBlock.numIds();
Preconditions.checkState(!closed,"ID Pool has been closed for partition(%s)-namespace(%s) - cannot apply for new id block",
partition,idNamespace);
waitForIDRenewer();
if (nextBlock == ID_POOL_EXHAUSTION)
throw new IDPoolExhaustedException("Exhausted ID Pool for partition(" + partition+")-namespace("+idNamespace+")");
Preconditions.checkArgument(nextBlock!=null);
currentBlock = nextBlock;
currentIndex = 0;
log.debug("ID partition({})-namespace({}) acquired block: [{}]", partition, idNamespace, currentBlock);
assert currentBlock.numIds()>0;
nextBlock = null;
assert RENEW_ID_COUNT>0;
renewBlockIndex = Math.max(0,currentBlock.numIds()-Math.max(RENEW_ID_COUNT, Math.round(currentBlock.numIds()*renewBufferPercentage)));
assert renewBlockIndex<currentBlock.numIds() && renewBlockIndex>=currentIndex;
}
private void renewBuffer() {
Preconditions.checkArgument(nextBlock == null, nextBlock);
try {
long start = swStart();
IDBlock idBlock = idAuthority.getIDBlock(partition, idNamespace, renewTimeout);
log.debug("Retrieved ID block from authority on partition({})-namespace({}) in {}", partition, idNamespace, swStop(start));
Preconditions.checkArgument(idBlock!=null && idBlock.numIds()>0);
nextBlock = idBlock;
} catch (BackendException e) {
throw new TitanException("Could not acquire new ID block from storage", e);
} catch (IDPoolExhaustedException e) {
nextBlock = ID_POOL_EXHAUSTION;
}
}
private long swStart() {
return System.currentTimeMillis();
}
@Override
public synchronized long nextID() {
assert currentIndex <= currentBlock.numIds();
if (!initialized) {
startNextIDAcquisition();
initialized = true;
}
if (currentIndex == currentBlock.numIds()) {
try {
nextBlock();
} catch (InterruptedException e) {
throw new TitanException("Could not renew id block due to interruption", e);
}
}
if (currentIndex == renewBlockIndex) {
startNextIDAcquisition();
}
long returnId = currentBlock.getId(currentIndex);
currentIndex++;
if (returnId >= idUpperBound) throw new IDPoolExhaustedException("Reached id upper bound of " + idUpperBound);
log.trace("partition({})-namespace({}) Returned id: {}", partition, idNamespace, returnId);
return returnId;
}
@Override
public synchronized void close() {
closed=true;
//Wait for renewer to finish -- call exec.shutdownNow() instead?
try {
waitForIDRenewer();
} catch (InterruptedException e) {
throw new TitanException("Interrupted while waiting for id renewer thread to finish", e);
}
exec.shutdownNow();
}
private void startNextIDAcquisition() {
Preconditions.checkArgument(idBlockFuture == null, idBlockFuture);
if (closed) return; //Don't renew anymore if closed
//Renew buffer
log.debug("Starting id block renewal thread upon {}", currentIndex);
idBlockFuture = exec.submit(new IDBlockRunnable());
}
private class IDBlockRunnable implements Runnable {
@Override
public void run() {
renewBuffer();
}
}
}
graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/condition/PredicateCondition.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.graphdb.query.condition;
import com.google.common.base.Preconditions;
import com.thinkaurelius.titan.core.*;
import com.thinkaurelius.titan.graphdb.internal.InternalElement;
import com.thinkaurelius.titan.graphdb.internal.InternalRelationType;
import com.thinkaurelius.titan.graphdb.query.TitanPredicate;
import com.thinkaurelius.titan.graphdb.util.ElementHelper;
import com.tinkerpop.blueprints.Direction;
import org.apache.commons.lang.builder.HashCodeBuilder;
import java.util.Iterator;
/**
* @author Matthias Broecheler (me@matthiasb.com)
*/
public class PredicateCondition<K, E extends TitanElement> extends Literal<E> {
private final K key;
private final TitanPredicate predicate;
private final Object value;
public PredicateCondition(K key, TitanPredicate predicate, Object value) {
Preconditions.checkNotNull(key);
Preconditions.checkArgument(key instanceof String || key instanceof RelationType);
Preconditions.checkNotNull(predicate);
this.key = key;
this.predicate = predicate;
this.value = value;
}
private boolean satisfiesCondition(Object value) {
return predicate.evaluate(value, this.value);
}
@Override
public boolean evaluate(E element) {
RelationType type;
if (key instanceof String) {
type = ((InternalElement) element).tx().getRelationType((String) key);
if (type == null)
return satisfiesCondition(null);
} else {
type = (RelationType) key;
}
Preconditions.checkNotNull(type);
if (type.isPropertyKey()) {
Iterator<Object> iter = ElementHelper.getValues(element,(PropertyKey)type).iterator();
if (iter.hasNext()) {
while (iter.hasNext()) {
if (satisfiesCondition(iter.next()))
return true;
}
return false;
}
return satisfiesCondition(null);
} else {
assert ((InternalRelationType)type).getMultiplicity().isUnique(Direction.OUT);
return satisfiesCondition(((TitanRelation) element).getProperty((EdgeLabel) type));
}
}
public K getKey() {
return key;
}
public TitanPredicate getPredicate() {
return predicate;
}
public Object getValue() {
return value;
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(getType()).append(key).append(predicate).append(value).toHashCode();
}
@Override
public boolean equals(Object other) {
if (this == other)
return true;
if (other == null || !getClass().isInstance(other))
return false;
PredicateCondition oth = (PredicateCondition) other;
return key.equals(oth.key) && predicate.equals(oth.predicate) && compareValue(value, oth.value);
}
// ATLAS-2214: There's a issue when working with isNull, notNull operators
// When string/boolean attributes use the following sequence of filtering on AtlasVertex attributes then the code
// runs into NPE
// 1. boolean attr "x" != false/true | boolean attr "x" == false/true
// 2. boolean attr notNull 'query.has("x") | boolean attr isNull 'query.hasNot("x")'
// whereas if the sequence is reversed then the NPE is not encountered
// Similar behavior is exhibited for the string attributes
// Workaround is to allow null == null value comparision
private boolean compareValue(final Object left, final Object right) {
return left == null ? right == null : left.equals(right);
}
@Override
public String toString() {
return key + " " + predicate + " " + String.valueOf(value);
}
public static <K, E extends TitanElement> PredicateCondition<K, E> of(K key, TitanPredicate titanPredicate, Object condition) {
return new PredicateCondition<K, E>(key, titanPredicate, condition);
}
}
\ No newline at end of file
graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.graphdb.query.graph;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.*;
import com.thinkaurelius.titan.core.*;
import com.thinkaurelius.titan.core.attribute.Cmp;
import com.thinkaurelius.titan.core.schema.SchemaStatus;
import com.thinkaurelius.titan.core.schema.TitanSchemaType;
import com.thinkaurelius.titan.graphdb.database.IndexSerializer;
import com.thinkaurelius.titan.graphdb.internal.ElementCategory;
import com.thinkaurelius.titan.graphdb.internal.InternalRelationType;
import com.thinkaurelius.titan.graphdb.internal.OrderList;
import com.thinkaurelius.titan.graphdb.query.*;
import com.thinkaurelius.titan.graphdb.query.condition.*;
import com.thinkaurelius.titan.graphdb.transaction.StandardTitanTx;
import com.thinkaurelius.titan.graphdb.types.*;
import com.thinkaurelius.titan.graphdb.types.system.ImplicitKey;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.util.*;
/**
*
* Builds a {@link TitanGraphQuery}, optimizes the query and compiles the result into a {@link GraphCentricQuery} which
* is then executed through a {@link QueryProcessor}.
* This class from titan-0.5.4 has no major changes except adding a few logs for debugging index usage
*
* @author Matthias Broecheler (me@matthiasb.com)
*/
public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQueryBuilder> {
private static final Logger log = LoggerFactory.getLogger(GraphCentricQueryBuilder.class);
/**
* Transaction in which this query is executed.
*/
private final StandardTitanTx tx;
/**
* Serializer used to serialize the query conditions into backend queries.
*/
private final IndexSerializer serializer;
/**
* The constraints added to this query. None by default.
*/
private List<PredicateCondition<String, TitanElement>> constraints;
/**
* The order in which the elements should be returned. None by default.
*/
private OrderList orders = new OrderList();
/**
* The limit of this query. No limit by default.
*/
private int limit = Query.NO_LIMIT;
public GraphCentricQueryBuilder(StandardTitanTx tx, IndexSerializer serializer) {
log.debug("Loaded shaded version of class GraphCentricQueryBuilder");
Preconditions.checkNotNull(tx);
Preconditions.checkNotNull(serializer);
this.tx = tx;
this.serializer = serializer;
this.constraints = new ArrayList<>(5);
}
/* ---------------------------------------------------------------
* Query Construction
* ---------------------------------------------------------------
*/
private GraphCentricQueryBuilder has(String key, TitanPredicate predicate, Object condition) {
Preconditions.checkNotNull(key);
Preconditions.checkNotNull(predicate);
Preconditions.checkArgument(predicate.isValidCondition(condition), "Invalid condition: %s", condition);
constraints.add(new PredicateCondition<>(key, predicate, condition));
return this;
}
@Override
public GraphCentricQueryBuilder has(String key, com.tinkerpop.blueprints.Predicate predicate, Object condition) {
Preconditions.checkNotNull(key);
TitanPredicate titanPredicate = TitanPredicate.Converter.convert(predicate);
return has(key, titanPredicate, condition);
}
@Override
public GraphCentricQueryBuilder has(PropertyKey key, TitanPredicate predicate, Object condition) {
Preconditions.checkNotNull(key);
Preconditions.checkNotNull(predicate);
return has(key.getName(), predicate, condition);
}
@Override
public GraphCentricQueryBuilder has(String key) {
return has(key, Cmp.NOT_EQUAL, (Object) null);
}
@Override
public GraphCentricQueryBuilder hasNot(String key) {
return has(key, Cmp.EQUAL, (Object) null);
}
@Override
@Deprecated
public <T extends Comparable<T>> GraphCentricQueryBuilder has(String s, T t, Compare compare) {
return has(s, compare, t);
}
@Override
public GraphCentricQueryBuilder has(String key, Object value) {
return has(key, Cmp.EQUAL, value);
}
@Override
public GraphCentricQueryBuilder hasNot(String key, Object value) {
return has(key, Cmp.NOT_EQUAL, value);
}
@Override
public <T extends Comparable<?>> GraphCentricQueryBuilder interval(String s, T t1, T t2) {
has(s, Cmp.GREATER_THAN_EQUAL, t1);
return has(s, Cmp.LESS_THAN, t2);
}
@Override
public GraphCentricQueryBuilder limit(final int limit) {
Preconditions.checkArgument(limit >= 0, "Non-negative limit expected: %s", limit);
this.limit = limit;
return this;
}
@Override
public GraphCentricQueryBuilder orderBy(String key, Order order) {
Preconditions.checkArgument(tx.containsPropertyKey(key), "Provided key does not exist: %s", key);
return orderBy(tx.getPropertyKey(key), order);
}
@Override
public GraphCentricQueryBuilder orderBy(PropertyKey key, Order order) {
Preconditions.checkArgument(key!=null && order!=null, "Need to specify and key and an order");
Preconditions.checkArgument(Comparable.class.isAssignableFrom(key.getDataType()),
"Can only order on keys with comparable data type. [%s] has datatype [%s]", key.getName(), key.getDataType());
Preconditions.checkArgument(key.getCardinality()== Cardinality.SINGLE, "Ordering is undefined on multi-valued key [%s]", key.getName());
Preconditions.checkArgument(!orders.containsKey(key));
orders.add(key, order);
return this;
}
/* ---------------------------------------------------------------
* Query Execution
* ---------------------------------------------------------------
*/
@Override
public Iterable<Vertex> vertices() {
GraphCentricQuery query = constructQuery(ElementCategory.VERTEX);
return Iterables.filter(new QueryProcessor<>(query, tx.elementProcessor), Vertex.class);
}
@Override
public Iterable<Edge> edges() {
GraphCentricQuery query = constructQuery(ElementCategory.EDGE);
return Iterables.filter(new QueryProcessor<>(query, tx.elementProcessor), Edge.class);
}
@Override
public Iterable<TitanProperty> properties() {
GraphCentricQuery query = constructQuery(ElementCategory.PROPERTY);
return Iterables.filter(new QueryProcessor<>(query, tx.elementProcessor), TitanProperty.class);
}
private QueryDescription describe(ElementCategory category) {
return new StandardQueryDescription(1, constructQuery(category));
}
public QueryDescription describeForVertices() {
return describe(ElementCategory.VERTEX);
}
public QueryDescription describeForEdges() {
return describe(ElementCategory.EDGE);
}
public QueryDescription describeForProperties() {
return describe(ElementCategory.PROPERTY);
}
/* ---------------------------------------------------------------
* Query Construction
* ---------------------------------------------------------------
*/
private static final int DEFAULT_NO_LIMIT = 1000;
private static final int MAX_BASE_LIMIT = 20000;
private static final int HARD_MAX_LIMIT = 100000;
private static final double EQUAL_CONDITION_SCORE = 4;
private static final double OTHER_CONDITION_SCORE = 1;
private static final double ORDER_MATCH = 2;
private static final double ALREADY_MATCHED_ADJUSTOR = 0.1;
private static final double CARDINALITY_SINGE_SCORE = 1000;
private static final double CARDINALITY_OTHER_SCORE = 1000;
public GraphCentricQuery constructQuery(final ElementCategory resultType) {
Preconditions.checkNotNull(resultType);
if (limit == 0) return GraphCentricQuery.emptyQuery(resultType);
//Prepare constraints
And<TitanElement> conditions = QueryUtil.constraints2QNF(tx, constraints);
if (conditions == null) return GraphCentricQuery.emptyQuery(resultType);
//Prepare orders
orders.makeImmutable();
if (orders.isEmpty()) orders = OrderList.NO_ORDER;
//Compile all indexes that cover at least one of the query conditions
final Set<IndexType> indexCandidates = new HashSet<>();
ConditionUtil.traversal(conditions, new Predicate<Condition<TitanElement>>() {
@Override
public boolean apply(@Nullable Condition<TitanElement> condition) {
if (condition instanceof PredicateCondition) {
RelationType type = ((PredicateCondition<RelationType, TitanElement>)condition).getKey();
Preconditions.checkArgument(type!=null && type.isPropertyKey());
Iterables.addAll(indexCandidates, Iterables.filter(((InternalRelationType) type).getKeyIndexes(), new Predicate<IndexType>() {
@Override
public boolean apply(@Nullable IndexType indexType) {
return indexType.getElement()==resultType;
}
}));
}
return true;
}
});
/*
Determine the best join index query to answer this query:
Iterate over all potential indexes (as compiled above) and compute a score based on how many clauses
this index covers. The index with the highest score (as long as it covers at least one additional clause)
is picked and added to the joint query for as long as such exist.
*/
JointIndexQuery jointQuery = new JointIndexQuery();
boolean isSorted = orders.isEmpty();
Set<Condition> coveredClauses = Sets.newHashSet();
while (true) {
IndexType bestCandidate = null;
double candidateScore = 0.0;
Set<Condition> candidateSubcover = null;
boolean candidateSupportsSort = false;
Object candidateSubcondition = null;
for (IndexType index : indexCandidates) {
Set<Condition> subcover = Sets.newHashSet();
Object subcondition;
boolean supportsSort = orders.isEmpty();
//Check that this index actually applies in case of a schema constraint
if (index.hasSchemaTypeConstraint()) {
TitanSchemaType type = index.getSchemaTypeConstraint();
Map.Entry<Condition, Collection<Object>> equalCon = getEqualityConditionValues(conditions, ImplicitKey.LABEL);
if (equalCon==null) continue;
Collection<Object> labels = equalCon.getValue();
assert labels.size()>=1;
if (labels.size()>1) {
log.warn("The query optimizer currently does not support multiple label constraints in query: {}", this);
continue;
}
if (!type.getName().equals(Iterables.getOnlyElement(labels))) continue;
subcover.add(equalCon.getKey());
}
if (index.isCompositeIndex()) {
CompositeIndexType compositeIndex = (CompositeIndexType)index;
subcondition = indexCover(compositeIndex, conditions, subcover);
// if this is unique index, use it!!
if (compositeIndex.getCardinality() == Cardinality.SINGLE && subcondition != null) {
bestCandidate = null; // will cause the outer while() to bail out
candidateSubcover = subcover;
candidateSubcondition = subcondition;
candidateSupportsSort = supportsSort;
if (log.isDebugEnabled()) {
log.debug("selected unique index {}", compositeIndex.getName());
}
if (coveredClauses.isEmpty()) {
isSorted = candidateSupportsSort;
}
coveredClauses.clear();;
coveredClauses.addAll(candidateSubcover);
jointQuery = new JointIndexQuery();
jointQuery.add(compositeIndex, serializer.getQuery(compositeIndex, (List<Object[]>)candidateSubcondition));
break;
}
} else {
subcondition = indexCover((MixedIndexType) index, conditions, serializer, subcover);
if (coveredClauses.isEmpty() && !supportsSort
&& indexCoversOrder((MixedIndexType)index, orders)) supportsSort=true;
}
if (subcondition==null) continue;
assert !subcover.isEmpty();
double score = 0.0;
boolean coversAdditionalClause = false;
for (Condition c : subcover) {
double s = (c instanceof PredicateCondition && ((PredicateCondition)c).getPredicate()==Cmp.EQUAL)?
EQUAL_CONDITION_SCORE:OTHER_CONDITION_SCORE;
if (coveredClauses.contains(c)) s=s*ALREADY_MATCHED_ADJUSTOR;
else coversAdditionalClause = true;
score+=s;
if (index.isCompositeIndex())
score+=((CompositeIndexType)index).getCardinality()==Cardinality.SINGLE?
CARDINALITY_SINGE_SCORE:CARDINALITY_OTHER_SCORE;
}
if (supportsSort) score+=ORDER_MATCH;
if (coversAdditionalClause && score>candidateScore) {
candidateScore=score;
bestCandidate=index;
candidateSubcover = subcover;
candidateSubcondition = subcondition;
candidateSupportsSort = supportsSort;
}
}
if (bestCandidate!=null) {
if (coveredClauses.isEmpty()) isSorted=candidateSupportsSort;
coveredClauses.addAll(candidateSubcover);
log.debug("Index chosen for query {} {} " , bestCandidate.isCompositeIndex() ? "COMPOSITE" : "MIXED", coveredClauses);
if (bestCandidate.isCompositeIndex()) {
jointQuery.add((CompositeIndexType)bestCandidate,
serializer.getQuery((CompositeIndexType)bestCandidate, (List<Object[]>)candidateSubcondition));
} else {
jointQuery.add((MixedIndexType)bestCandidate,
serializer.getQuery((MixedIndexType)bestCandidate, (Condition)candidateSubcondition, orders));
}
} else {
break;
}
/* TODO: smarter optimization:
- use in-memory histograms to estimate selectivity of PredicateConditions and filter out low-selectivity ones
if they would result in an individual index call (better to filter afterwards in memory)
- move OR's up and extend GraphCentricQuery to allow multiple JointIndexQuery for proper or'ing of queries
*/
}
BackendQueryHolder<JointIndexQuery> query;
if (!coveredClauses.isEmpty()) {
int indexLimit = limit == Query.NO_LIMIT ? HARD_MAX_LIMIT : limit;
if (tx.getGraph().getConfiguration().adjustQueryLimit()) {
indexLimit = limit == Query.NO_LIMIT ? DEFAULT_NO_LIMIT : Math.min(MAX_BASE_LIMIT, limit);
}
indexLimit = Math.min(HARD_MAX_LIMIT, QueryUtil.adjustLimitForTxModifications(tx, coveredClauses.size(), indexLimit));
jointQuery.setLimit(indexLimit);
query = new BackendQueryHolder<>(jointQuery, coveredClauses.size() == conditions.numChildren(), isSorted, null);
} else {
query = new BackendQueryHolder<>(new JointIndexQuery(), false, isSorted, null);
}
return new GraphCentricQuery(resultType, conditions, orders, query, limit);
}
public static final boolean indexCoversOrder(MixedIndexType index, OrderList orders) {
for (int i = 0; i < orders.size(); i++) {
if (!index.indexesKey(orders.getKey(i))) return false;
}
return true;
}
public static List<Object[]> indexCover(final CompositeIndexType index, Condition<TitanElement> condition, Set<Condition> covered) {
assert QueryUtil.isQueryNormalForm(condition);
assert condition instanceof And;
if (index.getStatus()!= SchemaStatus.ENABLED) return null;
IndexField[] fields = index.getFieldKeys();
Object[] indexValues = new Object[fields.length];
Set<Condition> coveredClauses = new HashSet<>(fields.length);
List<Object[]> indexCovers = new ArrayList<>(4);
constructIndexCover(indexValues, 0, fields, condition, indexCovers, coveredClauses);
if (!indexCovers.isEmpty()) {
covered.addAll(coveredClauses);
return indexCovers;
} else return null;
}
private static void constructIndexCover(Object[] indexValues, int position, IndexField[] fields,
Condition<TitanElement> condition,
List<Object[]> indexCovers, Set<Condition> coveredClauses) {
if (position>=fields.length) {
indexCovers.add(indexValues);
} else {
IndexField field = fields[position];
Map.Entry<Condition, Collection<Object>> equalCon = getEqualityConditionValues(condition, field.getFieldKey());
if (equalCon != null) {
coveredClauses.add(equalCon.getKey());
assert equalCon.getValue().size()>0;
for (Object value : equalCon.getValue()) {
Object[] newValues = Arrays.copyOf(indexValues, fields.length);
newValues[position]=value;
constructIndexCover(newValues, position+1, fields, condition, indexCovers, coveredClauses);
}
}
}
}
private static final Map.Entry<Condition, Collection<Object>> getEqualityConditionValues(Condition<TitanElement> condition, RelationType type) {
for (Condition c : condition.getChildren()) {
if (c instanceof Or) {
Map.Entry<RelationType, Collection> orEqual = QueryUtil.extractOrCondition((Or)c);
if (orEqual!=null && orEqual.getKey().equals(type) && !orEqual.getValue().isEmpty()) {
return new AbstractMap.SimpleImmutableEntry(c, orEqual.getValue());
}
} else if (c instanceof PredicateCondition) {
PredicateCondition<RelationType, TitanRelation> atom = (PredicateCondition)c;
if (atom.getKey().equals(type) && atom.getPredicate()==Cmp.EQUAL && atom.getValue()!=null) {
return new AbstractMap.SimpleImmutableEntry(c, ImmutableList.of(atom.getValue()));
}
}
}
return null;
}
public static final Condition<TitanElement> indexCover(final MixedIndexType index, Condition<TitanElement> condition,
final IndexSerializer indexInfo, final Set<Condition> covered) {
assert QueryUtil.isQueryNormalForm(condition);
assert condition instanceof And;
And<TitanElement> subcondition = new And<>(condition.numChildren());
for (Condition<TitanElement> subclause : condition.getChildren()) {
if (coversAll(index, subclause, indexInfo)) {
subcondition.add(subclause);
covered.add(subclause);
}
}
return subcondition.isEmpty()?null:subcondition;
}
private static final boolean coversAll(final MixedIndexType index, Condition<TitanElement> condition, IndexSerializer indexInfo) {
if (condition.getType()==Condition.Type.LITERAL) {
if (!(condition instanceof PredicateCondition)) return false;
PredicateCondition<RelationType, TitanElement> atom = (PredicateCondition) condition;
if (atom.getValue()==null) return false;
Preconditions.checkArgument(atom.getKey().isPropertyKey());
PropertyKey key = (PropertyKey) atom.getKey();
ParameterIndexField[] fields = index.getFieldKeys();
ParameterIndexField match = null;
for (ParameterIndexField field : fields) {
if (field.getStatus() != SchemaStatus.ENABLED) continue;
if (field.getFieldKey().equals(key)) match = field;
}
if (match==null) return false;
return indexInfo.supports(index, match, atom.getPredicate());
} else {
for (Condition<TitanElement> child : condition.getChildren()) {
if (!coversAll(index, child, indexInfo)) return false;
}
return true;
}
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import com.thinkaurelius.titan.core.EdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
/**
* Factory that serves up instances of graph database abstraction layer classes
* that correspond to Titan/Tinkerpop classes.
*/
public final class GraphDbObjectFactory {
private GraphDbObjectFactory() {
}
/**
* Creates a Titan0Edge that corresponds to the given Gremlin Edge.
*
* @param graph The graph the edge should be created in
* @param source The gremlin edge
*/
public static Titan0Edge createEdge(Titan0Graph graph, Edge source) {
if (source == null) {
return null;
}
return new Titan0Edge(graph, source);
}
/**
* Creates a Titan0GraphQuery that corresponds to the given GraphQuery.
*
* @param graph the graph that is being quried
*/
public static Titan0GraphQuery createQuery(Titan0Graph graph) {
return new Titan0GraphQuery(graph);
}
/**
* Creates a Titan0Vertex that corresponds to the given Gremlin Vertex.
*
* @param graph The graph that contains the vertex
* @param source the Gremlin vertex
*/
public static Titan0Vertex createVertex(Titan0Graph graph, Vertex source) {
if (source == null) {
return null;
}
return new Titan0Vertex(graph, source);
}
/**
* @param propertyKey The Gremlin propertyKey.
*
*/
public static Titan0PropertyKey createPropertyKey(PropertyKey propertyKey) {
if (propertyKey == null) {
return null;
}
return new Titan0PropertyKey(propertyKey);
}
/**
* @param label The label.
*
*/
public static Titan0EdgeLabel createEdgeLabel(EdgeLabel label) {
if (label == null) {
return null;
}
return new Titan0EdgeLabel(label);
}
/**
* @param index The gremlin index.
* @return
*/
public static AtlasGraphIndex createGraphIndex(TitanGraphIndex index) {
if (index == null) {
return null;
}
return new Titan0GraphIndex(index);
}
/**
* Converts a Multiplicity to a Cardinality.
*
* @param cardinality
* @return
*/
public static AtlasCardinality createCardinality(Cardinality cardinality) {
if (cardinality == Cardinality.SINGLE) {
return AtlasCardinality.SINGLE;
} else if (cardinality == Cardinality.LIST) {
return AtlasCardinality.LIST;
}
return AtlasCardinality.SET;
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Edge.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
/**
* Titan 0.5.4 implementation of AtlasEdge.
*/
public class Titan0Edge extends Titan0Element<Edge> implements AtlasEdge<Titan0Vertex, Titan0Edge> {
public Titan0Edge(Titan0Graph graph, Edge edge) {
super(graph, edge);
}
@Override
public String getLabel() {
return wrappedElement.getLabel();
}
@Override
public Titan0Edge getE() {
return this;
}
@Override
public AtlasVertex<Titan0Vertex, Titan0Edge> getInVertex() {
Vertex v = wrappedElement.getVertex(Direction.IN);
return GraphDbObjectFactory.createVertex(graph, v);
}
@Override
public AtlasVertex<Titan0Vertex, Titan0Edge> getOutVertex() {
Vertex v = wrappedElement.getVertex(Direction.OUT);
return GraphDbObjectFactory.createVertex(graph, v);
}
@Override
public String toString() {
return "Titan0Edge [id=" + getId() + "]";
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0EdgeLabel.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import com.thinkaurelius.titan.core.EdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
/**
* Titan 0.5.4 implementaiton of AtlasEdgeLabel.
*/
public class Titan0EdgeLabel implements AtlasEdgeLabel {
private final EdgeLabel wrappedEdgeLabel;
public Titan0EdgeLabel(EdgeLabel toWrap) {
wrappedEdgeLabel = toWrap;
}
/*
* (non-Javadoc)
*
* @see org.apache.atlas.repository.graphdb.AtlasEdgeLabel#getName()
*/
@Override
public String getName() {
return wrappedEdgeLabel.getName();
}
@Override
public boolean equals(Object other) {
if (!(other instanceof Titan0EdgeLabel)) {
return false;
}
Titan0EdgeLabel otherLabel = (Titan0EdgeLabel) other;
return wrappedEdgeLabel.equals(otherLabel.wrappedEdgeLabel);
}
@Override
public int hashCode() {
int result = 17;
result = 37 * result + wrappedEdgeLabel.hashCode();
return result;
}
@Override
public String toString() {
return wrappedEdgeLabel.getName();
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Element.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import java.lang.Override;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanElement;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
/**
* Titan 0.5.4 implementation of AtlasElement.
*/
public class Titan0Element<T extends Element> implements AtlasElement {
protected Titan0Graph graph;
protected T wrappedElement;
public Titan0Element(Titan0Graph graph, T element) {
wrappedElement = element;
this.graph = graph;
}
@Override
public Object getId() {
return wrappedElement.getId();
}
@Override
public Set<String> getPropertyKeys() {
return wrappedElement.getPropertyKeys();
}
@Override
public <U> void setProperty(String propertyName, U value) {
try {
wrappedElement.setProperty(propertyName, value);
} catch (SchemaViolationException e) {
throw new AtlasSchemaViolationException(e);
}
}
@Override
public <U> U getProperty(String propertyName, Class<U> clazz) {
Object rawValue = wrappedElement.getProperty(propertyName);
if (rawValue == null) {
return null;
}
if (AtlasEdge.class == clazz) {
return (U)graph.getEdge(rawValue.toString());
}
if (AtlasVertex.class == clazz) {
return (U)graph.getVertex(rawValue.toString());
}
return (U)rawValue;
}
/**
* Gets all of the values of the given property.
* @param propertyName
* @return
*/
@Override
public <T> Collection<T> getPropertyValues(String propertyName, Class<T> type) {
return Collections.singleton(getProperty(propertyName, type));
}
@Override
public void removeProperty(String propertyName) {
wrappedElement.removeProperty(propertyName);
}
@Override
public JSONObject toJson(Set<String> propertyKeys) throws JSONException {
return GraphSONUtility.jsonFromElement(wrappedElement, propertyKeys, GraphSONMode.NORMAL);
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasElement#getListProperty(java.
* lang.String)
*/
@Override
public List<String> getListProperty(String propertyName) {
return getProperty(propertyName, List.class);
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasElement#setListProperty(java.
* lang.String, java.util.List)
*/
@Override
public void setListProperty(String propertyName, List<String> values) {
setProperty(propertyName, values);
}
@Override
public T getWrappedElement() {
return wrappedElement;
}
@Override
public int hashCode() {
int result = 37;
result = 17 * result + getClass().hashCode();
result = 17 * result + getWrappedElement().hashCode();
return result;
}
@Override
public boolean equals(Object other) {
if (other == null) {
return false;
}
if (other.getClass() != getClass()) {
return false;
}
Titan0Element otherElement = (Titan0Element) other;
return getWrappedElement().equals(otherElement.getWrappedElement());
}
/*
* (non-Javadoc)
*
* @see org.apache.atlas.repository.graphdb.AtlasElement#exists()
*/
@Override
public boolean exists() {
try {
return !((TitanElement)wrappedElement).isRemoved();
} catch(IllegalStateException e) {
return false;
}
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasElement#setJsonProperty(java.
* lang.String, java.lang.Object)
*/
@Override
public <T> void setJsonProperty(String propertyName, T value) {
setProperty(propertyName, value);
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasElement#getJsonProperty(java.
* lang.String)
*/
@Override
public <T> T getJsonProperty(String propertyName) {
return (T) getProperty(propertyName, String.class);
}
@Override
public String getIdForDisplay() {
return getId().toString();
}
@Override
public <V> List<V> getListProperty(String propertyName, Class<V> elementType) {
List<String> value = getListProperty(propertyName);
if (value == null) {
return null;
}
if (AtlasEdge.class == elementType) {
return (List<V>)Lists.transform(value, new Function<String, AtlasEdge>(){
@Override
public AtlasEdge apply(String input) {
return graph.getEdge(input);
}
});
}
if (AtlasVertex.class == elementType) {
return (List<V>)Lists.transform(value, new Function<String, AtlasVertex>(){
@Override
public AtlasVertex apply(String input) {
return graph.getVertex(input);
}
});
}
return (List<V>)value;
}
@Override
public void setPropertyFromElementsIds(String propertyName, List<AtlasElement> values) {
List<String> propertyValue = new ArrayList<>(values.size());
for(AtlasElement element: values) {
propertyValue.add(element.getId().toString());
}
setProperty(propertyName, propertyValue);
}
@Override
public void setPropertyFromElementId(String propertyName, AtlasElement value) {
setProperty(propertyName, value.getId().toString());
}
@Override
public boolean isIdAssigned() {
return true;
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
import com.tinkerpop.pipes.util.structures.Row;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.model.impexp.MigrationStatus;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.atlas.type.AtlasType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.script.Bindings;
import javax.script.ScriptContext;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Titan 0.5.4 implementation of AtlasGraph.
*/
public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
private static final Logger LOG = LoggerFactory.getLogger(Titan0Graph.class);
private final Set<String> multiProperties;
public Titan0Graph() {
//determine multi-properties once at startup
TitanManagement mgmt = null;
try {
mgmt = Titan0GraphDatabase.getGraphInstance().getManagementSystem();
Iterable<PropertyKey> keys = mgmt.getRelationTypes(PropertyKey.class);
multiProperties = Collections.synchronizedSet(new HashSet<String>());
for(PropertyKey key : keys) {
if (key.getCardinality() != Cardinality.SINGLE) {
multiProperties.add(key.getName());
}
}
} finally {
if (mgmt != null) {
mgmt.rollback();
}
}
}
@Override
public AtlasEdge<Titan0Vertex, Titan0Edge> addEdge(AtlasVertex<Titan0Vertex, Titan0Edge> outVertex,
AtlasVertex<Titan0Vertex, Titan0Edge> inVertex, String edgeLabel) {
try {
Edge edge = getGraph().addEdge(null, outVertex.getV().getWrappedElement(),
inVertex.getV().getWrappedElement(), edgeLabel);
return GraphDbObjectFactory.createEdge(this, edge);
} catch (SchemaViolationException e) {
throw new AtlasSchemaViolationException(e);
}
}
@Override
public AtlasGraphQuery<Titan0Vertex, Titan0Edge> query() {
return new Titan0GraphQuery(this);
}
@Override
public AtlasEdge<Titan0Vertex, Titan0Edge> getEdge(String edgeId) {
Edge edge = getGraph().getEdge(edgeId);
return GraphDbObjectFactory.createEdge(this, edge);
}
@Override
public void removeEdge(AtlasEdge<Titan0Vertex, Titan0Edge> edge) {
getGraph().removeEdge(edge.getE().getWrappedElement());
}
@Override
public void removeVertex(AtlasVertex<Titan0Vertex, Titan0Edge> vertex) {
getGraph().removeVertex(vertex.getV().getWrappedElement());
}
@Override
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> getEdges() {
Iterable<Edge> edges = getGraph().getEdges();
return wrapEdges(edges);
}
@Override
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> getVertices() {
Iterable<Vertex> vertices = getGraph().getVertices();
return wrapVertices(vertices);
}
@Override
public AtlasVertex<Titan0Vertex, Titan0Edge> addVertex() {
Vertex result = getGraph().addVertex(null);
return GraphDbObjectFactory.createVertex(this, result);
}
@Override
public void commit() {
getGraph().commit();
}
@Override
public void rollback() {
getGraph().rollback();
}
@Override
public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex, String graphQuery) {
return indexQuery(fulltextIndex, graphQuery, 0);
}
@Override
public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex, String graphQuery, int offset) {
TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).offset(offset);
return new Titan0IndexQuery(this, query);
}
@Override
public AtlasGraphManagement getManagementSystem() {
return new Titan0GraphManagement(this, getGraph().getManagementSystem());
}
@Override
public void shutdown() {
getGraph().shutdown();
}
@Override
public Set<String> getVertexIndexKeys() {
return getIndexKeys(Vertex.class);
}
@Override
public Set<String> getEdgeIndexKeys() {
return getIndexKeys(Edge.class);
}
private Set<String> getIndexKeys(Class<? extends Element> titanClass) {
return getGraph().getIndexedKeys(titanClass);
}
@Override
public AtlasVertex<Titan0Vertex, Titan0Edge> getVertex(String vertexId) {
Vertex v = getGraph().getVertex(vertexId);
return GraphDbObjectFactory.createVertex(this, v);
}
@Override
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> getVertices(String key, Object value) {
Iterable<Vertex> result = getGraph().getVertices(key, value);
return wrapVertices(result);
}
private Object convertGremlinValue(Object rawValue) {
if (rawValue instanceof Vertex) {
return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue);
} else if (rawValue instanceof Edge) {
return GraphDbObjectFactory.createEdge(this, (Edge) rawValue);
} else if (rawValue instanceof Row) {
Row rowValue = (Row)rawValue;
Map<String, Object> result = new HashMap<>(rowValue.size());
List<String> columnNames = rowValue.getColumnNames();
for(int i = 0; i < rowValue.size(); i++) {
String key = columnNames.get(i);
Object value = convertGremlinValue(rowValue.get(i));
result.put(key, value);
}
return result;
} else if (rawValue instanceof List) {
return Lists.transform((List)rawValue, new Function<Object, Object>() {
@Override
public Object apply(Object input) {
return convertGremlinValue(input);
}
});
} else if (rawValue instanceof Collection) {
throw new UnsupportedOperationException("Unhandled collection type: " + rawValue.getClass());
}
return rawValue;
}
@Override
public GremlinVersion getSupportedGremlinVersion() {
return GremlinVersion.TWO;
}
private List<Object> convertPathQueryResultToList(Object rawValue) {
return (List<Object>) rawValue;
}
@Override
public void clear() {
TitanGraph graph = getGraph();
if (graph.isOpen()) {
// only a shut down graph can be cleared
graph.shutdown();
}
TitanCleanup.clear(graph);
}
private TitanGraph getGraph() {
// return the singleton instance of the graph in the plugin
return Titan0GraphDatabase.getGraphInstance();
}
@Override
public void exportToGson(OutputStream os) throws IOException {
GraphSONWriter.outputGraph(getGraph(), os);
}
@Override
public Object executeGremlinScript(String query, boolean isPath) throws AtlasBaseException {
Object result = executeGremlinScript(query);
return convertGremlinScriptResult(isPath, result);
}
private Object convertGremlinScriptResult(boolean isPath, Object result) {
if (isPath) {
List<Object> path = convertPathQueryResultToList(result);
List<Object> convertedResult = new ArrayList<>(path.size());
for(Object o : path) {
convertedResult.add(convertGremlinValue(o));
}
return convertedResult;
} else {
return convertGremlinValue(result);
}
}
@Override
public ScriptEngine getGremlinScriptEngine() throws AtlasBaseException {
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
if (engine == null) {
throw new AtlasBaseException(AtlasErrorCode.FAILED_TO_OBTAIN_GREMLIN_SCRIPT_ENGINE, "gremlin-groovy");
}
//Do not cache script compilations due to memory implications
engine.getContext().setAttribute("#jsr223.groovy.engine.keep.globals", "phantom", ScriptContext.ENGINE_SCOPE);
return engine;
}
@Override
public void releaseGremlinScriptEngine(ScriptEngine scriptEngine) {
// no action needed
}
@Override
public Object executeGremlinScript(ScriptEngine scriptEngine, Map<? extends String, ? extends Object> userBindings, String query, boolean isPath) throws ScriptException {
if (LOG.isDebugEnabled()) {
LOG.debug("executeGremlinScript(query={}, userBindings={})", query, userBindings);
}
Bindings bindings = scriptEngine.createBindings();
if (userBindings != null) {
bindings.putAll(userBindings);
}
bindings.put("g", getGraph());
Object result = scriptEngine.eval(query, bindings);
return convertGremlinScriptResult(isPath, result);
}
private Object executeGremlinScript(String gremlinQuery) throws AtlasBaseException {
Object result = null;
ScriptEngine engine = getGremlinScriptEngine();
try {
Bindings bindings = engine.createBindings();
bindings.put("g", getGraph());
result = engine.eval(gremlinQuery, bindings);
} catch (ScriptException e) {
throw new AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, gremlinQuery);
} finally {
releaseGremlinScriptEngine(engine);
}
return result;
}
@Override
public GroovyExpression generatePersisentToLogicalConversionExpression(GroovyExpression expr, AtlasType type) {
//nothing special needed, value is stored in required type
return expr;
}
@Override
public boolean isPropertyValueConversionNeeded(AtlasType type) {
return false;
}
@Override
public boolean requiresInitialIndexedPredicate() {
return false;
}
@Override
public GroovyExpression getInitialIndexedPredicate(GroovyExpression expr) {
return expr;
}
@Override
public GroovyExpression addOutputTransformationPredicate(GroovyExpression expr, boolean inSelect, boolean isPath) {
return expr;
}
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> wrapEdges(Iterator<Edge> it) {
Iterable<Edge> iterable = new IteratorToIterableAdapter<>(it);
return wrapEdges(iterable);
}
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> wrapVertices(Iterator<Vertex> it) {
Iterable<Vertex> iterable = new IteratorToIterableAdapter<>(it);
return wrapVertices(iterable);
}
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> wrapVertices(Iterable<Vertex> it) {
return Iterables.transform(it, new Function<Vertex, AtlasVertex<Titan0Vertex, Titan0Edge>>(){
@Override
public AtlasVertex<Titan0Vertex, Titan0Edge> apply(Vertex input) {
return GraphDbObjectFactory.createVertex(Titan0Graph.this, input);
}
});
}
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> wrapEdges(Iterable<Edge> it) {
Iterable<Edge> result = it;
return Iterables.transform(result, new Function<Edge, AtlasEdge<Titan0Vertex, Titan0Edge>>(){
@Override
public AtlasEdge<Titan0Vertex, Titan0Edge> apply(Edge input) {
return GraphDbObjectFactory.createEdge(Titan0Graph.this, input);
}
});
}
@Override
public boolean isMultiProperty(String propertyName) {
return multiProperties.contains(propertyName);
}
@Override
public void loadLegacyGraphSON(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
}
@Override
public MigrationStatus getMigrationStatus() {
return new MigrationStatus();
}
public void addMultiProperties(Set<String> names) {
multiProperties.addAll(names);
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphDatabase.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.HashMap;
import java.util.Map;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.thinkaurelius.titan.diskstorage.StandardIndexProvider;
import com.thinkaurelius.titan.diskstorage.solr.Solr5Index;
/**
* Titan 0.5.4 implementation of GraphDatabase.
*/
public class Titan0GraphDatabase implements GraphDatabase<Titan0Vertex, Titan0Edge> {
private static final Logger LOG = LoggerFactory.getLogger(Titan0GraphDatabase.class);
/**
* Constant for the configuration property that indicates the prefix.
*/
public static final String GRAPH_PREFIX = "atlas.graph";
public static final String INDEX_BACKEND_CONF = "index.search.backend";
public static final String INDEX_BACKEND_LUCENE = "lucene";
public static final String INDEX_BACKEND_ES = "elasticsearch";
private static volatile Titan0Graph atlasGraphInstance = null;
private static volatile TitanGraph graphInstance = null;
public static Configuration getConfiguration() throws AtlasException {
Configuration configProperties = ApplicationProperties.get();
return ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
}
static {
addSolr5Index();
}
/**
* Titan loads index backend name to implementation using
* StandardIndexProvider.ALL_MANAGER_CLASSES But
* StandardIndexProvider.ALL_MANAGER_CLASSES is a private static final
* ImmutableMap Only way to inject Solr5Index is to modify this field. So,
* using hacky reflection to add Sol5Index
*/
private static void addSolr5Index() {
try {
Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES");
field.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
Map<String, String> customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses());
customMap.put("solr", Solr5Index.class.getName()); // for
// consistency
// with Titan
// 1.0.0
customMap.put("solr5", Solr5Index.class.getName()); // for backward
// compatibility
ImmutableMap<String, String> immap = ImmutableMap.copyOf(customMap);
field.set(null, immap);
LOG.debug("Injected solr5 index - {}", Solr5Index.class.getName());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static TitanGraph getGraphInstance() {
if (graphInstance == null) {
synchronized (Titan0GraphDatabase.class) {
if (graphInstance == null) {
Configuration config;
try {
config = getConfiguration();
} catch (AtlasException e) {
throw new RuntimeException(e);
}
graphInstance = TitanFactory.open(config);
atlasGraphInstance = new Titan0Graph();
validateIndexBackend(config);
}
}
}
return graphInstance;
}
public static void unload() {
synchronized (Titan0GraphDatabase.class) {
if (graphInstance == null) {
return;
}
graphInstance.commit();
//shutdown invalidates the graph instance
graphInstance.shutdown();
graphInstance = null;
}
}
static void validateIndexBackend(Configuration config) {
String configuredIndexBackend = config.getString(INDEX_BACKEND_CONF);
TitanManagement managementSystem = null;
try {
managementSystem = getGraphInstance().getManagementSystem();
String currentIndexBackend = managementSystem.get(INDEX_BACKEND_CONF);
if (!equals(configuredIndexBackend, currentIndexBackend)) {
throw new RuntimeException("Configured Index Backend " + configuredIndexBackend
+ " differs from earlier configured Index Backend " + currentIndexBackend + ". Aborting!");
}
} finally {
if (managementSystem != null) {
managementSystem.commit();
}
}
}
private static boolean equals(Object o1, Object o2) {
if (o1 == null) {
return o2 == null;
}
return o1.equals(o2);
}
@Override
public AtlasGraph<Titan0Vertex, Titan0Edge> getGraph() {
// force graph loading up front to avoid bootstrapping
// issues
getGraphInstance();
return atlasGraphInstance;
}
@Override
public boolean isGraphLoaded() {
return graphInstance != null;
}
@Override
public void initializeTestGraph() {
//nothing to do
}
@Override
public void cleanup() {
try {
getGraphInstance().shutdown();
} catch(Throwable t) {
LOG.warn("Could not shutdown test TitanGraph", t);
t.printStackTrace();
}
try {
TitanCleanup.clear(getGraphInstance());
} catch(Throwable t) {
LOG.warn("Could not clear test TitanGraph", t);
t.printStackTrace();
}
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphIndex.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import java.util.HashSet;
import java.util.Set;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
/**
* Titan 0.5.4 implementation of AtlasGraphIndex.
*/
public class Titan0GraphIndex implements AtlasGraphIndex {
private TitanGraphIndex wrappedIndex;
public Titan0GraphIndex(TitanGraphIndex toWrap) {
wrappedIndex = toWrap;
}
/* (non-Javadoc)
* @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isMixedIndex()
*/
@Override
public boolean isMixedIndex() {
return wrappedIndex.isMixedIndex();
}
/* (non-Javadoc)
* @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isEdgeIndex()
*/
@Override
public boolean isEdgeIndex() {
return Edge.class.isAssignableFrom(wrappedIndex.getIndexedElement());
}
/* (non-Javadoc)
* @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isVertexIndex()
*/
@Override
public boolean isVertexIndex() {
return Vertex.class.isAssignableFrom(wrappedIndex.getIndexedElement());
}
/* (non-Javadoc)
* @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isCompositeIndex()
*/
@Override
public boolean isCompositeIndex() {
return wrappedIndex.isCompositeIndex();
}
/* (non-Javadoc)
* @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isUnique()
*/
@Override
public boolean isUnique() {
return wrappedIndex.isUnique();
}
/* (non-Javadoc)
* @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#getFieldKeys()
*/
@Override
public Set<AtlasPropertyKey> getFieldKeys() {
PropertyKey[] keys = wrappedIndex.getFieldKeys();
Set<AtlasPropertyKey> result = new HashSet<>();
for(PropertyKey key : keys) {
result.add(GraphDbObjectFactory.createPropertyKey(key));
}
return result;
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.PropertyKeyMaker;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* Titan 0.5.4 implementation of AtlasGraphManagement.
*/
public class Titan0GraphManagement implements AtlasGraphManagement {
private static final Logger LOG = LoggerFactory.getLogger(Titan0GraphManagement.class);
private final Titan0Graph graph;
private final TitanManagement management;
private Set<String> newMultProperties = new HashSet<>();
public Titan0GraphManagement(Titan0Graph graph, TitanManagement managementSystem) {
this.graph = graph;
management = managementSystem;
}
@Override
public void createEdgeMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
}
@Override
public void createEdgeIndex(String label, String indexName, AtlasEdgeDirection edgeDirection, List<AtlasPropertyKey> propertyKeys) {
EdgeLabel edgeLabel = management.getEdgeLabel(label);
if (edgeLabel == null) {
edgeLabel = management.makeEdgeLabel(label).make();
}
Direction direction = TitanObjectFactory.createDirection(edgeDirection);
PropertyKey[] keys = TitanObjectFactory.createPropertyKeys(propertyKeys);
management.buildEdgeIndex(edgeLabel, indexName, direction, keys);
}
@Override
public void createFullTextMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
}
private void buildMixedIndex(String index, Class<? extends Element> titanClass, String backingIndex) {
management.buildIndex(index, titanClass).buildMixedIndex(backingIndex);
}
@Override
public boolean containsPropertyKey(String propertyKey) {
return management.containsPropertyKey(propertyKey);
}
@Override
public void rollback() {
management.rollback();
}
@Override
public void commit() {
graph.addMultiProperties(newMultProperties);
newMultProperties.clear();
management.commit();
}
@Override
public AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality) {
if (cardinality.isMany()) {
newMultProperties.add(propertyName);
}
PropertyKeyMaker propertyKeyBuilder = management.makePropertyKey(propertyName).dataType(propertyClass);
if (cardinality != null) {
Cardinality titanCardinality = TitanObjectFactory.createCardinality(cardinality);
propertyKeyBuilder.cardinality(titanCardinality);
}
PropertyKey propertyKey = propertyKeyBuilder.make();
return GraphDbObjectFactory.createPropertyKey(propertyKey);
}
@Override
public AtlasEdgeLabel makeEdgeLabel(String label) {
EdgeLabel edgeLabel = management.makeEdgeLabel(label).make();
return GraphDbObjectFactory.createEdgeLabel(edgeLabel);
}
@Override
public void deletePropertyKey(String propertyKey) {
PropertyKey titanPropertyKey = management.getPropertyKey(propertyKey);
if (null == titanPropertyKey) return;
for (int i = 0;; i++) {
String deletedKeyName = titanPropertyKey + "_deleted_" + i;
if (null == management.getPropertyKey(deletedKeyName)) {
management.changeName(titanPropertyKey, deletedKeyName);
break;
}
}
}
@Override
public AtlasPropertyKey getPropertyKey(String propertyName) {
return GraphDbObjectFactory.createPropertyKey(management.getPropertyKey(propertyName));
}
@Override
public AtlasEdgeLabel getEdgeLabel(String label) {
return GraphDbObjectFactory.createEdgeLabel(management.getEdgeLabel(label));
}
@Override
public void createVertexCompositeIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for(AtlasPropertyKey key : propertyKeys) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
indexBuilder.addKey(titanKey);
}
if (enforceUniqueness) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
@Override
public void createEdgeCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Edge.class);
for(AtlasPropertyKey key : propertyKeys) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
indexBuilder.addKey(titanKey);
}
if (isUnique) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
@Override
public void createVertexMixedIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for(AtlasPropertyKey key : propertyKeys) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
indexBuilder.addKey(titanKey);
}
indexBuilder.buildMixedIndex(backingIndex);
}
@Override
public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
TitanGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, titanKey);
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraphManagement#getGraphIndex(
* java.lang.String)
*/
@Override
public AtlasGraphIndex getGraphIndex(String indexName) {
TitanGraphIndex index = management.getGraphIndex(indexName);
return GraphDbObjectFactory.createGraphIndex(index);
}
@Override
public boolean edgeIndexExist(String label, String indexName) {
EdgeLabel edgeLabel = management.getEdgeLabel(label);
return edgeLabel != null && management.getRelationIndex(edgeLabel, indexName) != null;
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import java.util.Iterator;
import com.google.common.base.Preconditions;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import com.google.common.base.Function;
import com.google.common.collect.Iterators;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.tinkerpop.blueprints.Vertex;
/**
* Titan 0.5.4 implementation of AtlasIndexQuery.
*/
public class Titan0IndexQuery implements AtlasIndexQuery<Titan0Vertex, Titan0Edge> {
private Titan0Graph graph;
private TitanIndexQuery wrappedIndexQuery;
public Titan0IndexQuery(Titan0Graph graph, TitanIndexQuery query) {
wrappedIndexQuery = query;
this.graph = graph;
}
@Override
public Iterator<AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>> vertices() {
Iterator<TitanIndexQuery.Result<Vertex>> results = wrappedIndexQuery.vertices().iterator();
Function<TitanIndexQuery.Result<Vertex>, AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>> function =
new Function<TitanIndexQuery.Result<Vertex>, AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>>() {
@Override
public AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> apply(TitanIndexQuery.Result<Vertex> source) {
return new ResultImpl(source);
}
};
return Iterators.transform(results, function);
}
@Override
public Iterator<Result<Titan0Vertex, Titan0Edge>> vertices(int offset, int limit) {
Preconditions.checkArgument(offset >=0, "Index offset should be greater than or equals to 0");
Preconditions.checkArgument(limit >=0, "Index limit should be greater than or equals to 0");
Iterator<TitanIndexQuery.Result<Vertex>> results = wrappedIndexQuery
.offset(offset)
.limit(limit)
.vertices().iterator();
Function<TitanIndexQuery.Result<Vertex>, AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>> function =
new Function<TitanIndexQuery.Result<Vertex>, AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>>() {
@Override
public AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> apply(TitanIndexQuery.Result<Vertex> source) {
return new ResultImpl(source);
}
};
return Iterators.transform(results, function);
}
private final class ResultImpl implements AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> {
private TitanIndexQuery.Result<Vertex> wrappedResult;
ResultImpl(TitanIndexQuery.Result<Vertex> source) {
wrappedResult = source;
}
@Override
public AtlasVertex<Titan0Vertex, Titan0Edge> getVertex() {
return GraphDbObjectFactory.createVertex(graph, wrappedResult.getElement());
}
@Override
public double getScore() {
return wrappedResult.getScore();
}
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0PropertyKey.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import com.thinkaurelius.titan.core.PropertyKey;
/**
* Titan 0.5.4 implementaiton of AtlasPropertyKey.
*/
public class Titan0PropertyKey implements AtlasPropertyKey {
private PropertyKey wrappedPropertyKey;
public Titan0PropertyKey(PropertyKey toWrap) {
wrappedPropertyKey = toWrap;
}
/*
* (non-Javadoc)
*
* @see org.apache.atlas.repository.graphdb.AtlasPropertyKey#getName()
*/
@Override
public String getName() {
return wrappedPropertyKey.getName();
}
/**
* @return
*/
public PropertyKey getWrappedPropertyKey() {
return wrappedPropertyKey;
}
@Override
public AtlasCardinality getCardinality() {
return GraphDbObjectFactory.createCardinality(wrappedPropertyKey.getCardinality());
}
@Override
public boolean equals(Object other) {
if (!(other instanceof Titan0PropertyKey)) {
return false;
}
Titan0PropertyKey otherKey = (Titan0PropertyKey) other;
return wrappedPropertyKey.equals(otherKey.wrappedPropertyKey);
}
@Override
public int hashCode() {
int result = 17;
result = 37 * result + wrappedPropertyKey.hashCode();
return result;
}
@Override
public String toString() {
return wrappedPropertyKey.getName();
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Vertex.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import java.util.ArrayList;
import java.util.Collection;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
/**
* Titan 0.5.4 implementation of AtlasVertex.
*/
public class Titan0Vertex extends Titan0Element<Vertex> implements AtlasVertex<Titan0Vertex, Titan0Edge> {
public Titan0Vertex(Titan0Graph graph, Vertex source) {
super(graph, source);
}
@Override
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> getEdges(AtlasEdgeDirection dir, String edgeLabel) {
Iterable<Edge> titanEdges = wrappedElement.getEdges(TitanObjectFactory.createDirection(dir), edgeLabel);
return graph.wrapEdges(titanEdges);
}
private TitanVertex getAsTitanVertex() {
return (TitanVertex) wrappedElement;
}
@Override
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> getEdges(AtlasEdgeDirection in) {
Iterable<Edge> titanResult = wrappedElement.getEdges(TitanObjectFactory.createDirection(in));
return graph.wrapEdges(titanResult);
}
@Override
public <T> T getProperty(String propertyName, Class<T> clazz) {
if (graph.isMultiProperty(propertyName)) {
// throw exception in this case to be consistent with Titan 1.0.0
// behavior.
throw new IllegalStateException();
}
return super.getProperty(propertyName, clazz);
}
public <T> void setProperty(String propertyName, T value) {
try {
super.setProperty(propertyName, value);
} catch (UnsupportedOperationException e) {
// For consistency with Titan 1.0.0, treat sets of multiplicity many
// properties as adds. Handle this here since this is an uncommon
// occurrence.
if (graph.isMultiProperty(propertyName)) {
addProperty(propertyName, value);
} else {
throw e;
}
}
}
@Override
public <T> void addProperty(String propertyName, T value) {
try {
getAsTitanVertex().addProperty(propertyName, value);
} catch (SchemaViolationException e) {
if (getPropertyValues(propertyName, value.getClass()).contains(value)) {
// follow java set semantics, don't throw an exception if
// value is already there.
return;
}
throw new AtlasSchemaViolationException(e);
}
}
@Override
public <T> void addListProperty(String propertyName, T value) {
try {
getAsTitanVertex().addProperty(propertyName, value);
} catch (SchemaViolationException e) {
if (getPropertyValues(propertyName, value.getClass()).contains(value)) {
// follow java set semantics, don't throw an exception if
// value is already there.
return;
}
throw new AtlasSchemaViolationException(e);
}
}
@Override
public <T> Collection<T> getPropertyValues(String key, Class<T> clazz) {
TitanVertex tv = getAsTitanVertex();
Collection<T> result = new ArrayList<>();
for (TitanProperty property : tv.getProperties(key)) {
result.add((T) property.getValue());
}
return result;
}
@Override
public AtlasVertexQuery<Titan0Vertex, Titan0Edge> query() {
return new Titan0VertexQuery(graph, wrappedElement.query());
}
@Override
public Titan0Vertex getV() {
return this;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
return "Titan0Vertex [id=" + getId() + "]";
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import com.google.common.base.Preconditions;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.VertexQuery;
/**
* Titan 0.5.4 implementation of AtlasVertexQuery.
*/
public class Titan0VertexQuery implements AtlasVertexQuery<Titan0Vertex, Titan0Edge> {
private Titan0Graph graph;
private VertexQuery vertexQuery;
public Titan0VertexQuery(Titan0Graph graph, VertexQuery vertexQuery) {
this.vertexQuery = vertexQuery;
this.graph = graph;
}
@Override
public AtlasVertexQuery<Titan0Vertex, Titan0Edge> direction(AtlasEdgeDirection queryDirection) {
vertexQuery.direction(TitanObjectFactory.createDirection(queryDirection));
return this;
}
@Override
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices() {
Iterable<Vertex> vertices = vertexQuery.vertices();
return graph.wrapVertices(vertices);
}
@Override
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices(int limit) {
Preconditions.checkArgument(limit >=0, "Limit should be greater than or equals to 0");
Iterable<Vertex> vertices = vertexQuery.limit(limit).vertices();
return graph.wrapVertices(vertices);
}
@Override
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> edges() {
Iterable<Edge> edges = vertexQuery.edges();
return graph.wrapEdges(edges);
}
@Override
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> edges(int limit) {
Preconditions.checkArgument(limit >=0, "Limit should be greater than or equals to 0");
Iterable<Edge> edges = vertexQuery.limit(limit).edges();
return graph.wrapEdges(edges);
}
@Override
public long count() {
return vertexQuery.count();
}
@Override
public AtlasVertexQuery<Titan0Vertex, Titan0Edge> label(String label) {
vertexQuery.labels(label);
return this;
}
@Override
public AtlasVertexQuery<Titan0Vertex, Titan0Edge> has(String key, Object value) {
vertexQuery.has(key, value);
return this;
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/TitanObjectFactory.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.tinkerpop.blueprints.Direction;
import java.util.ArrayList;
import java.util.List;
/**
* Factory that serves up instances of Titan/Tinkerpop classes that correspond to
* graph database abstraction layer/Atlas classes.
*/
public final class TitanObjectFactory {
private TitanObjectFactory() {
}
/**
* Retrieves the titan direction corresponding to the given
* AtlasEdgeDirection.
*
* @param dir
* @return
*/
public static Direction createDirection(AtlasEdgeDirection dir) {
switch(dir) {
case IN:
return Direction.IN;
case OUT:
return Direction.OUT;
case BOTH:
return Direction.BOTH;
default:
throw new RuntimeException("Unrecognized direction: " + dir);
}
}
/**
* Converts a Multiplicity to a Cardinality.
*
* @param cardinality
* @return
*/
public static Cardinality createCardinality(AtlasCardinality cardinality) {
switch(cardinality) {
case SINGLE:
return Cardinality.SINGLE;
case LIST:
return Cardinality.LIST;
case SET:
return Cardinality.SET;
default:
throw new IllegalStateException("Unrecognized cardinality: " + cardinality);
}
}
public static PropertyKey createPropertyKey(AtlasPropertyKey key) {
return ((Titan0PropertyKey)key).getWrappedPropertyKey();
}
public static PropertyKey[] createPropertyKeys(List<AtlasPropertyKey> keys) {
PropertyKey[] ret = new PropertyKey[keys.size()];
int i = 0;
for (AtlasPropertyKey key : keys) {
ret[i] = createPropertyKey(key);
i++;
}
return ret;
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0.query;
import com.thinkaurelius.titan.core.Order;
import com.thinkaurelius.titan.core.TitanGraphQuery;
import com.thinkaurelius.titan.core.attribute.Contain;
import com.thinkaurelius.titan.core.attribute.Text;
import com.thinkaurelius.titan.graphdb.query.TitanPredicate;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.MatchingOperator;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.QueryOperator;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.tinkerpop.query.NativeTinkerpopGraphQuery;
import org.apache.atlas.repository.graphdb.titan0.Titan0Edge;
import org.apache.atlas.repository.graphdb.titan0.Titan0Graph;
import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase;
import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
/**
* Titan 0.5.4 implementation of NativeTitanGraphQuery.
*
* @author jeff
*
*/
public class NativeTitan0GraphQuery implements NativeTinkerpopGraphQuery<Titan0Vertex, Titan0Edge> {
private Titan0Graph graph;
private TitanGraphQuery<?> query;
public NativeTitan0GraphQuery(Titan0Graph graph) {
query = Titan0GraphDatabase.getGraphInstance().query();
this.graph = graph;
}
@Override
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices() {
Iterable it = query.vertices();
return graph.wrapVertices(it);
}
@Override
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> edges() {
Iterable it = query.edges();
return graph.wrapEdges(it);
}
@Override
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> edges(int limit) {
Iterable it = query.limit(limit).edges();
return graph.wrapEdges(it);
}
@Override
public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> edges(int offset, int limit) {
List<Edge> result = new ArrayList<>(limit);
Iterator<Edge> iter = query.limit(offset + limit).edges().iterator();
for (long resultIdx = 0; iter.hasNext() && result.size() < limit; resultIdx++) {
if (resultIdx < offset) {
continue;
}
result.add(iter.next());
}
return graph.wrapEdges(result);
}
@Override
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices(int limit) {
Iterable it = query.limit(limit).vertices();
return graph.wrapVertices(it);
}
@Override
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices(int offset, int limit) {
List<Vertex> result = new ArrayList<>(limit);
Iterator<Vertex> iter = query.limit(offset + limit).vertices().iterator();
for (long resultIdx = 0; iter.hasNext() && result.size() < limit; resultIdx++) {
if (resultIdx < offset) {
continue;
}
result.add(iter.next());
}
return graph.wrapVertices(result);
}
@Override
public void in(String propertyName, Collection<?> values) {
query.has(propertyName, Contain.IN, values);
}
@Override
public void has(String propertyName, QueryOperator op, Object value) {
TitanPredicate pred;
if (op instanceof ComparisionOperator) {
Compare c = getGremlinPredicate((ComparisionOperator) op);
pred = TitanPredicate.Converter.convert(c);
} else {
pred = getGremlinPredicate((MatchingOperator) op);
}
query.has(propertyName, pred, value);
}
@Override
public void orderBy(final String propertyName, final AtlasGraphQuery.SortOrder sortOrder) {
query.orderBy(propertyName, sortOrder == AtlasGraphQuery.SortOrder.ASC ? Order.ASC : Order.DESC);
}
private Text getGremlinPredicate(MatchingOperator op) {
switch (op) {
case CONTAINS:
return Text.CONTAINS;
case PREFIX:
return Text.PREFIX;
case SUFFIX:
return Text.CONTAINS_REGEX;
case REGEX:
return Text.REGEX;
default:
throw new RuntimeException("Unsupported matching operator:" + op);
}
}
private Compare getGremlinPredicate(ComparisionOperator op) {
switch (op) {
case EQUAL:
return Compare.EQUAL;
case GREATER_THAN:
return Compare.GREATER_THAN;
case GREATER_THAN_EQUAL:
return Compare.GREATER_THAN_EQUAL;
case LESS_THAN:
return Compare.LESS_THAN;
case LESS_THAN_EQUAL:
return Compare.LESS_THAN_EQUAL;
case NOT_EQUAL:
return Compare.NOT_EQUAL;
default:
throw new RuntimeException("Unsupported comparison operator:" + op);
}
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/Titan0GraphQuery.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0.query;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.tinkerpop.query.TinkerpopGraphQuery;
import org.apache.atlas.repository.graphdb.tinkerpop.query.NativeTinkerpopGraphQuery;
import org.apache.atlas.repository.graphdb.tinkerpop.query.NativeTinkerpopQueryFactory;
import org.apache.atlas.repository.graphdb.titan0.Titan0Edge;
import org.apache.atlas.repository.graphdb.titan0.Titan0Graph;
import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex;
/**
* Titan 0.5.4 implementation of AtlasGraphQuery.
*/
public class Titan0GraphQuery extends TinkerpopGraphQuery<Titan0Vertex, Titan0Edge>
implements NativeTinkerpopQueryFactory<Titan0Vertex, Titan0Edge> {
public Titan0GraphQuery(Titan0Graph graph, boolean isChildQuery) {
super(graph, isChildQuery);
}
public Titan0GraphQuery(Titan0Graph graph) {
super(graph);
}
@Override
public AtlasGraphQuery<Titan0Vertex, Titan0Edge> createChildQuery() {
return new Titan0GraphQuery((Titan0Graph)graph, true);
}
@Override
protected NativeTinkerpopQueryFactory<Titan0Vertex, Titan0Edge> getQueryFactory() {
return this;
}
@Override
public NativeTinkerpopGraphQuery<Titan0Vertex, Titan0Edge> createNativeTinkerpopQuery() {
return new NativeTitan0GraphQuery((Titan0Graph)graph);
}
}
graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.hbase;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.testng.Assert.fail;
import java.util.concurrent.TimeUnit;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.EntryMetaData;
import com.thinkaurelius.titan.diskstorage.StaticBuffer;
import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
import com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException;
import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
import com.thinkaurelius.titan.diskstorage.util.time.StandardDuration;
import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
public class HBaseKeyColumnValueStoreTest {
@Mock
HBaseStoreManager storeManager;
@Mock
ConnectionMask connectionMask;
@Mock
LocalLockMediator localLockMediator;
@Mock
StaticBuffer key;
@Mock
StaticBuffer column;
@Mock
StaticBuffer expectedValue;
@Mock
HBaseTransaction transaction;
@Mock
Configuration storageConfig;
@BeforeMethod
public void setup() {
MockitoAnnotations.initMocks(this);
}
@Test
public void shouldSucceedInLockingIfLockMediatorSucceeds() throws BackendException {
when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
when(storeManager.getStorageConfig()).thenReturn(storageConfig);
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
new StandardDuration(300L, TimeUnit.MILLISECONDS));
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
new StandardDuration(10L, TimeUnit.MILLISECONDS));
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
KeyColumn lockID = new KeyColumn(key, column);
when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
thenReturn(true);
HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
verify(transaction).updateLocks(lockID, expectedValue);
verify(localLockMediator, times(1)).lock(eq(lockID), eq(transaction), any(Timepoint.class));
}
@Test
public void shouldRetryRightNumberOfTimesIfLockMediationFails() throws BackendException {
when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
when(storeManager.getStorageConfig()).thenReturn(storageConfig);
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
new StandardDuration(300L, TimeUnit.MILLISECONDS));
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
new StandardDuration(10L, TimeUnit.MILLISECONDS));
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
KeyColumn lockID = new KeyColumn(key, column);
when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
thenReturn(false).thenReturn(false).thenReturn(true);
HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
verify(transaction).updateLocks(lockID, expectedValue);
verify(localLockMediator, times(3)).lock(eq(lockID), eq(transaction), any(Timepoint.class));
}
@Test(expectedExceptions = PermanentLockingException.class)
public void shouldThrowExceptionAfterConfiguredRetriesIfLockMediationFails() throws BackendException {
when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
when(storeManager.getStorageConfig()).thenReturn(storageConfig);
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
new StandardDuration(300L, TimeUnit.MILLISECONDS));
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
new StandardDuration(10L, TimeUnit.MILLISECONDS));
when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
KeyColumn lockID = new KeyColumn(key, column);
when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
thenReturn(false).thenReturn(false).thenReturn(false);
HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
fail("Should fail as lock could not be acquired after 3 retries.");
}
}
graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java deleted 100644 → 0
/*
* Copyright 2012-2013 Aurelius LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.thinkaurelius.titan.diskstorage.locking;
import com.thinkaurelius.titan.diskstorage.hbase.HBaseTransaction;
import com.thinkaurelius.titan.diskstorage.util.time.TimestampProvider;
import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
import com.thinkaurelius.titan.diskstorage.StaticBuffer;
import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
import org.mockito.Mockito;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.util.concurrent.TimeUnit;
public class LocalLockMediatorTest {
private static final String LOCK_NAMESPACE = "test";
private static final StaticBuffer LOCK_ROW = StaticArrayBuffer.of(new byte[]{1});
private static final StaticBuffer LOCK_COL = StaticArrayBuffer.of(new byte[]{1});
private static final KeyColumn kc = new KeyColumn(LOCK_ROW, LOCK_COL);
private static final HBaseTransaction mockTx1 = Mockito.mock(HBaseTransaction.class);
private static final HBaseTransaction mockTx2 = Mockito.mock(HBaseTransaction.class);
@Test
public void testLock() throws InterruptedException {
TimestampProvider times = Timestamps.MICRO;
LocalLockMediator<HBaseTransaction> llm =
new LocalLockMediator<>(LOCK_NAMESPACE, times);
//Expire immediately
Assert.assertTrue(llm.lock(kc, mockTx1, times.getTime(0, TimeUnit.NANOSECONDS)));
Assert.assertTrue(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
llm = new LocalLockMediator<>(LOCK_NAMESPACE, times);
//Expire later
Assert.assertTrue(llm.lock(kc, mockTx1, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
//So second lock should fail on same keyCol
Assert.assertFalse(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
//Unlock
Assert.assertTrue(llm.unlock(kc, mockTx1));
//Now locking should succeed
Assert.assertTrue(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
}
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.graph.GraphSandboxUtil;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
*
*/
public abstract class AbstractGraphDatabaseTest {
protected static final String WEIGHT_PROPERTY = "weight";
protected static final String TRAIT_NAMES = Constants.TRAIT_NAMES_PROPERTY_KEY;
protected static final String TYPE_PROPERTY_NAME = "__type";
protected static final String TYPESYSTEM = "TYPESYSTEM";
private static final String BACKING_INDEX_NAME = "backing";
private AtlasGraph<?, ?> graph = null;
@BeforeClass
public static void createIndices() {
GraphSandboxUtil.create();
Titan0GraphDatabase db = new Titan0GraphDatabase();
AtlasGraphManagement mgmt = db.getGraph().getManagementSystem();
if (mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
mgmt.createVertexMixedIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX, Collections.emptyList());
}
mgmt.makePropertyKey("age13", Integer.class, AtlasCardinality.SINGLE);
createIndices(mgmt, "name", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, WEIGHT_PROPERTY, Integer.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "size15", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "typeName", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "__type", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, Constants.GUID_PROPERTY_KEY, String.class, true, AtlasCardinality.SINGLE);
createIndices(mgmt, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET);
createIndices(mgmt, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET);
mgmt.commit();
}
@AfterMethod
public void commitGraph() {
//force any pending actions to be committed so we can be sure they don't cause errors.
pushChangesAndFlushCache();
getGraph().commit();
}
@AfterClass
public static void cleanUp() {
Titan0Graph graph = new Titan0Graph();
graph.clear();
}
protected <V, E> void pushChangesAndFlushCache() {
getGraph().commit();
}
private static void createIndices(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality) {
if (management.containsPropertyKey(propertyName)) {
//index was already created
return;
}
AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
try {
if (propertyClass != Integer.class) {
management.addMixedIndex(BACKING_INDEX_NAME, key);
}
} catch(Throwable t) {
//ok
t.printStackTrace();
}
try {
management.createVertexCompositeIndex(propertyName, isUnique, Collections.singletonList(key));
} catch(Throwable t) {
//ok
t.printStackTrace();
}
}
protected final <V, E> AtlasGraph<V, E> getGraph() {
if (graph == null) {
graph = new Titan0Graph();
}
return (AtlasGraph<V, E>)graph;
}
protected Titan0Graph getTitan0Graph() {
AtlasGraph g = getGraph();
return (Titan0Graph)g;
}
protected List<AtlasVertex> newVertices = new ArrayList<>();
protected final <V, E> AtlasVertex<V, E> createVertex(AtlasGraph<V, E> theGraph) {
AtlasVertex<V, E> vertex = theGraph.addVertex();
newVertices.add(vertex);
return vertex;
}
@AfterMethod
public void removeVertices() {
for(AtlasVertex vertex : newVertices) {
if (vertex.exists()) {
getGraph().removeVertex(vertex);
}
}
getGraph().commit();
newVertices.clear();
}
protected void runSynchronouslyInNewThread(final Runnable r) throws Throwable {
RunnableWrapper wrapper = new RunnableWrapper(r);
Thread th = new Thread(wrapper);
th.start();
th.join();
Throwable ex = wrapper.getExceptionThrown();
if (ex != null) {
throw ex;
}
}
private static final class RunnableWrapper implements Runnable {
private final Runnable r;
private Throwable exceptionThrown = null;
private RunnableWrapper(Runnable r) {
this.r = r;
}
@Override
public void run() {
try {
r.run();
} catch(Throwable e) {
exceptionThrown = e;
}
}
public Throwable getExceptionThrown() {
return exceptionThrown;
}
}
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/GraphQueryTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import static org.testng.AssertJUnit.assertEquals;
import static org.testng.AssertJUnit.assertTrue;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.testng.annotations.Test;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
/**
* Tests for Titan0GraphQuery.
*/
@Test
public class GraphQueryTest extends AbstractGraphDatabaseTest {
@Test
public <V, E> void testQueryThatCannotRunInMemory() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
graph.commit();
AtlasVertex<V, E> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
AtlasGraphQuery q = graph.query();
q.has("name", ComparisionOperator.NOT_EQUAL, "George");
q.has("size15", "15");
graph.commit();
pause(); //pause to let the index get updated
assertQueryMatches(q, v1, v3, v4);
}
@Test
public void testCombinationOfAndsAndOrs() throws AtlasException {
Titan0Graph graph = getTitan0Graph();
AtlasVertex<Titan0Vertex, Titan0Edge> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
v1.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v2 = createVertex(graph);
v2.setProperty("name", "George");
v2.setProperty("size15", "16");
v2.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v3 = createVertex(graph);
v3.setProperty("name", "Jane");
v3.setProperty("size15", "17");
v3.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v4 = createVertex(graph);
v4.setProperty("name", "Bob");
v4.setProperty("size15", "18");
v4.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v5 = createVertex(graph);
v5.setProperty("name", "Julia");
v5.setProperty("size15", "19");
v5.setProperty("typeName", "Manager");
AtlasGraphQuery q = getGraphQuery();
q.has("typeName", "Person");
//initially match
AtlasGraphQuery inner1a = q.createChildQuery();
AtlasGraphQuery inner1b = q.createChildQuery();
inner1a.has("name", "Fred");
inner1b.has("name", "Jane");
q.or(toList(inner1a, inner1b));
AtlasGraphQuery inner2a = q.createChildQuery();
AtlasGraphQuery inner2b = q.createChildQuery();
AtlasGraphQuery inner2c = q.createChildQuery();
inner2a.has("size15", "18");
inner2b.has("size15", "15");
inner2c.has("size15", "16");
q.or(toList(inner2a, inner2b, inner2c));
assertQueryMatches(q, v1);
graph.commit();
pause(); //let the index update
assertQueryMatches(q, v1);
}
@Test
public void testWithinStep() throws AtlasException {
Titan0Graph graph = getTitan0Graph();
AtlasVertex<Titan0Vertex, Titan0Edge> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
v1.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v2 = createVertex(graph);
v2.setProperty("name", "George");
v2.setProperty("size15", "16");
v2.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v3 = createVertex(graph);
v3.setProperty("name", "Jane");
v3.setProperty("size15", "17");
v3.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v4 = createVertex(graph);
v4.setProperty("name", "Bob");
v4.setProperty("size15", "18");
v4.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v5 = createVertex(graph);
v5.setProperty("name", "Julia");
v5.setProperty("size15", "19");
v5.setProperty("typeName", "Manager");
AtlasGraphQuery q = getGraphQuery();
q.has("typeName", "Person");
//initially match
q.in("name", toList("Fred", "Jane"));
q.in("size15", toList("18", "15", "16"));
assertQueryMatches(q, v1);
graph.commit();
pause(); //let the index update
assertQueryMatches(q, v1);
}
@Test
public void testWithinStepWhereGraphIsStale() throws AtlasException {
Titan0Graph graph = getTitan0Graph();
AtlasVertex<Titan0Vertex, Titan0Edge> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
v1.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v2 = createVertex(graph);
v2.setProperty("name", "George");
v2.setProperty("size15", "16");
v2.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v3 = createVertex(graph);
v3.setProperty("name", "Jane");
v3.setProperty("size15", "17");
v3.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v4 = createVertex(graph);
v4.setProperty("name", "Bob");
v4.setProperty("size15", "18");
v4.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex, Titan0Edge> v5 = createVertex(graph);
v5.setProperty("name", "Julia");
v5.setProperty("size15", "19");
v5.setProperty("typeName", "Manager");
AtlasGraphQuery q = getGraphQuery();
q.has("typeName", "Person");
//initially match
q.in("name", toList("Fred", "Jane"));
graph.commit();
pause(); //let the index update
assertQueryMatches(q, v1, v3);
//make v3 no longer match the query. Within step should filter out the vertex since it no longer matches.
v3.setProperty("name", "Janet");
assertQueryMatches(q, v1);
}
@Test
public void testSimpleOrQuery() throws AtlasException {
Titan0Graph graph = getTitan0Graph();
AtlasVertex<Titan0Vertex, Titan0Edge> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<Titan0Vertex, Titan0Edge> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<Titan0Vertex, Titan0Edge> v3 = createVertex(graph);
v3.setProperty("size15", "15");
graph.commit();
AtlasVertex<Titan0Vertex, Titan0Edge> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
AtlasVertex<Titan0Vertex, Titan0Edge> v5 = createVertex(graph);
v5.setProperty("name", "George");
v5.setProperty("size15", "16");
AtlasGraphQuery q = graph.query();
AtlasGraphQuery inner1 = q.createChildQuery().has("name", "Fred");
AtlasGraphQuery inner2 = q.createChildQuery().has("size15", "15");
q.or(toList(inner1, inner2));
assertQueryMatches(q, v1, v2, v3, v4);
graph.commit();
pause(); //pause to let the indexer get updated (this fails frequently without a pause)
assertQueryMatches(q, v1, v2, v3, v4);
}
@Test
public <V, E> void testQueryMatchesAddedVertices() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
graph.commit();
AtlasVertex<V, E> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
AtlasGraphQuery q = getGraphQuery();
q.has("name", "Fred");
q.has("size15", "15");
assertQueryMatches(q, v1, v4);
graph.commit();
assertQueryMatches(q, v1, v4);
}
@Test
public <V, E> void testQueryDoesNotMatchRemovedVertices() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
AtlasVertex<V, E> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
graph.commit();
graph.removeVertex(v1);
AtlasGraphQuery q = getGraphQuery();
q.has("name", "Fred");
q.has("size15", "15");
assertQueryMatches(q, v4);
graph.commit();
assertQueryMatches(q, v4);
}
@Test
public <V, E> void testQueryDoesNotMatchUncommittedAddedAndRemovedVertices() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
AtlasVertex<V, E> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
AtlasGraphQuery q = getGraphQuery();
q.has("name", "Fred");
q.has("size15", "15");
assertQueryMatches(q, v1, v4);
graph.removeVertex(v1);
assertQueryMatches(q, v4);
graph.commit();
assertQueryMatches(q, v4);
}
@Test
public <V, E> void testQueryResultsReflectPropertyAdd() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
v1.addProperty(TRAIT_NAMES, "trait1");
v1.addProperty(TRAIT_NAMES, "trait2");
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
v2.addProperty(TRAIT_NAMES, "trait1");
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
v3.addProperty(TRAIT_NAMES, "trait2");
AtlasGraphQuery query = getGraphQuery();
query.has("name", "Fred");
query.has(TRAIT_NAMES, "trait1");
query.has("size15", "15");
assertQueryMatches(query, v1);
//make v3 match the query
v3.setProperty("name", "Fred");
v3.addProperty(TRAIT_NAMES, "trait1");
assertQueryMatches(query, v1, v3);
v3.removeProperty(TRAIT_NAMES);
assertQueryMatches(query, v1);
v3.addProperty(TRAIT_NAMES, "trait2");
assertQueryMatches(query, v1);
v1.removeProperty(TRAIT_NAMES);
assertQueryMatches(query);
graph.commit();
assertQueryMatches(query);
}
private static <T> List<T> toList(Iterable<T> itr) {
List<T> result = new ArrayList<>();
for(T object : itr) {
result.add(object);
}
return result;
}
private <V, E> void assertQueryMatches(AtlasGraphQuery expr, AtlasVertex... expectedResults) throws AtlasException {
//getGraph().commit();
Collection<AtlasVertex<Titan0Vertex, Titan0Edge>> temp = toList(expr.vertices());
//filter out vertices from previous test executions
Collection<AtlasVertex<Titan0Vertex, Titan0Edge>> result =
Collections2.filter(temp, new Predicate<AtlasVertex<Titan0Vertex, Titan0Edge>>() {
@Override
public boolean apply(AtlasVertex<Titan0Vertex, Titan0Edge> input) {
return newVertices.contains(input);
}
});
String errorMessage = "Expected/found result sizes differ. Expected: "
+ Arrays.asList(expectedResults).toString() +", found: " + result;
assertEquals(errorMessage, expectedResults.length, result.size());
for(AtlasVertex<V, E> v : expectedResults) {
assertTrue(result.contains(v));
}
}
private static List<Object> toList(Object...objects) {
return Arrays.asList(objects);
}
private AtlasGraphQuery<Titan0Vertex, Titan0Edge> getGraphQuery() {
return getTitan0Graph().query();
}
private void pause() {
try {
Thread.sleep(5000);
} catch(InterruptedException e) {
//ignore
}
}
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.AtlasException;
import org.apache.atlas.graph.GraphSandboxUtil;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Test;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import static org.testng.Assert.*;
/**
* Sanity test of basic graph operations using the Titan 0.5.4 graphdb
* abstraction layer implementation.
*/
public class Titan0DatabaseTest {
private AtlasGraph<?, ?> atlasGraph;
private <V, E> AtlasGraph<V, E> getGraph() {
GraphSandboxUtil.create();
if (atlasGraph == null) {
Titan0GraphDatabase db = new Titan0GraphDatabase();
atlasGraph = db.getGraph();
AtlasGraphManagement mgmt = atlasGraph.getManagementSystem();
// create the index (which defines these properties as being mult
// many)
for (String propertyName : new String[]{"__superTypeNames", "__traitNames"}) {
AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = mgmt.makePropertyKey(propertyName, String.class, AtlasCardinality.SET);
mgmt.createVertexCompositeIndex(propertyName, false, Collections.singletonList(propertyKey));
}
}
mgmt.commit();
}
return (AtlasGraph<V, E>) atlasGraph;
}
@AfterClass
public void cleanup() {
atlasGraph.clear();
atlasGraph = null;
}
@Test
public <V, E> void testPropertyDataTypes() {
// primitives
AtlasGraph<V, E> graph = getGraph();
testProperty(graph, "booleanProperty", Boolean.TRUE);
testProperty(graph, "booleanProperty", Boolean.FALSE);
testProperty(graph, "byteProperty", Byte.MAX_VALUE);
testProperty(graph, "byteProperty", Byte.MIN_VALUE);
testProperty(graph, "shortProperty", Short.MAX_VALUE);
testProperty(graph, "shortProperty", Short.MIN_VALUE);
testProperty(graph, "intProperty", Integer.MAX_VALUE);
testProperty(graph, "intProperty", Integer.MIN_VALUE);
testProperty(graph, "longProperty", Long.MIN_VALUE);
testProperty(graph, "longProperty", Long.MAX_VALUE);
testProperty(graph, "doubleProperty", Double.MAX_VALUE);
testProperty(graph, "doubleProperty", Double.MIN_VALUE);
testProperty(graph, "floatProperty", Float.MAX_VALUE);
testProperty(graph, "floatProperty", Float.MIN_VALUE);
// enumerations - TypeCategory
testProperty(graph, "typeCategoryProperty", TypeCategory.CLASS);
// biginteger
testProperty(graph, "bigIntegerProperty",
new BigInteger(String.valueOf(Long.MAX_VALUE)).multiply(BigInteger.TEN));
// bigdecimal
BigDecimal bigDecimal = new BigDecimal(Double.MAX_VALUE);
testProperty(graph, "bigDecimalProperty", bigDecimal.multiply(bigDecimal));
}
private <V, E> void testProperty(AtlasGraph<V, E> graph, String name, Object value) {
AtlasVertex<V, E> vertex = graph.addVertex();
vertex.setProperty(name, value);
assertEquals(value, vertex.getProperty(name, value.getClass()));
AtlasVertex<V, E> loaded = graph.getVertex(vertex.getId().toString());
assertEquals(value, loaded.getProperty(name, value.getClass()));
}
@Test
public <V, E> void testMultiplicityOnePropertySupport() {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> vertex = graph.addVertex();
vertex.setProperty("name", "Jeff");
vertex.setProperty("location", "Littleton");
assertEquals("Jeff", vertex.getProperty("name", String.class));
assertEquals("Littleton", vertex.getProperty("location", String.class));
AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString());
assertEquals("Jeff", vertexCopy.getProperty("name", String.class));
assertEquals("Littleton", vertexCopy.getProperty("location", String.class));
assertTrue(vertexCopy.getPropertyKeys().contains("name"));
assertTrue(vertexCopy.getPropertyKeys().contains("location"));
assertTrue(vertexCopy.getPropertyValues("name", String.class).contains("Jeff"));
assertTrue(vertexCopy.getPropertyValues("location", String.class).contains("Littleton"));
assertTrue(vertexCopy.getPropertyValues("test", String.class).isEmpty());
assertNull(vertexCopy.getProperty("test", String.class));
vertex.removeProperty("name");
assertFalse(vertex.getPropertyKeys().contains("name"));
assertNull(vertex.getProperty("name", String.class));
assertTrue(vertex.getPropertyValues("name", String.class).isEmpty());
vertexCopy = graph.getVertex(vertex.getId().toString());
assertFalse(vertexCopy.getPropertyKeys().contains("name"));
assertNull(vertexCopy.getProperty("name", String.class));
assertTrue(vertexCopy.getPropertyValues("name", String.class).isEmpty());
}
@Test
public <V, E> void testRemoveEdge() {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = graph.addVertex();
AtlasVertex<V, E> v2 = graph.addVertex();
AtlasEdge<V, E> edge = graph.addEdge(v1, v2, "knows");
// make sure the edge exists
AtlasEdge<V, E> edgeCopy = graph.getEdge(edge.getId().toString());
assertNotNull(edgeCopy);
assertEquals(edgeCopy, edge);
graph.removeEdge(edge);
edgeCopy = graph.getEdge(edge.getId().toString());
// should return null now, since edge was deleted
assertNull(edgeCopy);
}
@Test
public <V, E> void testRemoveVertex() {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = graph.addVertex();
assertNotNull(graph.getVertex(v1.getId().toString()));
graph.removeVertex(v1);
assertNull(graph.getVertex(v1.getId().toString()));
}
@Test
public <V, E> void testGetEdges() {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = graph.addVertex();
AtlasVertex<V, E> v2 = graph.addVertex();
AtlasVertex<V, E> v3 = graph.addVertex();
AtlasEdge<V, E> knows = graph.addEdge(v2, v1, "knows");
AtlasEdge<V, E> eats = graph.addEdge(v3, v1, "eats");
AtlasEdge<V, E> drives = graph.addEdge(v3, v2, "drives");
AtlasEdge<V, E> sleeps = graph.addEdge(v2, v3, "sleeps");
assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.IN), knows, eats);
assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.OUT));
assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.BOTH), knows, eats);
assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.IN, "knows"), knows);
assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.OUT, "knows"));
assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.BOTH, "knows"), knows);
assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.IN), drives);
assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.OUT), knows, sleeps);
assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.BOTH), knows, sleeps, drives);
assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.BOTH, "delivers"));
}
private <V, E> void assertEdgesMatch(Iterable<AtlasEdge<V, E>> edgesIt, AtlasEdge<V, E>... expected) {
List<AtlasEdge<V, E>> edges = toList(edgesIt);
assertEquals(expected.length, edges.size());
for (AtlasEdge<V, E> edge : expected) {
assertTrue(edges.contains(edge));
}
}
@Test
public <V, E> void testMultiplictyManyPropertySupport() {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> vertex = graph.addVertex();
String vertexId = vertex.getId().toString();
vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2");
assertEquals(vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class).size(), 2);
vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait3");
vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait4");
assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
validateMultManyPropertiesInVertex(vertex);
// fetch a copy of the vertex, make sure result
// is the same
validateMultManyPropertiesInVertex(graph.getVertex(vertexId));
}
private <V, E> void validateMultManyPropertiesInVertex(AtlasVertex<V, E> vertex) {
assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
Collection<String> traitNames = vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class);
assertTrue(traitNames.contains("trait1"));
assertTrue(traitNames.contains("trait2"));
assertTrue(traitNames.contains("trait3"));
assertTrue(traitNames.contains("trait4"));
try {
vertex.getProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class);
fail("Expected exception not thrown");
} catch (IllegalStateException expected) {
// multiple property values exist
}
}
@Test
public <V, E> void testListProperties() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> vertex = graph.addVertex();
vertex.setListProperty("colors", Arrays.asList("red", "blue", "green"));
List<String> colors = vertex.getListProperty("colors");
assertTrue(colors.contains("red"));
assertTrue(colors.contains("blue"));
assertTrue(colors.contains("green"));
AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString());
colors = vertexCopy.getListProperty("colors");
assertTrue(colors.contains("red"));
assertTrue(colors.contains("blue"));
assertTrue(colors.contains("green"));
}
@Test
public <V, E> void testRemoveProperty() {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> vertex = graph.addVertex();
vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
vertex.setProperty("name", "Jeff");
// remove existing property - multiplicity one
vertex.removeProperty("jeff");
assertFalse(vertex.getPropertyKeys().contains("jeff"));
// remove existing property - multiplicity many
vertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
assertFalse(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString());
assertFalse(vertexCopy.getPropertyKeys().contains("jeff"));
assertFalse(vertexCopy.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
// remove non-existing property
vertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
vertex.removeProperty("jeff");
}
@Test
public <V, E> void getGetGraphQueryForVertices() {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = graph.addVertex();
AtlasVertex<V, E> v2 = graph.addVertex();
AtlasVertex<V, E> v3 = graph.addVertex();
v1.setProperty("name", "Jeff");
v1.setProperty("weight", 1);
v2.setProperty("name", "Fred");
v2.setProperty("weight", 2);
v3.setProperty("name", "Chris");
v3.setProperty("weight", 3);
AtlasEdge<V, E> knows = graph.addEdge(v2, v1, "knows");
knows.setProperty("weight", 1);
AtlasEdge<V, E> eats = graph.addEdge(v3, v1, "eats");
eats.setProperty("weight", 2);
AtlasEdge<V, E> drives = graph.addEdge(v3, v2, "drives");
drives.setProperty("weight", 3);
AtlasEdge<V, E> sleeps = graph.addEdge(v2, v3, "sleeps");
sleeps.setProperty("weight", 4);
testExecuteGraphQuery("name", null, "Jeff", v1);
testExecuteGraphQuery("weight", ComparisionOperator.EQUAL, 2, v2);
testExecuteGraphQuery("weight", ComparisionOperator.GREATER_THAN_EQUAL, 2, v2, v3);
testExecuteGraphQuery("weight", ComparisionOperator.LESS_THAN_EQUAL, 2, v2, v1);
}
private <V, E> void testExecuteGraphQuery(String property, ComparisionOperator op, Object value,
AtlasVertex<V, E>... expected) {
AtlasGraph<V, E> graph = getGraph();
AtlasGraphQuery<V, E> query = graph.query();
if (op != null) {
query.has(property, op, value);
} else {
query.has(property, value);
}
Iterable<? extends AtlasVertex<V, E>> result = query.vertices();
List<AtlasVertex<V, E>> list = toList(result);
assertEquals(expected.length, list.size());
for (AtlasVertex<V, E> vertex : expected) {
assertTrue(list.contains(vertex));
}
}
@Test
public <V, E> void testAddMultManyPropertyValueTwice() {
AtlasGraph<V, E> graph = getGraph();
String vertexId;
AtlasVertex<V, E> vertex = graph.addVertex();
vertexId = vertex.getId().toString();
vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2");
vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2");
validateDuplicatePropertyVertex(vertex);
// fetch a copy of the vertex, make sure result is the same
validateDuplicatePropertyVertex(graph.getVertex(vertexId));
}
private <V, E> void validateDuplicatePropertyVertex(AtlasVertex<V, E> vertex) {
assertEquals(2, vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class).size());
assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
Collection<String> traitNames = vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class);
assertTrue(traitNames.contains("trait1"));
assertTrue(traitNames.contains("trait2"));
}
private static <T> List<T> toList(Iterable<? extends T> iterable) {
List<T> result = new ArrayList<>();
for (T item : iterable) {
result.add(item);
}
return result;
}
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseValidationTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.graph.GraphSandboxUtil;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.commons.configuration.Configuration;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@Test
public class Titan0DatabaseValidationTest {
private Configuration configuration;
private AtlasGraph<?, ?> graph;
@BeforeTest
public void setUp() throws AtlasException {
GraphSandboxUtil.create();
// First get Instance
graph = new Titan0Graph();
configuration = ApplicationProperties.getSubsetConfiguration(ApplicationProperties.get(),
Titan0GraphDatabase.GRAPH_PREFIX);
}
@AfterClass
public void tearDown() throws Exception {
try {
graph.shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
graph.clear();
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void testValidate() throws AtlasException {
try {
Titan0GraphDatabase.validateIndexBackend(configuration);
} catch (Exception e) {
Assert.fail("Unexpected exception ", e);
}
// Change backend
configuration.setProperty(Titan0GraphDatabase.INDEX_BACKEND_CONF, Titan0GraphDatabase.INDEX_BACKEND_LUCENE);
try {
Titan0GraphDatabase.validateIndexBackend(configuration);
Assert.fail("Expected exception");
} catch (Exception e) {
Assert.assertEquals(e.getMessage(),
"Configured Index Backend lucene differs from earlier configured "
+ "Index Backend elasticsearch. Aborting!");
}
}
}
graphdb/titan0/src/test/resources/atlas-application.properties deleted 100644 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
######### Graph Database to Use #########
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase
######### Atlas Server Configs #########
atlas.rest.address=http://localhost:31000
######### Graph Database Configs #########
# Graph Storage
atlas.graph.storage.backend=${graph.storage.backend}
# Graph Search Index Backend
atlas.graph.index.search.backend=${graph.index.backend}
#Berkeley storage directory
atlas.graph.storage.directory=${sys:atlas.data}/berkeley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
atlas.graph.storage.lock.wait-time=10000
#ElasticSearch
atlas.graph.index.search.directory=${sys:atlas.data}/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
# Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
# Solr-specific configuration property
atlas.graph.index.search.max-result-set-size=150
######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
#atlas.lineage.hive.table.type.name=DataSet
#atlas.lineage.hive.process.type.name=Process
#atlas.lineage.hive.process.inputs.name=inputs
#atlas.lineage.hive.process.outputs.name=outputs
## Schema
atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
######### Notification Configs #########
atlas.notification.embedded=true
atlas.kafka.zookeeper.connect=localhost:19026
atlas.kafka.bootstrap.servers=localhost:19027
atlas.kafka.data=${sys:atlas.data}/kafka
atlas.kafka.zookeeper.session.timeout.ms=4000
atlas.kafka.zookeeper.sync.time.ms=20
atlas.kafka.consumer.timeout.ms=100
atlas.kafka.auto.commit.interval.ms=100
atlas.kafka.hook.group.id=atlas
atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.offsets.topic.replication.factor=1
######### Entity Audit Configs #########
atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
atlas.audit.zookeeper.session.timeout.ms=1000
atlas.audit.hbase.zookeeper.quorum=localhost
atlas.audit.hbase.zookeeper.property.clientPort=19026
######### Security Properties #########
# SSL config
atlas.enableTLS=false
atlas.server.https.port=31443
######### Security Properties #########
hbase.security.authentication=simple
atlas.hook.falcon.synchronous=true
######### High Availability Configuration ########
atlas.server.ha.enabled=false
#atlas.server.ids=id1
#atlas.server.address.id1=localhost:21000
intg/src/main/resources/atlas-log4j.xml
......@@ -72,7 +72,7 @@
<appender-ref ref="AUDIT"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="console"/>
</logger>
......
intg/src/test/resources/atlas-application.properties
......@@ -57,7 +57,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
......
pom.xml
......@@ -650,10 +650,9 @@
These profiles are mutually exclusive and should be activated by setting the system
property GRAPH-PROVIDER.
This can be optionally specified when invoking mvn:
e.g. mvn clean install -DGRAPH-PROVIDER=titan0
e.g. mvn clean install -DGRAPH-PROVIDER=janus
The settings for GRAPH-PROVIDER have the following effects:
* If GRAPH-PROVIDER is not specified, the graph-provider-default profile is activated.
* If GRAPH-PROVIDER is set to titan0, the graph-provider-titan0 profile is activated.
* If GRAPH-PROVIDER is not specified, the graph-provider-default profile (janus) is activated.
* If GRAPH-PROVIDER is set to anything else, the build will fail.
Do not activate the graph-provider selection profiles using -P.
-->
......@@ -667,7 +666,6 @@
</property>
</activation>
<properties>
<!-- Define graph dependency type/version -->
<graphGroup>org.apache.atlas</graphGroup>
<graphArtifact>atlas-graphdb-janus</graphArtifact>
<skipDocs>false</skipDocs>
......@@ -687,7 +685,6 @@
</property>
</activation>
<properties>
<!-- Define graph dependency type/version -->
<graphGroup>org.apache.atlas</graphGroup>
<graphArtifact>atlas-graphdb-janus</graphArtifact>
<skipDocs>false</skipDocs>
......@@ -699,26 +696,6 @@
</profile>
<profile>
<id>graph-provider-titan0</id>
<activation>
<property>
<name>GRAPH-PROVIDER</name>
<value>titan0</value>
</property>
</activation>
<properties>
<!-- Define graph dependency type/version -->
<graphGroup>org.apache.atlas</graphGroup>
<graphArtifact>atlas-graphdb-titan0</graphArtifact>
<skipDocs>false</skipDocs>
<graphdb.backend.impl>org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase</graphdb.backend.impl>
<graph.index.backend>elasticsearch</graph.index.backend>
<tests.solr.embedded>false</tests.solr.embedded>
<distro.exclude.packages>WEB-INF/lib/titan-*.jar,WEB-INF/lib/je-*.jar,WEB-INF/lib/elasticsearch-*.jar,WEB-INF/lib/lucene-*.jar</distro.exclude.packages>
</properties>
</profile>
<profile>
<id>skipMinify</id>
<properties>
<project.build.dashboardv2.gruntBuild>build</project.build.dashboardv2.gruntBuild>
......
repository/pom.xml
......@@ -251,30 +251,6 @@
</dependency>
</dependencies>
</profile>
<profile>
<id>graph-provider-titan0</id>
<activation>
<property>
<name>GRAPH-PROVIDER</name>
<value>titan0</value>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-testtools</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
</profiles>
<build>
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -116,7 +116,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
/**
* Initialize global indices for Titan graph on server activation.
* Initialize global indices for JanusGraph on server activation.
*
* Since the indices are shared state, we need to do this only from an active instance.
*/
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -761,7 +761,7 @@ public abstract class DeleteHandlerV1 {
List<String> elements = GraphHelper.getListProperty(outVertex, propertyName);
if (elements != null) {
elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty()
elements = new ArrayList<>(elements);
for (String elementEdgeId : elements) {
AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
......@@ -807,7 +807,7 @@ public abstract class DeleteHandlerV1 {
List<String> keys = GraphHelper.getListProperty(outVertex, propertyName);
if (keys != null) {
keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty()
keys = new ArrayList<>(keys);
for (String key : keys) {
String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
......
tools/atlas-migration-exporter/pom.xml
......@@ -61,15 +61,5 @@
<artifactId>blueprints-core</artifactId>
<version>${tinkerpop.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
......@@ -18,12 +18,7 @@
package org.apache.atlas.migration;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.cli.BasicParser;
......@@ -143,10 +138,6 @@ public class Exporter {
try {
os = new FileOutputStream(dataFileName);
Graph graph = getTitan0GraphDatabase();
GraphSONWriter.outputGraph(graph, os, GraphSONMode.EXTENDED);
} finally {
if (os != null) {
try {
......@@ -167,10 +158,6 @@ public class Exporter {
new ArrayList<>(registry.getAllEntityDefs()));
}
private TitanGraph getTitan0GraphDatabase() {
return Titan0GraphDatabase.getGraphInstance();
}
private static void displayMessage(String msg) {
LOG.info(LOG_MSG_PREFIX + msg);
......
webapp/pom.xml
......@@ -460,9 +460,9 @@
</manifest>
</archive>
<packagingExcludes>
<!-- Titan and hbase jars should be excluded because an uber jar with shaded dependencies is created.
<!-- HBase jars should be excluded because an uber jar with shaded dependencies is created.
But mvn 3.3.x includes them for some reason. So, excluding them explicitly here -->
WEB-INF/lib/titan*.jar,WEB-INF/lib/hbase*.jar,WEB-INF/lib/junit*.jar,${packages.to.exclude}
WEB-INF/lib/hbase*.jar,WEB-INF/lib/junit*.jar,${packages.to.exclude}
</packagingExcludes>
</configuration>
</plugin>
......
webapp/src/test/resources/atlas-application.properties
......@@ -39,7 +39,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#for distributed mode, specify zookeeper quorum here
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment