Skip to content

A
atlas
Commit d2d6ff7d by Suma Shivaprasad
Options

ATLAS-694 Update Atlas code to use graph abstraction layer (jnhagelb via sumasai)

parent 3e4f28f5
Showing with 1448 additions and 1081 deletions
.gitignore
......@@ -18,7 +18,7 @@
target
dependency-reduced-pom.xml
core*.dmp
pom.xml.releaseBackup
# IntelliJ
*.iml
......
addons/falcon-bridge/pom.xml
......@@ -93,6 +93,13 @@
<artifactId>jetty-server</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
......
addons/hdfs-model/pom.xml
......@@ -73,6 +73,13 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
<scope>test</scope>
</dependency>
......
addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
......@@ -18,10 +18,11 @@
package org.apache.atlas.fs.model;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import javax.inject.Inject;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
......@@ -33,11 +34,10 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import scala.Enumeration;
import scala.collection.Iterator;
import javax.inject.Inject;
@Test
@Guice(modules = RepositoryMetadataModule.class)
public class HDFSModelTest {
......@@ -48,9 +48,6 @@ public class HDFSModelTest {
@Inject
private MetadataService metadataService;
@Inject
private GraphProvider<TitanGraph> graphProvider;
@BeforeClass
public void setUp() throws Exception {
}
......@@ -58,17 +55,7 @@ public class HDFSModelTest {
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
@Test
......
addons/hive-bridge/pom.xml
......@@ -149,6 +149,13 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
......@@ -18,8 +18,10 @@
package org.apache.atlas.hive.bridge;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientResponse;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
......@@ -57,9 +59,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientResponse;
/**
* A Bridge Utility that imports metadata from the Hive Meta Store
......
addons/sqoop-bridge/pom.xml
......@@ -151,6 +151,13 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
......
addons/storm-bridge/pom.xml
......@@ -117,6 +117,14 @@
</dependency>
<!-- to bring up atlas server for integration tests -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
......
catalog/pom.xml
......@@ -60,6 +60,18 @@
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-api</artifactId>
</dependency>
<!-- for now, catalog can only be used with Titan 0.5.4 -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
......
catalog/src/main/java/org/apache/atlas/catalog/query/BaseQuery.java
......@@ -18,12 +18,11 @@
package org.apache.atlas.catalog.query;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.gremlin.java.GremlinPipeline;
import com.tinkerpop.pipes.Pipe;
import com.tinkerpop.pipes.filter.PropertyFilterPipe;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.atlas.catalog.Request;
import org.apache.atlas.catalog.VertexWrapper;
import org.apache.atlas.catalog.definition.ResourceDefinition;
......@@ -31,13 +30,15 @@ import org.apache.atlas.catalog.exception.ResourceNotFoundException;
import org.apache.atlas.catalog.projection.Projection;
import org.apache.atlas.catalog.projection.ProjectionResult;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase;
import org.apache.atlas.typesystem.persistence.Id;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.gremlin.java.GremlinPipeline;
import com.tinkerpop.pipes.Pipe;
import com.tinkerpop.pipes.filter.PropertyFilterPipe;
/**
* Base Query implementation.
......@@ -167,7 +168,7 @@ public abstract class BaseQuery implements AtlasQuery {
//todo: abstract
// Underlying method is synchronized and caches the graph in a static field
protected TitanGraph getGraph() {
return TitanGraphProvider.getGraphInstance();
return Titan0GraphDatabase.getGraphInstance();
}
protected VertexWrapper wrapVertex(Vertex v) {
......
distro/src/conf/atlas-application.properties
......@@ -17,6 +17,12 @@
#
######### Graph Database Configs #########
# Graph Database
#Configures the graph database to use. Defaults to Titan 0.5.4.
#atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase
# Graph Storage
atlas.graph.storage.backend=${titan.storage.backend}
atlas.graph.storage.hbase.table=apache_atlas_titan
......
repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java → graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasCardinality.java 100755 → 100644
......@@ -15,36 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.graphdb.titan0.Titan0Database;
import org.apache.commons.configuration.Configuration;
import com.thinkaurelius.titan.core.TitanGraph;
package org.apache.atlas.repository.graphdb;
/**
* Temporary TitanGraphProvider to use until the graph database abstraction
* layer is fully in place. Delegates to the Titan 0.5.4 implementation. This
* will be removed once the abstraction layer is being used.
* Enumeration of vertex property cardinalities.
*
*/
public class TitanGraphProvider implements GraphProvider<TitanGraph> {
/* (non-Javadoc)
* @see org.apache.atlas.repository.graph.GraphProvider#get()
*/
@Override
public TitanGraph get() {
return Titan0Database.getGraphInstance();
public enum AtlasCardinality {
SINGLE(false),
LIST(true),
SET(true);
private boolean isMany;
AtlasCardinality(boolean isMany) {
this.isMany = isMany;
}
public static TitanGraph getGraphInstance() {
return Titan0Database.getGraphInstance();
}
public static Configuration getConfiguration() throws AtlasException {
return Titan0Database.getConfiguration();
public boolean isMany() {
return isMany;
}
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdge.java
......@@ -19,7 +19,7 @@
package org.apache.atlas.repository.graphdb;
/**
* Represent an edge in the graph
* Represent an edge in the graph.
*
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
......@@ -27,14 +27,14 @@ package org.apache.atlas.repository.graphdb;
public interface AtlasEdge<V, E> extends AtlasElement {
/**
* Gets the incoming vertex for this edge
* Gets the incoming vertex for this edge.
* @param in
* @return
*/
AtlasVertex<V, E> getInVertex();
/**
* Gets the outgoing vertex for this edge
* Gets the outgoing vertex for this edge.
*
* @param in
* @return
......@@ -55,6 +55,6 @@ public interface AtlasEdge<V, E> extends AtlasElement {
*
* @return
*/
public E getE();
E getE();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java
......@@ -123,9 +123,10 @@ public interface AtlasElement {
/**
* Creates a Jettison JSONObject from this Element
* Creates a Jettison JSONObject from this Element.
*
* @param propertyKeys The property keys at the root of the element to serialize. If null, then all keys are serialized.
* @param propertyKeys The property keys at the root of the element to serialize.
* If null, then all keys are serialized.
*/
JSONObject toJson(Set<String> propertyKeys) throws JSONException;
......@@ -156,7 +157,7 @@ public interface AtlasElement {
*
* @return
*/
public String getIdForDisplay();
String getIdForDisplay();
/**
* Whether or not an id has been assigned yet for this Element. This can happen if the element has been created
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
......@@ -19,7 +19,6 @@ package org.apache.atlas.repository.graphdb;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import java.util.Set;
import javax.script.ScriptException;
......@@ -27,7 +26,7 @@ import javax.script.ScriptException;
import org.apache.atlas.typesystem.types.IDataType;
/**
* Represents a graph
* Represents a graph.
*
* @param <V> vertex implementation class
* @param <E> edge implementation class
......@@ -35,7 +34,7 @@ import org.apache.atlas.typesystem.types.IDataType;
public interface AtlasGraph<V, E> {
/**
* Adds an edge to the graph
* Adds an edge to the graph.
*
* @param outVertex
* @param inVertex
......@@ -45,14 +44,14 @@ public interface AtlasGraph<V, E> {
AtlasEdge<V, E> addEdge(AtlasVertex<V, E> outVertex, AtlasVertex<V, E> inVertex, String label);
/**
* Adds a vertex to the graph
* Adds a vertex to the graph.
*
* @return
*/
AtlasVertex<V, E> addVertex();
/**
* Removes the specified edge from the graph
* Removes the specified edge from the graph.
*
* @param edge
*/
......@@ -136,18 +135,21 @@ public interface AtlasGraph<V, E> {
Iterable<AtlasVertex<V, E>> getVertices(String key, Object value);
/**
* Creates a graph query
* Creates a graph query.
*
* @return
*/
AtlasGraphQuery<V, E> query();
/**
* Creates an index query
* Creates an index query.
*
* @param indexName index name
* @param queryString the query
*
* @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html">Elastic Search Reference</a> for query syntax
* @see <a
* href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html">
* Elastic Search Reference</a> for query syntax
*/
AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString);
......@@ -176,14 +178,14 @@ public interface AtlasGraph<V, E> {
/**
* Deletes all data in the graph. May or may not delete
* the indices, depending on the what the underlying graph supports.
*
*
* For testing only.
*
*
*/
void clear();
/**
* Converts the graph to gson and writes it to the specified stream
* Converts the graph to gson and writes it to the specified stream.
*
* @param os
* @throws IOException
......@@ -193,59 +195,6 @@ public interface AtlasGraph<V, E> {
//the following methods insulate Atlas from the details
//of the interaction with Gremlin
/**
*
* When we construct Gremlin select queries, the information we request
* is grouped by the vertex the information is coming from. Each vertex
* is assigned a column name which uniquely identifies it. The queries
* are specially formatted so that the value associated with each of
* these column names is an array with the various things we need
* about that particular vertex. The query evaluator creates a mapping
* that knows what index each bit of information is stored at within
* this array.
* <p/>
* When we execute a Gremlin query, the exact java objects we get
* back vary depending on whether Gremlin 2 or Gremlin 3 is being used.
* This method takes as input a raw row result that was obtained by
* executing a Gremlin query and extracts the value that was found
* at the given index in the array for the given column name.
* <p/>
* If the value found is a vertex or edge, it is automatically converted
* to an AtlasVertex/AtlasEdge.
*
* @param rowValue the raw row value that was returned by Gremin
* @param colName the column name to use
* @param idx the index of the value within the column to retrieve.
*
*/
Object getGremlinColumnValue(Object rowValue, String colName, int idx);
/**
* When Gremlin queries are executed, they return
* Vertex and Edge objects that are specific to the underlying
* graph database. This method provides a way to convert these
* objects back into the AtlasVertex/AtlasEdge objects that
* Atlas requires.
*
* @param rawValue the value that was obtained from Gremlin
* @return either an AtlasEdge, an AtlasVertex, or the original
* value depending on whether the rawValue represents an edge,
* vertex, or something else.
*
*/
Object convertGremlinValue(Object rawValue);
/**
* Gremlin 2 and Gremlin 3 represent the results of "path"
* queries differently. This method takes as input the
* path from Gremlin and returns the list of objects in the path.
*
* @param rawValue
* @return
*/
List<Object> convertPathQueryResultToList(Object rawValue);
/**
* This method is used in the generation of queries. It is used to
* convert property values from the value that is stored in the graph
......@@ -256,7 +205,7 @@ public interface AtlasGraph<V, E> {
* @return
*/
String generatePersisentToLogicalConversionExpression(String valueExpr, IDataType<?> type);
/**
* Indicates whether or not stored values with the specified type need to be converted
* within generated gremlin queries before they can be compared with literal values.
......@@ -292,7 +241,7 @@ public interface AtlasGraph<V, E> {
* @return
*/
String getInitialIndexedPredicate();
/**
* As an optimization, a graph database implementation may want to retrieve additional
* information about the query results. For example, in the IBM Graph implementation,
......@@ -303,12 +252,25 @@ public interface AtlasGraph<V, E> {
String getOutputTransformationPredicate(boolean isSelect, boolean isPath);
/**
* Executes a gremlin query, returns an object with the raw
* result.
* Executes a Gremlin script, returns an object with the result.
*
* @param gremlinQuery
* @param isPath whether this is a path query
*
* @return the result from executing the script
*
* @throws ScriptException
*/
Object executeGremlinScript(String query, boolean isPath) throws ScriptException;
/**
* Convenience method to check whether the given property is
* a multi-property.
*
* @param name
* @return
*/
Object executeGremlinScript(String gremlinQuery) throws ScriptException;
boolean isMultiProperty(String name);
}
\ No newline at end of file
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndex.java
......@@ -21,7 +21,7 @@ package org.apache.atlas.repository.graphdb;
import java.util.Set;
/**
* Represents a graph index on the database
* Represents a graph index on the database.
*/
public interface AtlasGraphIndex {
......@@ -39,14 +39,14 @@ public interface AtlasGraphIndex {
boolean isCompositeIndex();
/**
* Indicates if the index applies to edges
* Indicates if the index applies to edges.
*
* @return
*/
boolean isEdgeIndex();
/**
* Indicates if the index applies to vertices
* Indicates if the index applies to vertices.
*
* @return
*/
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
......@@ -18,28 +18,14 @@
package org.apache.atlas.repository.graphdb;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.types.Multiplicity;
import java.util.List;
/**
* Management interface for a graph
* Management interface for a graph.
*
*/
public interface AtlasGraphManagement {
public static final Set<String> MULTIPLICITY_MANY_PROPERTY_KEYS =
Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(
Constants.SUPER_TYPES_PROPERTY_KEY,
Constants.TRAIT_NAMES_PROPERTY_KEY )));
/**
* Checks whether a property with the given key has been defined in the graph schema.
*
......@@ -49,25 +35,7 @@ public interface AtlasGraphManagement {
boolean containsPropertyKey(String key);
/**
* Creates a mixed Vertex index for the graph
*
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
void buildMixedVertexIndex(String index, String backingIndex);
/**
* Creates a mixed Edge index for the graph
*
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
void buildMixedEdgeIndex(String index, String backingIndex);
/**
* Creates a full text index for the given property
* Creates a full text index for the given property.
*
* @param indexName the name of the index to create
* @param propertyKey full text property to index
......@@ -92,7 +60,7 @@ public interface AtlasGraphManagement {
* @param cardinality
* @return
*/
AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, Multiplicity multiplicity);
AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality);
/**
* @param propertyKey
......@@ -107,7 +75,24 @@ public interface AtlasGraphManagement {
* @param propertyKey
* @param isUnique
*/
void createCompositeIndex(String propertyName, AtlasPropertyKey propertyKey, boolean isUnique);
void createExactMatchIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys);
/**
* Looks up the index with the specified name in the graph. Returns null if
* there is no index with the given name.
*
* @param edgeIndex
* @return
*/
AtlasGraphIndex getGraphIndex(String indexName);
/**
* Creates a mixed Vertex index for the graph.
*
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
void createVertexIndex(String name, String backingIndex, List<AtlasPropertyKey> propertyKeys);
/**
* Adds a property key to the given index in the graph.
......@@ -115,16 +100,14 @@ public interface AtlasGraphManagement {
* @param vertexIndex
* @param propertyKey
*/
void addIndexKey(String vertexIndex, AtlasPropertyKey propertyKey);
void addVertexIndexKey(String vertexIndex, AtlasPropertyKey propertyKey);
/**
* Looks up the index with the specified name in the graph. Returns null if
* there is no index with the given name.
* Creates a mixed Edge index for the graph.
*
* @param edgeIndex
* @return
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
AtlasGraphIndex getGraphIndex(String indexName);
void createEdgeIndex(String index, String backingIndex);
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java
......@@ -21,8 +21,6 @@ package org.apache.atlas.repository.graphdb;
import java.util.Collection;
import java.util.List;
import org.apache.atlas.AtlasException;
/**
* Represents a query against the graph within the context of the
* current transaction.
......@@ -30,7 +28,7 @@ import org.apache.atlas.AtlasException;
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
*/
public interface AtlasGraphQuery<V,E> {
public interface AtlasGraphQuery<V, E> {
/**
* Adds a predicate that the returned vertices must have the specified
......@@ -41,7 +39,7 @@ public interface AtlasGraphQuery<V,E> {
* @param value
* @return
*/
AtlasGraphQuery<V,E> has(String propertyKey, Object value);
AtlasGraphQuery<V, E> has(String propertyKey, Object value);
/**
* Adds a predicate that the returned vertices must have the specified
......@@ -52,7 +50,7 @@ public interface AtlasGraphQuery<V,E> {
* @param value
* @return
*/
AtlasGraphQuery<V,E> in(String propertyKey, Collection<? extends Object> values);
AtlasGraphQuery<V, E> in(String propertyKey, Collection<? extends Object> values);
/**
......@@ -71,7 +69,7 @@ public interface AtlasGraphQuery<V,E> {
* @param value
* @return
*/
AtlasGraphQuery<V,E> has(String propertyKey, ComparisionOperator compMethod, Object values);
AtlasGraphQuery<V, E> has(String propertyKey, ComparisionOperator compMethod, Object values);
/**
* Adds a predicate that the vertices returned must satisfy the
......@@ -80,17 +78,20 @@ public interface AtlasGraphQuery<V,E> {
* @param childQueries
* @return
*/
AtlasGraphQuery<V,E> or(List<AtlasGraphQuery<V,E>> childQueries);
AtlasGraphQuery<V, E> or(List<AtlasGraphQuery<V, E>> childQueries);
/**
* Creates a child query that can be used to add "or" conditions
* Creates a child query that can be used to add "or" conditions.
*
* @return
*/
AtlasGraphQuery<V,E> createChildQuery();
AtlasGraphQuery<V, E> createChildQuery();
public static enum ComparisionOperator {
/**
* Comparison operators that can be used in an AtlasGraphQuery.
*/
enum ComparisionOperator {
GREATER_THAN_EQUAL,
EQUAL,
LESS_THAN_EQUAL,
......@@ -106,8 +107,8 @@ public interface AtlasGraphQuery<V,E> {
AtlasGraphQuery<V, E> addConditionsFrom(AtlasGraphQuery<V, E> otherQuery);
/**
* Whether or not this is a child query
*
* Whether or not this is a child query.
*
* @return
*/
boolean isChildQuery();
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
......@@ -44,7 +44,7 @@ public interface AtlasIndexQuery<V, E> {
public interface Result<V, E> {
/**
* Gets the vertex for this result
* Gets the vertex for this result.
*/
AtlasVertex<V, E> getVertex();
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasPropertyKey.java
......@@ -19,12 +19,12 @@
package org.apache.atlas.repository.graphdb;
/**
* Represent a property key
* Represent a property key.
*
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
*/
public interface AtlasPropertyKey {
String getName();
AtlasCardinality getCardinality();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graphdb;
/**
* Represents a Vertex
* Represents a Vertex.
*
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java
......@@ -27,7 +27,7 @@ package org.apache.atlas.repository.graphdb;
public interface AtlasVertexQuery<V, E> {
/**
* Specifies the edge direction that should be query
* Specifies the edge direction that should be query.
*
* @param queryDirection
* @return
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graphdb;
/**
* Represents a graph database
* Represents a graph database.
*
* @param <V> vertex class used by the graph database
* @param <E> edge class used by the graph database
......@@ -32,13 +32,13 @@ public interface GraphDatabase<V, E> {
boolean isGraphLoaded();
/**
* Gets the graph, loading it if it has not been loaded already
* Gets the graph, loading it if it has not been loaded already.
* @return
*/
AtlasGraph<V, E> getGraph();
/**
* Sets things up so that getGraph() will return a graph that can be used for running
* Sets things up so that getGraph() will return a graph that can be used for running
* tests.
*/
void initializeTestGraph();
......@@ -46,5 +46,5 @@ public interface GraphDatabase<V, E> {
/**
* Removes the test graph that was created.
*/
void removeTestGraph();
}
\ No newline at end of file
void cleanup();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GremlinVersion.java
......@@ -19,7 +19,7 @@
package org.apache.atlas.repository.graphdb;
/**
* Enumeration of the supported versions of Gremlin
* Enumeration of the supported versions of Gremlin.
*
*
*/
......
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java
......@@ -31,17 +31,17 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
* @param <V>
* @param <E>
*/
public interface NativeTitanGraphQuery<V,E> {
public interface NativeTitanGraphQuery<V, E> {
/**
* Executes the graph query
* Executes the graph query.
* @return
*/
Iterable<AtlasVertex<V,E>> vertices();
Iterable<AtlasVertex<V, E>> vertices();
/**
* Adds an in condition to the query
* Adds an in condition to the query.
*
* @param propertyName
* @param values
......@@ -49,7 +49,7 @@ public interface NativeTitanGraphQuery<V,E> {
void in(String propertyName, Collection<? extends Object> values);
/**
* Adds a has condition to the query
* Adds a has condition to the query.
*
* @param propertyName
* @param op
......
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanQueryFactory.java
......@@ -24,11 +24,11 @@ package org.apache.atlas.repository.graphdb.titan.query;
* @param <V>
* @param <E>
*/
public interface NativeTitanQueryFactory<V,E> {
public interface NativeTitanQueryFactory<V, E> {
/**
* Creates a NativeTitanGraphQuery
* Creates a NativeTitanGraphQuery.
* @return
*/
NativeTitanGraphQuery<V,E> createNativeTitanQuery();
NativeTitanGraphQuery<V, E> createNativeTitanQuery();
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java
......@@ -84,51 +84,51 @@ import org.slf4j.LoggerFactory;
*
*
*/
public abstract class TitanGraphQuery<V,E> implements AtlasGraphQuery<V, E> {
public abstract class TitanGraphQuery<V, E> implements AtlasGraphQuery<V, E> {
private final Logger LOG = LoggerFactory.getLogger(TitanGraphQuery.class);
protected final AtlasGraph<V,E> graph_;
private final OrCondition queryCondition_ = new OrCondition();
private final boolean isChildQuery_;
private static final Logger LOG = LoggerFactory.getLogger(TitanGraphQuery.class);
protected final AtlasGraph<V, E> graph;
private final OrCondition queryCondition = new OrCondition();
private final boolean isChildQuery;
protected abstract NativeTitanQueryFactory<V, E> getQueryFactory();
/**
* Creates a TitanGraphQuery
* Creates a TitanGraphQuery.
*
* @param graph
*/
public TitanGraphQuery(AtlasGraph<V,E> graph) {
graph_ = graph;
isChildQuery_ = false;
public TitanGraphQuery(AtlasGraph<V, E> graph) {
this.graph = graph;
this.isChildQuery = false;
}
/**
* Creates a TitanGraphQuery
* Creates a TitanGraphQuery.
*
* @param graph
* @param isChildQuery
*/
public TitanGraphQuery(AtlasGraph<V,E> graph, boolean isChildQuery) {
graph_ = graph;
isChildQuery_ = isChildQuery;
public TitanGraphQuery(AtlasGraph<V, E> graph, boolean isChildQuery) {
this.graph = graph;
this.isChildQuery = isChildQuery;
}
@Override
public AtlasGraphQuery<V, E> has(String propertyKey, Object value) {
queryCondition_.andWith(new HasPredicate(propertyKey, ComparisionOperator.EQUAL, value));
queryCondition.andWith(new HasPredicate(propertyKey, ComparisionOperator.EQUAL, value));
return this;
}
@Override
public Iterable<AtlasVertex<V, E>> vertices() {
LOG.debug("Executing: " );
LOG.debug(queryCondition_.toString());
LOG.debug("Executing: ");
LOG.debug(queryCondition.toString());
//compute the overall result by unioning the results from all of the
//AndConditions together.
Set<AtlasVertex<V, E>> result = new HashSet<>();
for(AndCondition andExpr : queryCondition_.getAndTerms()) {
NativeTitanGraphQuery<V,E> andQuery = andExpr.create(getQueryFactory());
for(AtlasVertex<V,E> vertex : andQuery.vertices()) {
for(AndCondition andExpr : queryCondition.getAndTerms()) {
NativeTitanGraphQuery<V, E> andQuery = andExpr.create(getQueryFactory());
for(AtlasVertex<V, E> vertex : andQuery.vertices()) {
result.add(vertex);
}
}
......@@ -138,14 +138,14 @@ public abstract class TitanGraphQuery<V,E> implements AtlasGraphQuery<V, E> {
@Override
public AtlasGraphQuery<V, E> has(String propertyKey, ComparisionOperator operator,
Object value) {
queryCondition_.andWith(new HasPredicate(propertyKey, operator, value));
queryCondition.andWith(new HasPredicate(propertyKey, operator, value));
return this;
}
@Override
public AtlasGraphQuery<V, E> in(String propertyKey, Collection<? extends Object> values) {
queryCondition_.andWith(new InPredicate(propertyKey, values));
queryCondition.andWith(new InPredicate(propertyKey, values));
return this;
}
......@@ -159,31 +159,31 @@ public abstract class TitanGraphQuery<V,E> implements AtlasGraphQuery<V, E> {
OrCondition overallChildQuery = new OrCondition(false);
for(AtlasGraphQuery<V, E> atlasChildQuery : childQueries) {
if(! atlasChildQuery.isChildQuery()) {
if (!atlasChildQuery.isChildQuery()) {
throw new IllegalArgumentException(atlasChildQuery + " is not a child query");
}
TitanGraphQuery<V,E> childQuery = (TitanGraphQuery<V,E>)atlasChildQuery;
TitanGraphQuery<V, E> childQuery = (TitanGraphQuery<V, E>)atlasChildQuery;
overallChildQuery.orWith(childQuery.getOrCondition());
}
queryCondition_.andWith(overallChildQuery);
queryCondition.andWith(overallChildQuery);
return this;
}
private OrCondition getOrCondition() {
return queryCondition_;
return queryCondition;
}
@Override
public AtlasGraphQuery<V, E> addConditionsFrom(AtlasGraphQuery<V, E> otherQuery) {
TitanGraphQuery<V, E> childQuery = (TitanGraphQuery<V, E>)otherQuery;
queryCondition_.andWith(childQuery.getOrCondition());
queryCondition.andWith(childQuery.getOrCondition());
return this;
}
@Override
public boolean isChildQuery() {
return isChildQuery_;
return isChildQuery;
}
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java
......@@ -32,27 +32,27 @@ import org.apache.atlas.repository.graphdb.titan.query.NativeTitanQueryFactory;
*/
public class AndCondition {
private List<QueryPredicate> children_ = new ArrayList<>();
private List<QueryPredicate> children = new ArrayList<>();
public AndCondition() {
}
/**
* Adds a query predicate that must be met by vertices
* Adds a query predicate that must be met by vertices.
* @param predicate
*/
public void andWith(QueryPredicate predicate) {
children_.add(predicate);
children.add(predicate);
}
/**
* Adds multiple predicates that much be met by the vertices
* Adds multiple predicates that much be met by the vertices.
*
* @param predicates
*/
public void andWith(List<QueryPredicate> predicates) {
children_.addAll(predicates);
children.addAll(predicates);
}
/**
......@@ -62,17 +62,17 @@ public class AndCondition {
*/
public AndCondition copy() {
AndCondition builder = new AndCondition();
builder.children_.addAll(children_);
builder.children.addAll(children);
return builder;
}
/**
* Gets the query predicates
* Gets the query predicates.
*
* @return
*/
public List<QueryPredicate> getTerms() {
return children_;
return children;
}
/**
......@@ -81,9 +81,9 @@ public class AndCondition {
* @param graph
* @return
*/
public <V,E> NativeTitanGraphQuery<V, E> create(NativeTitanQueryFactory<V,E> factory) {
public <V, E> NativeTitanGraphQuery<V, E> create(NativeTitanQueryFactory<V, E> factory) {
NativeTitanGraphQuery<V, E> query = factory.createNativeTitanQuery();
for (QueryPredicate predicate : children_) {
for (QueryPredicate predicate : children) {
predicate.addTo(query);
}
return query;
......@@ -91,6 +91,6 @@ public class AndCondition {
@Override
public String toString() {
return "AndExpr [predicates=" + children_ + "]";
return "AndExpr [predicates=" + children + "]";
}
}
\ No newline at end of file
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java
......@@ -26,25 +26,24 @@ import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
*/
public class HasPredicate implements QueryPredicate {
private String propertyName_;
private ComparisionOperator op_;
private Object value_;
private String propertyName;
private ComparisionOperator op;
private Object value;
public HasPredicate(String propertyName, ComparisionOperator op, Object value) {
super();
propertyName_ = propertyName;
op_ = op;
value_ = value;
this.propertyName = propertyName;
this.op = op;
this.value = value;
}
@Override
public void addTo(NativeTitanGraphQuery query) {
query.has(propertyName_, op_, value_);
query.has(propertyName, op, value);
}
@Override
public String toString() {
return "HasTerm [propertyName_=" + propertyName_ + ", op_=" + op_ + ", value_=" + value_ + "]";
return "HasTerm [propertyName=" + propertyName + ", op=" + op + ", value=" + value + "]";
}
}
\ No newline at end of file
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/InPredicate.java
......@@ -27,23 +27,23 @@ import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
*/
public class InPredicate implements QueryPredicate {
private String propertyName_;
private Collection<? extends Object> values_;
private String propertyName;
private Collection<? extends Object> values;
public InPredicate(String propertyName, Collection<? extends Object> values) {
super();
propertyName_ = propertyName;
values_ = values;
this.propertyName = propertyName;
this.values = values;
}
@Override
public void addTo(NativeTitanGraphQuery query) {
query.in(propertyName_, values_);
query.in(propertyName, values);
}
@Override
public String toString() {
return "InPredicate [propertyName_=" + propertyName_ + ", values_=" + values_ + "]";
return "InPredicate [propertyName=" + propertyName + ", values=" + values + "]";
}
}
\ No newline at end of file
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/OrCondition.java
......@@ -32,20 +32,20 @@ import java.util.List;
*/
public class OrCondition {
private List<AndCondition> children_;
private List<AndCondition> children;
public OrCondition() {
this(true);
}
private OrCondition(List<AndCondition> children) {
children_ = children;
this.children = children;
}
public OrCondition(boolean addInitialTerm) {
children_ = new ArrayList<AndCondition>();
this.children = new ArrayList<AndCondition>();
if (addInitialTerm) {
children_.add(new AndCondition());
children.add(new AndCondition());
}
}
......@@ -58,18 +58,18 @@ public class OrCondition {
*/
public void andWith(QueryPredicate predicate) {
for (AndCondition child : children_) {
for (AndCondition child : children) {
child.andWith(predicate);
}
}
public List<AndCondition> getAndTerms() {
return children_;
return children;
}
/**
* Updates this OrCondition in place so that it matches vertices that satisfy the current
* OrCondition AND that satisfy the provided OrCondition
* Updates this OrCondition in place so that it matches vertices that satisfy the current
* OrCondition AND that satisfy the provided OrCondition.
*
* @param other
*/
......@@ -98,30 +98,30 @@ public class OrCondition {
List<AndCondition> expandedExpressionChildren = new ArrayList<AndCondition>();
for (AndCondition otherExprTerm : other.getAndTerms()) {
for (AndCondition currentExpr : children_) {
for (AndCondition currentExpr : children) {
AndCondition currentAndConditionCopy = currentExpr.copy();
currentAndConditionCopy.andWith(otherExprTerm.getTerms());
expandedExpressionChildren.add(currentAndConditionCopy);
}
}
children_ = expandedExpressionChildren;
children = expandedExpressionChildren;
}
/**
* Updates this OrCondition in place so that it matches vertices that satisfy the current
* OrCondition OR that satisfy the provided OrCondition
* Updates this OrCondition in place so that it matches vertices that satisfy the current
* OrCondition OR that satisfy the provided OrCondition.
*
* @param other
*/
public void orWith(OrCondition other) {
children_.addAll(other.getAndTerms());
children.addAll(other.getAndTerms());
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("OrCondition [andExprs=");
Iterator<AndCondition> it = children_.iterator();
Iterator<AndCondition> it = children.iterator();
while (it.hasNext()) {
AndCondition andExpr = it.next();
builder.append(andExpr.toString());
......@@ -133,4 +133,4 @@ public class OrCondition {
return builder.toString();
}
}
\ No newline at end of file
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/QueryPredicate.java
......@@ -20,7 +20,7 @@ package org.apache.atlas.repository.graphdb.titan.query.expr;
import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
/**
* Represents a predicate in an AndExpression
* Represents a predicate in an AndExpression.
*/
public interface QueryPredicate {
......@@ -30,4 +30,4 @@ public interface QueryPredicate {
* @param query
*/
void addTo(NativeTitanGraphQuery query);
}
\ No newline at end of file
}
graphdb/graphdb-impls/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb</artifactId>
<version>0.8-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>atlas-graphdb-impls</artifactId>
<!-- Convenience dependency project that allows
the dependency on the correct implementation class
to be configured through profiles. In implementation
specific profiles, the dependency on this project
should be configured to exclude all but the
proper dependency
-->
<description>Apache Atlas Graph Database Implementation Dependencies</description>
<name>Apache Atlas Graph Database Implementation Dependencies</name>
<packaging>pom</packaging>
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Additional dependencies should be added here
as more implementations are introduced. In the
profile corresponding to the implementation in
the root pom.xml, all dependencies except for
that one should be excluded. -->
</dependencies>
</project>
graphdb/pom.xml
......@@ -29,10 +29,14 @@
<description>Apache Atlas Graph Database Projects</description>
<name>Apache Atlas Graph Database Projects</name>
<packaging>pom</packaging>
<properties>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
</properties>
<modules>
<module>api</module>
<module>titan0</module>
<module>common</module>
<module>graphdb-impls</module>
</modules>
</project>
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
......@@ -18,9 +18,11 @@
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.tinkerpop.blueprints.Edge;
......@@ -39,8 +41,8 @@ public final class GraphDbObjectFactory {
/**
* Creates a Titan0Edge that corresponds to the given Gremlin Edge.
*
* @param source
* @return
* @param graph The graph the edge should be created in
* @param source The gremlin edge
*/
public static Titan0Edge createEdge(Titan0Graph graph, Edge source) {
......@@ -52,9 +54,8 @@ public final class GraphDbObjectFactory {
/**
* Creates a Titan0GraphQuery that corresponds to the given GraphQuery.
* @param source
*
* @return
* @param graph the graph that is being quried
*/
public static Titan0GraphQuery createQuery(Titan0Graph graph) {
......@@ -64,8 +65,8 @@ public final class GraphDbObjectFactory {
/**
* Creates a Titan0Vertex that corresponds to the given Gremlin Vertex.
*
* @param source
* @return
* @param graph The graph that contains the vertex
* @param source the Gremlin vertex
*/
public static Titan0Vertex createVertex(Titan0Graph graph, Vertex source) {
......@@ -76,8 +77,8 @@ public final class GraphDbObjectFactory {
}
/**
* @param propertyKey
* @return
* @param propertyKey The Gremlin propertyKey.
*
*/
public static Titan0PropertyKey createPropertyKey(PropertyKey propertyKey) {
if (propertyKey == null) {
......@@ -87,7 +88,7 @@ public final class GraphDbObjectFactory {
}
/**
* @param index
* @param index The gremlin index.
* @return
*/
public static AtlasGraphIndex createGraphIndex(TitanGraphIndex index) {
......@@ -97,4 +98,20 @@ public final class GraphDbObjectFactory {
return new Titan0GraphIndex(index);
}
/**
* Converts a Multiplicity to a Cardinality.
*
* @param cardinality
* @return
*/
public static AtlasCardinality createCardinality(Cardinality cardinality) {
if (cardinality == Cardinality.SINGLE) {
return AtlasCardinality.SINGLE;
} else if (cardinality == Cardinality.LIST) {
return AtlasCardinality.LIST;
}
return AtlasCardinality.SET;
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Element.java
......@@ -72,7 +72,20 @@ public class Titan0Element<T extends Element> implements AtlasElement {
@Override
public <U> U getProperty(String propertyName, Class<U> clazz) {
return (U)convert(wrappedElement.getProperty(propertyName), clazz);
Object rawValue = wrappedElement.getProperty(propertyName);
if (rawValue == null) {
return null;
}
if (AtlasEdge.class == clazz) {
return (U)graph.getEdge(rawValue.toString());
}
if (AtlasVertex.class == clazz) {
return (U)graph.getVertex(rawValue.toString());
}
return (U)rawValue;
}
/**
......@@ -136,7 +149,7 @@ public class Titan0Element<T extends Element> implements AtlasElement {
@Override
public boolean equals(Object other) {
if(other == null) {
if (other == null) {
return false;
}
if (other.getClass() != getClass()) {
......@@ -154,9 +167,8 @@ public class Titan0Element<T extends Element> implements AtlasElement {
@Override
public boolean exists() {
try {
return ! ((TitanElement)wrappedElement).isRemoved();
}
catch(IllegalStateException e) {
return !((TitanElement)wrappedElement).isRemoved();
} catch(IllegalStateException e) {
return false;
}
......@@ -192,18 +204,6 @@ public class Titan0Element<T extends Element> implements AtlasElement {
return getId().toString();
}
private <T> T convert(Object propertyValue, Class<T> clazz) {
if(propertyValue == null) {
return null;
}
if(AtlasEdge.class.isAssignableFrom(clazz)) {
return (T)graph.getEdge(propertyValue.toString());
}
if(AtlasVertex.class.isAssignableFrom(clazz)) {
return (T)graph.getVertex(propertyValue.toString());
}
return (T)propertyValue;
}
@Override
......@@ -211,14 +211,13 @@ public class Titan0Element<T extends Element> implements AtlasElement {
List<String> value = getListProperty(propertyName);
if(value == null) {
if (value == null) {
return null;
}
if(AtlasEdge.class.isAssignableFrom(elementType)) {
if (AtlasEdge.class == elementType) {
return (List<V>)Lists.transform(value, new Function<String,AtlasEdge>(){
return (List<V>)Lists.transform(value, new Function<String, AtlasEdge>(){
@Override
public AtlasEdge apply(String input) {
......@@ -227,9 +226,9 @@ public class Titan0Element<T extends Element> implements AtlasElement {
});
}
if(AtlasVertex.class.isAssignableFrom(elementType)) {
if (AtlasVertex.class == elementType) {
return (List<V>)Lists.transform(value, new Function<String,AtlasVertex>(){
return (List<V>)Lists.transform(value, new Function<String, AtlasVertex>(){
@Override
public AtlasVertex apply(String input) {
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
......@@ -19,8 +19,14 @@ package org.apache.atlas.repository.graphdb.titan0;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.script.Bindings;
......@@ -42,9 +48,13 @@ import org.apache.atlas.utils.IteratorToIterableAdapter;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
......@@ -58,8 +68,25 @@ import com.tinkerpop.pipes.util.structures.Row;
*/
public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
public Titan0Graph() {
private final Set<String> multiProperties;
public Titan0Graph() {
//determine multi-properties once at startup
TitanManagement mgmt = null;
try {
mgmt = Titan0GraphDatabase.getGraphInstance().getManagementSystem();
Iterable<PropertyKey> keys = mgmt.getRelationTypes(PropertyKey.class);
multiProperties = Collections.synchronizedSet(new HashSet<String>());
for(PropertyKey key : keys) {
if (key.getCardinality() != Cardinality.SINGLE) {
multiProperties.add(key.getName());
}
}
} finally {
if (mgmt != null) {
mgmt.rollback();
}
}
}
@Override
......@@ -74,6 +101,7 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
}
}
@Override
public AtlasGraphQuery<Titan0Vertex, Titan0Edge> query() {
......@@ -134,7 +162,7 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
@Override
public AtlasGraphManagement getManagementSystem() {
return new Titan0DatabaseManager(getGraph().getManagementSystem());
return new Titan0GraphManagement(this, getGraph().getManagementSystem());
}
@Override
......@@ -170,20 +198,31 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
return wrapVertices(result);
}
@Override
public Object getGremlinColumnValue(Object rowValue, String colName, int idx) {
Row<List> rV = (Row<List>) rowValue;
Object value = rV.getColumn(colName).get(idx);
return convertGremlinValue(value);
}
private Object convertGremlinValue(Object rawValue) {
@Override
public Object convertGremlinValue(Object rawValue) {
if (rawValue instanceof Vertex) {
return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue);
}
if (rawValue instanceof Edge) {
} else if (rawValue instanceof Edge) {
return GraphDbObjectFactory.createEdge(this, (Edge) rawValue);
} else if (rawValue instanceof Row) {
Row rowValue = (Row)rawValue;
Map<String, Object> result = new HashMap<>(rowValue.size());
List<String> columnNames = rowValue.getColumnNames();
for(int i = 0; i < rowValue.size(); i++) {
String key = columnNames.get(i);
Object value = convertGremlinValue(rowValue.get(i));
result.put(key, value);
}
return result;
} else if (rawValue instanceof List) {
return Lists.transform((List)rawValue, new Function<Object, Object>() {
@Override
public Object apply(Object input) {
return convertGremlinValue(input);
}
});
} else if (rawValue instanceof Collection) {
throw new UnsupportedOperationException("Unhandled collection type: " + rawValue.getClass());
}
return rawValue;
}
......@@ -194,11 +233,11 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
return GremlinVersion.TWO;
}
@Override
public List<Object> convertPathQueryResultToList(Object rawValue) {
private List<Object> convertPathQueryResultToList(Object rawValue) {
return (List<Object>) rawValue;
}
@Override
public void clear() {
TitanGraph graph = getGraph();
......@@ -211,7 +250,7 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
private TitanGraph getGraph() {
// return the singleton instance of the graph in the plugin
return Titan0Database.getGraphInstance();
return Titan0GraphDatabase.getGraphInstance();
}
@Override
......@@ -219,15 +258,24 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
GraphSONWriter.outputGraph(getGraph(), os);
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraph#executeGremlinScript(java.
* lang.String)
*/
@Override
public Object executeGremlinScript(String gremlinQuery) throws ScriptException {
public Object executeGremlinScript(String query, boolean isPath) throws ScriptException {
Object result = executeGremlinScript(query);
if (isPath) {
List<Object> path = convertPathQueryResultToList(result);
List<Object> convertedResult = new ArrayList<>(path.size());
for(Object o : path) {
convertedResult.add(convertGremlinValue(o));
}
return convertedResult;
} else {
return convertGremlinValue(result);
}
}
private Object executeGremlinScript(String gremlinQuery) throws ScriptException {
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
......@@ -298,4 +346,14 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
}
});
}
@Override
public boolean isMultiProperty(String propertyName) {
return multiProperties.contains(propertyName);
}
public void addMultiProperties(Set<String> names) {
multiProperties.addAll(names);
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Database.java → graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphDatabase.java
......@@ -42,9 +42,9 @@ import com.thinkaurelius.titan.diskstorage.solr.Solr5Index;
/**
* Titan 0.5.4 implementation of GraphDatabase.
*/
public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
public class Titan0GraphDatabase implements GraphDatabase<Titan0Vertex, Titan0Edge> {
private static final Logger LOG = LoggerFactory.getLogger(Titan0Database.class);
private static final Logger LOG = LoggerFactory.getLogger(Titan0GraphDatabase.class);
/**
* Constant for the configuration property that indicates the prefix.
......@@ -57,7 +57,8 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
public static final String INDEX_BACKEND_ES = "elasticsearch";
private static volatile TitanGraph graphInstance;
private static volatile Titan0Graph atlasGraphInstance = null;
private static volatile TitanGraph graphInstance = null;
public static Configuration getConfiguration() throws AtlasException {
Configuration configProperties = ApplicationProperties.get();
......@@ -102,7 +103,7 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
public static TitanGraph getGraphInstance() {
if (graphInstance == null) {
synchronized (Titan0Database.class) {
synchronized (Titan0GraphDatabase.class) {
if (graphInstance == null) {
Configuration config;
try {
......@@ -112,6 +113,7 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
}
graphInstance = TitanFactory.open(config);
atlasGraphInstance = new Titan0Graph();
validateIndexBackend(config);
}
}
......@@ -121,7 +123,7 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
public static void unload() {
synchronized (Titan0Database.class) {
synchronized (Titan0GraphDatabase.class) {
if (graphInstance == null) {
return;
}
......@@ -167,7 +169,7 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
// force graph loading up front to avoid bootstrapping
// issues
getGraphInstance();
return new Titan0Graph();
return atlasGraphInstance;
}
@Override
......@@ -178,27 +180,25 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
@Override
public void initializeTestGraph() {
//nothing to do
}
@Override
public void removeTestGraph() {
public void cleanup() {
try {
getGraphInstance().shutdown();
}
catch(Throwable t) {
} catch(Throwable t) {
LOG.warn("Could not shutdown test TitanGraph", t);
t.printStackTrace();
}
try {
TitanCleanup.clear(getGraphInstance());
}
catch(Throwable t) {
} catch(Throwable t) {
LOG.warn("Could not clear test TitanGraph", t);
t.printStackTrace();
}
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseManager.java → graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java
......@@ -17,10 +17,14 @@
*/
package org.apache.atlas.repository.graphdb.titan0;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -37,24 +41,23 @@ import com.tinkerpop.blueprints.Vertex;
/**
* Titan 0.5.4 implementation of AtlasGraphManagement.
*/
public class Titan0DatabaseManager implements AtlasGraphManagement {
public class Titan0GraphManagement implements AtlasGraphManagement {
private static final Logger LOG = LoggerFactory.getLogger(Titan0DatabaseManager.class);
private static final Logger LOG = LoggerFactory.getLogger(Titan0GraphManagement.class);
private TitanManagement management;
private final Titan0Graph graph;
private final TitanManagement management;
public Titan0DatabaseManager(TitanManagement managementSystem) {
private Set<String> newMultProperties = new HashSet<>();
management = managementSystem;
}
public Titan0GraphManagement(Titan0Graph graph, TitanManagement managementSystem) {
@Override
public void buildMixedVertexIndex(String index, String backingIndex) {
buildMixedIndex(index, Vertex.class, backingIndex);
this.graph = graph;
management = managementSystem;
}
@Override
public void buildMixedEdgeIndex(String index, String backingIndex) {
public void createEdgeIndex(String index, String backingIndex) {
buildMixedIndex(index, Edge.class, backingIndex);
}
......@@ -86,69 +89,63 @@ public class Titan0DatabaseManager implements AtlasGraphManagement {
@Override
public void commit() {
graph.addMultiProperties(newMultProperties);
newMultProperties.clear();
management.commit();
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraphManagement#makePropertyKey(
* java.lang.String, java.lang.Class,
* org.apache.atlas.typesystem.types.Multiplicity)
*/
@Override
public AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, Multiplicity multiplicity) {
public AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality) {
if (cardinality.isMany()) {
newMultProperties.add(propertyName);
}
PropertyKeyMaker propertyKeyBuilder = management.makePropertyKey(propertyName).dataType(propertyClass);
if (multiplicity != null) {
Cardinality cardinality = TitanObjectFactory.createCardinality(multiplicity);
propertyKeyBuilder.cardinality(cardinality);
if (cardinality != null) {
Cardinality titanCardinality = TitanObjectFactory.createCardinality(cardinality);
propertyKeyBuilder.cardinality(titanCardinality);
}
PropertyKey propertyKey = propertyKeyBuilder.make();
return GraphDbObjectFactory.createPropertyKey(propertyKey);
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraphManagement#getPropertyKey(
* java.lang.String)
*/
@Override
public AtlasPropertyKey getPropertyKey(String propertyName) {
return GraphDbObjectFactory.createPropertyKey(management.getPropertyKey(propertyName));
}
/*
* (non-Javadoc)
*
* @see org.apache.atlas.repository.graphdb.AtlasGraphManagement#
* createCompositeIndex(java.lang.String,
* org.apache.atlas.repository.graphdb.AtlasPropertyKey, boolean)
*/
@Override
public void createCompositeIndex(String propertyName, AtlasPropertyKey propertyKey, boolean enforceUniqueness) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class).addKey(titanKey);
public void createExactMatchIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for(AtlasPropertyKey key : propertyKeys) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
indexBuilder.addKey(titanKey);
}
if (enforceUniqueness) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraphManagement#addIndexKey(java
* .lang.String, org.apache.atlas.repository.graphdb.AtlasPropertyKey)
*/
@Override
public void addIndexKey(String indexName, AtlasPropertyKey propertyKey) {
public void createVertexIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for(AtlasPropertyKey key : propertyKeys) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
indexBuilder.addKey(titanKey);
}
indexBuilder.buildMixedIndex(backingIndex);
}
@Override
public void addVertexIndexKey(String indexName, AtlasPropertyKey propertyKey) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
TitanGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, titanKey);
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java
......@@ -59,7 +59,7 @@ public class Titan0IndexQuery implements AtlasIndexQuery<Titan0Vertex, Titan0Edg
private final class ResultImpl implements AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> {
private TitanIndexQuery.Result<Vertex> wrappedResult;
public ResultImpl(TitanIndexQuery.Result<Vertex> source) {
ResultImpl(TitanIndexQuery.Result<Vertex> source) {
wrappedResult = source;
}
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0PropertyKey.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import com.thinkaurelius.titan.core.PropertyKey;
......@@ -51,6 +52,11 @@ public class Titan0PropertyKey implements AtlasPropertyKey {
}
@Override
public AtlasCardinality getCardinality() {
return GraphDbObjectFactory.createCardinality(wrappedPropertyKey.getCardinality());
}
@Override
public boolean equals(Object other) {
if (!(other instanceof Titan0PropertyKey)) {
return false;
......@@ -66,4 +72,11 @@ public class Titan0PropertyKey implements AtlasPropertyKey {
return result;
}
@Override
public String toString() {
return wrappedPropertyKey.getName();
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Vertex.java
......@@ -22,7 +22,6 @@ import java.util.Collection;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
......@@ -62,7 +61,7 @@ public class Titan0Vertex extends Titan0Element<Vertex> implements AtlasVertex<T
@Override
public <T> T getProperty(String propertyName, Class<T> clazz) {
if (AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS.contains(propertyName)) {
if (graph.isMultiProperty(propertyName)) {
// throw exception in this case to be consistent with Titan 1.0.0
// behavior.
throw new IllegalStateException();
......@@ -78,7 +77,7 @@ public class Titan0Vertex extends Titan0Element<Vertex> implements AtlasVertex<T
// For consistency with Titan 1.0.0, treat sets of multiplicity many
// properties as adds. Handle this here since this is an uncommon
// occurrence.
if (AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS.contains(propertyName)) {
if (graph.isMultiProperty(propertyName)) {
addProperty(propertyName, value);
} else {
throw e;
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/TitanObjectFactory.java
......@@ -17,9 +17,9 @@
*/
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.typesystem.types.Multiplicity;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
......@@ -63,17 +63,18 @@ public final class TitanObjectFactory {
* @param multiplicity
* @return
*/
public static Cardinality createCardinality(Multiplicity multiplicity) {
public static Cardinality createCardinality(AtlasCardinality cardinality) {
switch(cardinality) {
if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
case SINGLE:
return Cardinality.SINGLE;
} else if (multiplicity == Multiplicity.COLLECTION) {
case LIST:
return Cardinality.LIST;
} else if (multiplicity == Multiplicity.SET) {
case SET:
return Cardinality.SET;
default:
throw new IllegalStateException("Unrecognized cardinality: " + cardinality);
}
// default to LIST as this is the most forgiving
return Cardinality.LIST;
}
public static PropertyKey createPropertyKey(AtlasPropertyKey key) {
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java
......@@ -22,7 +22,7 @@ import java.util.Collection;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
import org.apache.atlas.repository.graphdb.titan0.Titan0Database;
import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase;
import org.apache.atlas.repository.graphdb.titan0.Titan0Edge;
import org.apache.atlas.repository.graphdb.titan0.Titan0Graph;
import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex;
......@@ -34,30 +34,30 @@ import com.tinkerpop.blueprints.Compare;
/**
* Titan 0.5.4 implementation of NativeTitanGraphQuery.
*
*
* @author jeff
*
*/
public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Vertex,Titan0Edge> {
public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Vertex, Titan0Edge> {
private Titan0Graph graph_;
private TitanGraphQuery<?> query_;
private Titan0Graph graph;
private TitanGraphQuery<?> query;
public NativeTitan0GraphQuery(Titan0Graph graph) {
query_ = Titan0Database.getGraphInstance().query();
graph_ = graph;
query = Titan0GraphDatabase.getGraphInstance().query();
this.graph = graph;
}
@Override
public Iterable<AtlasVertex<Titan0Vertex,Titan0Edge>> vertices() {
Iterable it = query_.vertices();
return graph_.wrapVertices(it);
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices() {
Iterable it = query.vertices();
return graph.wrapVertices(it);
}
@Override
public void in(String propertyName, Collection<? extends Object> values) {
query_.has(propertyName, Contain.IN, values);
query.has(propertyName, Contain.IN, values);
}
......@@ -66,7 +66,7 @@ public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Verte
Compare c = getGremlinPredicate(op);
TitanPredicate pred = TitanPredicate.Converter.convert(c);
query_.has(propertyName, pred, value);
query.has(propertyName, pred, value);
}
private Compare getGremlinPredicate(ComparisionOperator op) {
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/Titan0GraphQuery.java
......@@ -28,7 +28,8 @@ import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex;
/**
* Titan 0.5.4 implementation of AtlasGraphQuery.
*/
public class Titan0GraphQuery extends TitanGraphQuery<Titan0Vertex,Titan0Edge> implements NativeTitanQueryFactory<Titan0Vertex,Titan0Edge> {
public class Titan0GraphQuery extends TitanGraphQuery<Titan0Vertex, Titan0Edge>
implements NativeTitanQueryFactory<Titan0Vertex, Titan0Edge> {
public Titan0GraphQuery(Titan0Graph graph, boolean isChildQuery) {
super(graph, isChildQuery);
......@@ -40,17 +41,17 @@ public class Titan0GraphQuery extends TitanGraphQuery<Titan0Vertex,Titan0Edge> i
@Override
public AtlasGraphQuery<Titan0Vertex, Titan0Edge> createChildQuery() {
return new Titan0GraphQuery((Titan0Graph)graph_, true);
return new Titan0GraphQuery((Titan0Graph)graph, true);
}
@Override
protected NativeTitanQueryFactory<Titan0Vertex,Titan0Edge> getQueryFactory() {
protected NativeTitanQueryFactory<Titan0Vertex, Titan0Edge> getQueryFactory() {
return this;
}
@Override
public NativeTitanGraphQuery<Titan0Vertex, Titan0Edge> createNativeTitanQuery() {
return new NativeTitan0GraphQuery((Titan0Graph)graph_);
return new NativeTitan0GraphQuery((Titan0Graph)graph);
}
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java
......@@ -19,14 +19,15 @@
package org.apache.atlas.repository.graphdb.titan0;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
......@@ -36,75 +37,49 @@ import org.testng.annotations.BeforeClass;
*/
public abstract class AbstractGraphDatabaseTest {
private static class RunnableWrapper implements Runnable {
private final Runnable r;
private Throwable exceptionThrown_ = null;
private RunnableWrapper(Runnable r) {
this.r = r;
}
@Override
public void run() {
try {
r.run();
}
catch(Throwable e) {
exceptionThrown_ = e;
}
}
public Throwable getExceptionThrown() {
return exceptionThrown_;
}
}
protected static final String WEIGHT_PROPERTY = "weight";
protected static final String TRAIT_NAMES = Constants.TRAIT_NAMES_PROPERTY_KEY;
protected static final String typeProperty = "__type";
protected static final String typeSystem = "typeSystem";
protected static final String TYPE_PROPERTY_NAME = "__type";
protected static final String TYPESYSTEM = "TYPESYSTEM";
/**
*
*/
private static final String BACKING_INDEX_NAME = "backing";
private AtlasGraph<?,?> graph = null;
private AtlasGraph<?, ?> graph = null;
@AfterMethod
public void commitGraph() {
//force any pending actions to be committed so we can be sure they don't cause errors.
pushChangesAndFlushCache();
graph.commit();
}
protected <V, E> void pushChangesAndFlushCache() {
AtlasGraph<V, E> graph = getGraph();
graph.commit();
}
public AbstractGraphDatabaseTest() {
super();
}
@BeforeClass
public static void createIndices() {
Titan0Database db = new Titan0Database();
Titan0GraphDatabase db = new Titan0GraphDatabase();
AtlasGraphManagement mgmt = db.getGraph().getManagementSystem();
if(mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
mgmt.buildMixedVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX);
if (mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
mgmt.createVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX,
Collections.<AtlasPropertyKey>emptyList());
}
mgmt.makePropertyKey("age13",Integer.class, Multiplicity.OPTIONAL);
createIndices(mgmt, "name", String.class, false, Multiplicity.REQUIRED);
createIndices(mgmt, WEIGHT_PROPERTY, Integer.class, false, Multiplicity.OPTIONAL);
createIndices(mgmt, "size15", String.class, false, Multiplicity.REQUIRED);
createIndices(mgmt, "typeName", String.class, false, Multiplicity.REQUIRED);
createIndices(mgmt, "__type", String.class, false, Multiplicity.REQUIRED);
createIndices(mgmt, Constants.GUID_PROPERTY_KEY, String.class, true, Multiplicity.REQUIRED);
createIndices(mgmt, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Multiplicity.SET);
createIndices(mgmt, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Multiplicity.SET);
mgmt.makePropertyKey("age13", Integer.class, AtlasCardinality.SINGLE);
createIndices(mgmt, "name", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, WEIGHT_PROPERTY, Integer.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "size15", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "typeName", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "__type", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, Constants.GUID_PROPERTY_KEY, String.class, true, AtlasCardinality.SINGLE);
createIndices(mgmt, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET);
createIndices(mgmt, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET);
mgmt.commit();
}
@AfterMethod
public void commitGraph() {
//force any pending actions to be committed so we can be sure they don't cause errors.
pushChangesAndFlushCache();
getGraph().commit();
}
@AfterClass
public static void cleanUp() {
Titan0Graph graph = new Titan0Graph();
......@@ -112,32 +87,31 @@ public abstract class AbstractGraphDatabaseTest {
}
private static void createIndices(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, Multiplicity cardinality) {
protected <V, E> void pushChangesAndFlushCache() {
getGraph().commit();
}
private static void createIndices(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality) {
if(management.containsPropertyKey(propertyName)) {
if (management.containsPropertyKey(propertyName)) {
//index was already created
return;
}
AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
try {
if(propertyClass != Integer.class) {
management.addIndexKey(BACKING_INDEX_NAME, key);
if (propertyClass != Integer.class) {
management.addVertexIndexKey(BACKING_INDEX_NAME, key);
}
}
catch(Throwable t) {
} catch(Throwable t) {
//ok
t.printStackTrace();
}
try {
//if(propertyClass != Integer.class) {
management.createCompositeIndex(propertyName, key, isUnique);
//}
}
catch(Throwable t) {
management.createExactMatchIndex(propertyName, isUnique, Collections.singletonList(key));
} catch(Throwable t) {
//ok
t.printStackTrace();
}
......@@ -145,19 +119,14 @@ public abstract class AbstractGraphDatabaseTest {
}
/**
*
*/
public AbstractGraphDatabaseTest() {
super();
}
protected final <V,E> AtlasGraph<V, E> getGraph() {
if(graph == null) {
protected final <V, E> AtlasGraph<V, E> getGraph() {
if (graph == null) {
graph = new Titan0Graph();
}
return (AtlasGraph<V,E>)graph;
return (AtlasGraph<V, E>)graph;
}
protected Titan0Graph getTitan0Graph() {
......@@ -166,23 +135,23 @@ public abstract class AbstractGraphDatabaseTest {
}
protected List<AtlasVertex> newVertices_ = new ArrayList<>();
protected List<AtlasVertex> newVertices = new ArrayList<>();
protected final <V, E> AtlasVertex<V, E> createVertex(AtlasGraph<V, E> graph) {
AtlasVertex<V,E> vertex = graph.addVertex();
newVertices_.add(vertex);
protected final <V, E> AtlasVertex<V, E> createVertex(AtlasGraph<V, E> theGraph) {
AtlasVertex<V, E> vertex = theGraph.addVertex();
newVertices.add(vertex);
return vertex;
}
@AfterMethod
public void removeVertices() {
for(AtlasVertex vertex : newVertices_) {
if(vertex.exists()) {
for(AtlasVertex vertex : newVertices) {
if (vertex.exists()) {
getGraph().removeVertex(vertex);
}
}
getGraph().commit();
newVertices_.clear();
newVertices.clear();
}
protected void runSynchronouslyInNewThread(final Runnable r) throws Throwable {
......@@ -191,10 +160,32 @@ public abstract class AbstractGraphDatabaseTest {
th.start();
th.join();
Throwable ex = wrapper.getExceptionThrown();
if(ex != null) {
if (ex != null) {
throw ex;
}
}
private static final class RunnableWrapper implements Runnable {
private final Runnable r;
private Throwable exceptionThrown = null;
private RunnableWrapper(Runnable r) {
this.r = r;
}
@Override
public void run() {
try {
r.run();
} catch(Throwable e) {
exceptionThrown = e;
}
}
public Throwable getExceptionThrown() {
return exceptionThrown;
}
}
}
\ No newline at end of file
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java
......@@ -30,10 +30,12 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
......@@ -43,7 +45,6 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Test;
......@@ -57,16 +58,16 @@ public class Titan0DatabaseTest {
private <V, E> AtlasGraph<V, E> getGraph() {
if (atlasGraph == null) {
Titan0Database db = new Titan0Database();
Titan0GraphDatabase db = new Titan0GraphDatabase();
atlasGraph = db.getGraph();
AtlasGraphManagement mgmt = atlasGraph.getManagementSystem();
// create the index (which defines these properties as being mult
// many)
for (String propertyName : AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS) {
for (String propertyName : new String[]{"__superTypeNames", "__traitNames"}) {
AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = mgmt.makePropertyKey(propertyName, String.class, Multiplicity.SET);
mgmt.createCompositeIndex(propertyName, propertyKey, false);
propertyKey = mgmt.makePropertyKey(propertyName, String.class, AtlasCardinality.SET);
mgmt.createExactMatchIndex(propertyName, false, Collections.singletonList(propertyKey));
}
}
mgmt.commit();
......
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseValidationTest.java
......@@ -37,7 +37,7 @@ public class Titan0DatabaseValidationTest {
// First get Instance
graph = new Titan0Graph();
configuration = ApplicationProperties.getSubsetConfiguration(ApplicationProperties.get(),
Titan0Database.GRAPH_PREFIX);
Titan0GraphDatabase.GRAPH_PREFIX);
}
@AfterClass
......@@ -58,15 +58,15 @@ public class Titan0DatabaseValidationTest {
@Test
public void testValidate() throws AtlasException {
try {
Titan0Database.validateIndexBackend(configuration);
Titan0GraphDatabase.validateIndexBackend(configuration);
} catch (Exception e) {
Assert.fail("Unexpected exception ", e);
}
// Change backend
configuration.setProperty(Titan0Database.INDEX_BACKEND_CONF, Titan0Database.INDEX_BACKEND_LUCENE);
configuration.setProperty(Titan0GraphDatabase.INDEX_BACKEND_CONF, Titan0GraphDatabase.INDEX_BACKEND_LUCENE);
try {
Titan0Database.validateIndexBackend(configuration);
Titan0GraphDatabase.validateIndexBackend(configuration);
Assert.fail("Expected exception");
} catch (Exception e) {
Assert.assertEquals(e.getMessage(),
......
graphdb/titan0/src/test/resources/atlas-application.properties
......@@ -17,7 +17,7 @@
#
######### Graph Database to Use #########
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan0.Titan0Database
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase
######### Atlas Server Configs #########
atlas.rest.address=http://localhost:31000
......
pom.xml 100755 → 100644
......@@ -455,6 +455,7 @@
<titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.index.backend>elasticsearch</titan.index.backend>
<entity.repository.impl>org.apache.atlas.repository.audit.InMemoryEntityAuditRepository</entity.repository.impl>
<graphdb.backend.impl>org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase</graphdb.backend.impl>
<atlas.surefire.options></atlas.surefire.options>
</properties>
......@@ -900,6 +901,57 @@
</dependency>
<!-- Graph DB -->
<dependency>
<groupId>com.tinkerpop.blueprints</groupId>
<artifactId>blueprints-core</artifactId>
<version>${tinkerpop.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId>
<version>${titan.version}</version>
<exclusions>
<!-- rexster does not work with servlet-api -->
<exclusion>
<groupId>com.tinkerpop.rexster</groupId>
<artifactId>rexster-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.tinkerpop.rexster</groupId>
<artifactId>rexster-server</artifactId>
</exclusion>
<!-- asm 4.0 does not work with jersey asm 3.1 -->
<exclusion>
<groupId>com.tinkerpop</groupId>
<artifactId>frames</artifactId>
</exclusion>
<exclusion>
<groupId>com.esotericsoftware.reflectasm</groupId>
<artifactId>reflectasm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion> <!-- GPL license imported from ganglia -->
<groupId>org.acplt</groupId>
<artifactId>oncrpc</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-hbase</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
......@@ -954,6 +1006,7 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.vividsolutions</groupId>
<artifactId>jts</artifactId>
......@@ -1027,12 +1080,6 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-server-api</artifactId>
<version>${project.version}</version>
</dependency>
......@@ -1083,6 +1130,17 @@
<type>war</type>
</dependency>
<!-- use titan 0.5.4 by default -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<version>${project.version}</version>
<type>pom</type>
<!-- exclusions should be added here for all of the non-titan0
implementations -->
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>hive-bridge</artifactId>
......@@ -1653,9 +1711,12 @@
<excludeSubProjects>true</excludeSubProjects>
<excludes>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>**/javax.script.ScriptEngineFactory</exclude>
<exclude>.reviewboardrc</exclude>
<exclude>3party-licenses/**</exclude>
<exclude>**/.cache</exclude>
<exclude>**/.cache-main</exclude>
<exclude>**/.cache-tests</exclude>
<exclude>**/.checkstyle</exclude>
<exclude>*.txt</exclude>
<exclude>**/*.json</exclude>
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-694 Update Atlas code to use graph abstraction layer (jnhagelb via sumasai)
ATLAS-1215 Atlas UI not working in firefox due to fix in ATLAS-1199 (kevalbhatt)
ATLAS-1171 Structured, high-level public APIs - Fix JAXB issues with PList, SearchFilter (mneethiraj via sumasai)
ATLAS-1206 Atlas UI not working with IE or Chrome on Windows OS in Kerberos mode (nixonrodrigues via sumasai)
......@@ -50,7 +51,7 @@ ATLAS-1104 Get outgoing edges by label doesn't work in some cases (shwethags)
ATLAS-1106 Fix Build failure due to wrong version in graphdb/common pom (sumasai)
ATLAS-1105 Disable HiveLiteralRewriterTest since its not used currently (sumasai)
ATLAS-1103 : UI: Search type list is not refreshed (Kalyanikashikar via sumasai)
ATLAS-693 Titan 0.5.4 implementation of the graph db abstraction {jnhagelb via dkantor)
ATLAS-693 Titan 0.5.4 implementation of the graph db abstraction (jnhagelb via dkantor)
ATLAS-1099 UI : multiple tag assign button hides wrongly (Kalyanikashikar via sumasai)
ATLAS-1087 Provide an option to turn off persisting entity definition in audits (sumasai, shwethags)
ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags)
......
repository/pom.xml
......@@ -52,11 +52,15 @@
<artifactId>atlas-graphdb-api</artifactId>
</dependency>
<dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
......
repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
......@@ -17,11 +17,10 @@
package org.apache.atlas;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import org.aopalliance.intercept.MethodInterceptor;
import org.aopalliance.intercept.MethodInvocation;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.SchemaNotFoundException;
import org.slf4j.Logger;
......@@ -29,30 +28,27 @@ import org.slf4j.LoggerFactory;
public class GraphTransactionInterceptor implements MethodInterceptor {
private static final Logger LOG = LoggerFactory.getLogger(GraphTransactionInterceptor.class);
private TitanGraph titanGraph;
@Inject
GraphProvider<TitanGraph> graphProvider;
private AtlasGraph graph;
@Override
public Object invoke(MethodInvocation invocation) throws Throwable {
if (titanGraph == null) {
titanGraph = graphProvider.get();
if (graph == null) {
graph = AtlasGraphProvider.getGraphInstance();
}
try {
Object response = invocation.proceed();
titanGraph.commit();
graph.commit();
LOG.info("graph commit");
return response;
} catch (Throwable t) {
titanGraph.rollback();
if (logException(t)) {
LOG.error("graph rollback due to exception ", t);
} else {
LOG.error("graph rollback due to exception " + t.getClass().getSimpleName() + ":" + t.getMessage());
}
graph.rollback();
throw t;
}
}
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -18,16 +18,9 @@
package org.apache.atlas;
import com.google.inject.Binder;
import com.google.inject.Singleton;
import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.DataSetLineageService;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.LineageService;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.listener.EntityChangeListener;
......@@ -35,13 +28,9 @@ import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.audit.EntityAuditListener;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.SoftDeleteHandler;
import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.service.Service;
......@@ -51,10 +40,15 @@ import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeSystemProvider;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.configuration.Configuration;
import com.google.inject.Binder;
import com.google.inject.Singleton;
import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
/**
* Guice module for Repository module.
*/
......@@ -62,9 +56,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
@Override
protected void configure() {
// special wiring for Titan Graph
ThrowingProviderBinder.create(binder()).bind(GraphProvider.class, TitanGraph.class).to(TitanGraphProvider.class)
.asEagerSingleton();
// allow for dynamic binding of the metadata repo & graph service
// bind the MetadataRepositoryService interface to an implementation
......@@ -95,9 +86,9 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
Configuration configuration = getConfiguration();
bindAuditRepository(binder(), configuration);
bind(DeleteHandler.class).to(getDeleteHandlerImpl(configuration)).asEagerSingleton();
bind(DeleteHandler.class).to((Class<? extends DeleteHandler>) AtlasRepositoryConfiguration.getDeleteHandlerImpl()).asEagerSingleton();
bind(TypeCache.class).to(getTypeCache(configuration)).asEagerSingleton();
bind(TypeCache.class).to((Class<? extends TypeCache>) AtlasRepositoryConfiguration.getTypeCache()).asEagerSingleton();
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
......@@ -119,7 +110,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
protected void bindAuditRepository(Binder binder, Configuration configuration) {
Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(getConfiguration());
Class<? extends EntityAuditRepository> auditRepoImpl = AtlasRepositoryConfiguration.getAuditRepositoryImpl();
//Map EntityAuditRepository interface to configured implementation
binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton();
......@@ -132,40 +123,4 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
}
}
private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
private Class<? extends EntityAuditRepository> getAuditRepositoryImpl(Configuration configuration) {
try {
return ApplicationProperties.getClass(configuration,
AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
private Class<? extends DeleteHandler> getDeleteHandlerImpl(Configuration configuration) {
try {
return ApplicationProperties.getClass(configuration,
DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
protected Class<? extends TypeCache> getTypeCache(Configuration configuration) {
// Get the type cache implementation class from Atlas configuration.
try {
return ApplicationProperties.getClass(configuration, TYPE_CACHE_IMPLEMENTATION_PROPERTY,
DefaultTypeCache.class.getName(), TypeCache.class);
} catch (AtlasException e) {
throw new RuntimeException("Error getting TypeCache implementation class", e);
}
}
}
repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
......@@ -18,7 +18,9 @@
package org.apache.atlas.discovery;
import com.thinkaurelius.titan.core.TitanGraph;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -31,7 +33,8 @@ import org.apache.atlas.query.InputLineageClosureQuery;
import org.apache.atlas.query.OutputLineageClosureQuery;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.SchemaNotFoundException;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
......@@ -39,13 +42,11 @@ import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Option;
import scala.Some;
import scala.collection.immutable.List;
import javax.inject.Inject;
import javax.inject.Singleton;
/**
* Hive implementation of Lineage service interface.
*/
......@@ -80,14 +81,14 @@ public class DataSetLineageService implements LineageService {
}
private final TitanGraph titanGraph;
private final AtlasGraph graph;
private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
private final GraphBackedDiscoveryService discoveryService;
@Inject
DataSetLineageService(GraphProvider<TitanGraph> graphProvider, MetadataRepository metadataRepository,
DataSetLineageService(MetadataRepository metadataRepository,
GraphBackedDiscoveryService discoveryService) throws DiscoveryException {
this.titanGraph = graphProvider.get();
this.graph = AtlasGraphProvider.getGraphInstance();
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
this.discoveryService = discoveryService;
}
......@@ -136,7 +137,7 @@ public class DataSetLineageService implements LineageService {
inputsQuery = new InputLineageClosureQuery(AtlasClient.DATA_SET_SUPER_TYPE, SELECT_INSTANCE_GUID,
guid, HIVE_PROCESS_TYPE_NAME,
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, titanGraph);
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
return inputsQuery.graph().toInstanceJson();
}
......@@ -153,7 +154,7 @@ public class DataSetLineageService implements LineageService {
OutputLineageClosureQuery outputsQuery =
new OutputLineageClosureQuery(AtlasClient.DATA_SET_SUPER_TYPE, SELECT_INSTANCE_GUID, guid, HIVE_PROCESS_TYPE_NAME,
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, titanGraph);
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
return outputsQuery.graph().toInstanceJson();
}
......
repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
......@@ -18,9 +18,10 @@
package org.apache.atlas.discovery.graph;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanVertex;
import java.util.List;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.query.Expressions;
import org.apache.atlas.query.GraphPersistenceStrategies;
......@@ -29,8 +30,12 @@ import org.apache.atlas.query.IntSequence;
import org.apache.atlas.query.TypeUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -44,8 +49,8 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.util.List;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
/**
* Default implementation of GraphPersistenceStrategy.
......@@ -95,8 +100,8 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
}
@Override
public List<String> traitNames(TitanVertex vertex) {
return GraphHelper.getTraitNames(vertex);
public List<String> traitNames(AtlasVertex AtlasVertex) {
return GraphHelper.getTraitNames(AtlasVertex);
}
@Override
......@@ -105,7 +110,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
}
@Override
public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
public Id getIdFromVertex(String dataTypeName, AtlasVertex vertex) {
return GraphHelper.getIdFromVertex(dataTypeName, vertex);
}
......@@ -133,16 +138,16 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
break;
case STRUCT:
TitanVertex structVertex = (TitanVertex) value;
AtlasVertex structVertex = (AtlasVertex) value;
StructType structType = (StructType) dataType;
ITypedStruct structInstance = structType.createInstance();
TypeSystem.IdType idType = TypeSystem.getInstance().getIdType();
if (dataType.getName().equals(idType.getName())) {
structInstance.set(idType.typeNameAttrName(), GraphHelper.getProperty(structVertex, typeAttributeName()));
structInstance.set(idType.idAttrName(), GraphHelper.getProperty(structVertex, idAttributeName()));
String stateValue = GraphHelper.getProperty(structVertex, stateAttributeName());
structInstance.set(idType.typeNameAttrName(), GraphHelper.getSingleValuedProperty(structVertex, typeAttributeName(), String.class));
structInstance.set(idType.idAttrName(), GraphHelper.getSingleValuedProperty(structVertex, idAttributeName(), String.class));
String stateValue = GraphHelper.getSingleValuedProperty(structVertex, stateAttributeName(), String.class);
if (stateValue != null) {
structInstance.set(idType.stateAttrName(), stateValue);
}
......@@ -153,7 +158,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
return dataType.convert(structInstance, Multiplicity.OPTIONAL);
case TRAIT:
TitanVertex traitVertex = (TitanVertex) value;
AtlasVertex traitVertex = (AtlasVertex) value;
TraitType traitType = (TraitType) dataType;
ITypedStruct traitInstance = traitType.createInstance();
// todo - this is not right, we should load the Instance associated with this
......@@ -165,9 +170,9 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
break;
case CLASS:
TitanVertex classVertex = (TitanVertex) value;
AtlasVertex classVertex = (AtlasVertex) value;
ITypedReferenceableInstance classInstance = metadataRepository.getGraphToInstanceMapper()
.mapGraphToTypedInstance(classVertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
.mapGraphToTypedInstance(GraphHelper.getSingleValuedProperty(classVertex, Constants.GUID_PROPERTY_KEY, String.class),
classVertex);
return dataType.convert(classInstance, Multiplicity.OPTIONAL);
......@@ -210,6 +215,11 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
public String gremlinCompOp(Expressions.ComparisonExpression op) {
return GraphPersistenceStrategies$class.gremlinCompOp(this, op);
}
@Override
public String gremlinPrimitiveOp(Expressions.ComparisonExpression op) {
return GraphPersistenceStrategies$class.gremlinPrimitiveOp(this, op);
}
@Override
public String loopObjectExpression(IDataType<?> dataType) {
......@@ -250,5 +260,30 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
public boolean addGraphVertexPrefix(scala.collection.Traversable<String> preStatements) {
return GraphPersistenceStrategies$class.addGraphVertexPrefix(this, preStatements);
}
@Override
public GremlinVersion getSupportedGremlinVersion() {
return GraphPersistenceStrategies$class.getSupportedGremlinVersion(this);
}
@Override
public String generatePersisentToLogicalConversionExpression(String expr, IDataType<?> t) {
return GraphPersistenceStrategies$class.generatePersisentToLogicalConversionExpression(this,expr, t);
}
@Override
public String initialQueryCondition() {
return GraphPersistenceStrategies$class.initialQueryCondition(this);
}
@Override
public boolean isPropertyValueConversionNeeded(IDataType<?> t) {
return GraphPersistenceStrategies$class.isPropertyValueConversionNeeded(this, t);
}
@Override
public AtlasGraph getGraph() throws RepositoryException {
return metadataRepository.getGraph();
}
}
repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
......@@ -18,13 +18,17 @@
package org.apache.atlas.discovery.graph;
import com.thinkaurelius.titan.core.TitanEdge;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Vertex;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.script.ScriptException;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.discovery.DiscoveryException;
......@@ -39,29 +43,21 @@ import org.apache.atlas.query.QueryParser;
import org.apache.atlas.query.QueryProcessor;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.util.Either;
import scala.util.parsing.combinator.Parsers;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Graph backed implementation of Search.
*/
......@@ -70,19 +66,19 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class);
private final TitanGraph titanGraph;
private final AtlasGraph graph;
private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
public final static String SCORE = "score";
@Inject
GraphBackedDiscoveryService(GraphProvider<TitanGraph> graphProvider, MetadataRepository metadataRepository)
GraphBackedDiscoveryService(MetadataRepository metadataRepository)
throws DiscoveryException {
this.titanGraph = graphProvider.get();
this.graph = AtlasGraphProvider.getGraphInstance();
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
}
//Refer http://s3.thinkaurelius.com/docs/titan/0.5.4/index-backends.html for indexed query
//For titan 0.5.4, refer to http://s3.thinkaurelius.com/docs/titan/0.5.4/index-backends.html for indexed query
//http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query
// .html#query-string-syntax for query syntax
@Override
......@@ -90,8 +86,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
public String searchByFullText(String query, QueryParams queryParams) throws DiscoveryException {
String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, query);
LOG.debug("Full text query: {}", graphQuery);
Iterator<TitanIndexQuery.Result<Vertex>> results =
titanGraph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery).vertices().iterator();
Iterator<AtlasIndexQuery.Result<?, ?>> results =graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery).vertices();
JSONArray response = new JSONArray();
int index = 0;
......@@ -101,8 +96,9 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
}
while (results.hasNext() && response.length() < queryParams.limit()) {
TitanIndexQuery.Result<Vertex> result = results.next();
Vertex vertex = result.getElement();
AtlasIndexQuery.Result<?,?> result = results.next();
AtlasVertex<?,?> vertex = result.getVertex();
JSONObject row = new JSONObject();
String guid = GraphHelper.getIdFromVertex(vertex);
......@@ -157,7 +153,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
LOG.debug("Query = {}", validatedExpression);
LOG.debug("Expression Tree = {}", validatedExpression.treeString());
LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr());
return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, titanGraph).evaluate();
return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, graph).evaluate();
}
/**
......@@ -173,72 +169,59 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
@GraphTransaction
public List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException {
LOG.debug("Executing gremlin query={}", gremlinQuery);
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
if(engine == null) {
throw new DiscoveryException("gremlin-groovy: engine not found");
}
Bindings bindings = engine.createBindings();
bindings.put("g", titanGraph);
try {
Object o = engine.eval(gremlinQuery, bindings);
Object o = graph.executeGremlinScript(gremlinQuery, false);
return extractResult(o);
} catch (ScriptException se) {
throw new DiscoveryException(se);
}
}
private List<Map<String, String>> extractResult(final Object o) throws DiscoveryException {
List<Map<String, String>> result = new ArrayList<>();
if (o instanceof List) {
List l = (List) o;
for (Object r : l) {
for (Object value : l) {
Map<String, String> oRow = new HashMap<>();
if (r instanceof Map) {
@SuppressWarnings("unchecked") Map<Object, Object> iRow = (Map) r;
if (value instanceof Map) {
@SuppressWarnings("unchecked") Map<Object, Object> iRow = (Map) value;
for (Map.Entry e : iRow.entrySet()) {
Object k = e.getKey();
Object v = e.getValue();
oRow.put(k.toString(), v.toString());
}
} else if (r instanceof TitanVertex) {
TitanVertex vertex = (TitanVertex) r;
oRow.put("id", vertex.getId().toString());
Iterable<TitanProperty> ps = vertex.getProperties();
for (TitanProperty tP : ps) {
String pName = tP.getPropertyKey().getName();
Object pValue = vertex.getProperty(pName);
if (pValue != null) {
oRow.put(pName, pValue.toString());
} else if (value instanceof AtlasVertex) {
AtlasVertex<?,?> vertex = (AtlasVertex<?,?>)value;
for (String key : vertex.getPropertyKeys()) {
Object propertyValue = GraphHelper.getProperty(vertex, key);
if (propertyValue != null) {
oRow.put(key, propertyValue.toString());
}
}
} else if (r instanceof String) {
oRow.put("", r.toString());
} else if (r instanceof TitanEdge) {
TitanEdge edge = (TitanEdge) r;
} else if (value instanceof String) {
oRow.put("", value.toString());
} else if(value instanceof AtlasEdge) {
AtlasEdge edge = (AtlasEdge) value;
oRow.put("id", edge.getId().toString());
oRow.put("label", edge.getLabel());
oRow.put("inVertex", edge.getVertex(Direction.IN).getId().toString());
oRow.put("outVertex", edge.getVertex(Direction.OUT).getId().toString());
Set<String> propertyKeys = edge.getPropertyKeys();
for (String propertyKey : propertyKeys) {
oRow.put(propertyKey, edge.getProperty(propertyKey).toString());
oRow.put("inVertex", edge.getInVertex().getId().toString());
oRow.put("outVertex", edge.getOutVertex().getId().toString());
for (String propertyKey : edge.getPropertyKeys()) {
oRow.put(propertyKey, GraphHelper.getProperty(edge, propertyKey).toString());
}
} else {
throw new DiscoveryException(String.format("Cannot process result %s", o.toString()));
throw new DiscoveryException(String.format("Cannot process result %s", String.valueOf(value)));
}
result.add(oRow);
}
} else {
}
else {
result.add(new HashMap<String, String>() {{
put("result", o.toString());
}});
}});
}
return result;
}
......
repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java 0 → 100755
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import com.google.common.annotations.VisibleForTesting;
/**
* Provides access to the AtlasGraph
*
*/
public class AtlasGraphProvider implements IAtlasGraphProvider {
private static volatile GraphDatabase<?,?> graphDb_;
public static <V, E> AtlasGraph<V, E> getGraphInstance() {
GraphDatabase<?,?> db = getGraphDatabase();
AtlasGraph<?, ?> graph = db.getGraph();
return (AtlasGraph<V, E>) graph;
}
private static <V, E> GraphDatabase<?,?> getGraphDatabase() {
try {
if (graphDb_ == null) {
synchronized(AtlasGraphProvider.class) {
if(graphDb_ == null) {
Class implClass = AtlasRepositoryConfiguration.getGraphDatabaseImpl();
graphDb_ = (GraphDatabase<V, E>) implClass.newInstance();
}
}
}
return graphDb_;
}
catch (IllegalAccessException e) {
throw new RuntimeException("Error initializing graph database", e);
} catch (InstantiationException e) {
throw new RuntimeException("Error initializing graph database", e);
}
}
@VisibleForTesting
public static void cleanup() {
getGraphDatabase().cleanup();
}
@Override
public AtlasGraph get() throws RepositoryException {
return getGraphInstance();
}
}
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
......@@ -17,7 +17,7 @@
*/
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -49,7 +49,7 @@ public class FullTextMapper {
instanceCache = new HashMap<>();
}
public String mapRecursive(Vertex instanceVertex, boolean followReferences) throws AtlasException {
public String mapRecursive(AtlasVertex instanceVertex, boolean followReferences) throws AtlasException {
String guid = GraphHelper.getIdFromVertex(instanceVertex);
ITypedReferenceableInstance typedReference;
if (instanceCache.containsKey(guid)) {
......@@ -121,7 +121,7 @@ public class FullTextMapper {
case CLASS:
if (followReferences) {
String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
Vertex refVertex = graphHelper.getVertexForGUID(refGuid);
AtlasVertex refVertex = graphHelper.getVertexForGUID(refGuid);
return mapRecursive(refVertex, false);
}
break;
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -18,13 +18,12 @@
package org.apache.atlas.repository.graph;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
......@@ -32,6 +31,10 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.exception.EntityExistsException;
......@@ -46,10 +49,9 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
/**
* An implementation backed by a Graph database provided
......@@ -64,16 +66,16 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private final TitanGraph titanGraph;
private final AtlasGraph graph;
private DeleteHandler deleteHandler;
private GraphToTypedInstanceMapper graphToInstanceMapper;
@Inject
public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider, DeleteHandler deleteHandler) {
this.titanGraph = graphProvider.get();
graphToInstanceMapper = new GraphToTypedInstanceMapper(titanGraph);
public GraphBackedMetadataRepository(DeleteHandler deleteHandler) {
this.graph = AtlasGraphProvider.getGraphInstance();
graphToInstanceMapper = new GraphToTypedInstanceMapper(graph);
this.deleteHandler = deleteHandler;
}
......@@ -148,7 +150,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException {
LOG.debug("Retrieving entity with guid={}", guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
try {
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
......@@ -164,7 +166,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.debug("Retrieving entity with type={} and {}={}", entityType, attribute, value);
IDataType type = typeSystem.getDataType(IDataType.class, entityType);
String propertyKey = getFieldNameInVertex(type, attribute);
Vertex instanceVertex = graphHelper.findVertex(propertyKey, value,
AtlasVertex instanceVertex = graphHelper.findVertex(propertyKey, value,
Constants.ENTITY_TYPE_PROPERTY_KEY, entityType,
Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
......@@ -176,15 +178,15 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@GraphTransaction
public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.debug("Retrieving entity list for type={}", entityType);
GraphQuery query = titanGraph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<Vertex> results = query.vertices().iterator();
AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<AtlasVertex> results = query.vertices().iterator();
if (!results.hasNext()) {
return Collections.emptyList();
}
ArrayList<String> entityList = new ArrayList<>();
while (results.hasNext()) {
Vertex vertex = results.next();
AtlasVertex vertex = results.next();
entityList.add(GraphHelper.getIdFromVertex(vertex));
}
......@@ -202,7 +204,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@GraphTransaction
public List<String> getTraitNames(String guid) throws AtlasException {
LOG.debug("Retrieving trait names for entity={}", guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
return GraphHelper.getTraitNames(instanceVertex);
}
......@@ -222,7 +224,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.debug("Adding a new trait={} for entity={}", traitName, guid);
try {
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
// add the trait instance as a new vertex
final String typeName = GraphHelper.getTypeName(instanceVertex);
......@@ -256,7 +258,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public void deleteTrait(String guid, String traitNameToBeDeleted) throws TraitNotFoundException, EntityNotFoundException, RepositoryException {
LOG.debug("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
if (!traitNames.contains(traitNameToBeDeleted)) {
......@@ -267,7 +269,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
try {
final String entityTypeName = GraphHelper.getTypeName(instanceVertex);
String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted);
Edge edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
AtlasEdge edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
if(edge != null) {
deleteHandler.deleteEdgeReference(edge, DataTypes.TypeCategory.TRAIT, false, true);
......@@ -281,7 +283,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
private void updateTraits(Vertex instanceVertex, List<String> traitNames) {
private void updateTraits(AtlasVertex instanceVertex, List<String> traitNames) {
// remove the key
instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
......@@ -332,14 +334,14 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throw new IllegalArgumentException("guids must be non-null and non-empty");
}
List<Vertex> vertices = new ArrayList<>(guids.size());
List<AtlasVertex> vertices = new ArrayList<>(guids.size());
for (String guid : guids) {
if (guid == null) {
LOG.warn("deleteEntities: Ignoring null guid");
continue;
}
try {
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
vertices.add(instanceVertex);
} catch (EntityNotFoundException e) {
// Entity does not exist - treat as non-error, since the caller
......@@ -360,4 +362,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return new AtlasClient.EntityResult(requestContext.getCreatedEntityIds(),
requestContext.getUpdatedEntityIds(), requestContext.getDeletedEntityIds());
}
public AtlasGraph getGraph() {
return AtlasGraphProvider.getGraphInstance();
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphSchemaInitializer.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.setup.SetupException;
import org.apache.atlas.setup.SetupStep;
......@@ -41,7 +41,7 @@ public class GraphSchemaInitializer implements SetupStep {
LOG.info("Initializing graph schema backend.");
try {
// The implementation of this method internally creates the schema.
TitanGraphProvider.getGraphInstance();
AtlasGraphProvider.getGraphInstance();
LOG.info("Completed initializing graph schema backend.");
} catch (Exception e) {
LOG.error("Could not initialize graph schema backend due to exception, {}", e.getMessage(), e);
......
repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java
......@@ -19,8 +19,8 @@
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.types.TypeSystem;
......@@ -32,12 +32,12 @@ public class HardDeleteHandler extends DeleteHandler {
}
@Override
protected void _deleteVertex(Vertex instanceVertex, boolean force) {
protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
graphHelper.removeVertex(instanceVertex);
}
@Override
protected void deleteEdge(Edge edge, boolean force) throws AtlasException {
protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasException {
graphHelper.removeEdge(edge);
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java → repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java
......@@ -18,10 +18,15 @@
package org.apache.atlas.repository.graph;
import com.google.inject.throwingproviders.CheckedProvider;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
@Override
T get();
/**
* Provides a mechanism to control what graph is used in various places. This
* allows the graph to be mocked out during unit testing and be initialized
* lazily.
*/
public interface IAtlasGraphProvider {
AtlasGraph get() throws RepositoryException;
}
repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java
......@@ -19,8 +19,8 @@
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -36,7 +36,7 @@ public class SoftDeleteHandler extends DeleteHandler {
}
@Override
protected void _deleteVertex(Vertex instanceVertex, boolean force) {
protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
if (force) {
graphHelper.removeVertex(instanceVertex);
} else {
......@@ -50,7 +50,7 @@ public class SoftDeleteHandler extends DeleteHandler {
}
@Override
protected void deleteEdge(Edge edge, boolean force) throws AtlasException {
protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasException {
if (force) {
graphHelper.removeEdge(edge);
} else {
......
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -18,10 +18,18 @@
package org.apache.atlas.services;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -36,6 +44,7 @@ import org.apache.atlas.query.QueryParser;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -60,29 +69,22 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.collection.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
/**
* Simple wrapper over TypeSystem and MetadataRepository services with hooks
......@@ -118,8 +120,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders,
TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
}
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
//for testing only
public DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders,
......@@ -153,8 +156,6 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
restoreTypeSystem();
}
AtlasPatchHandler.handlePatches(this, typeSystem);
maxAuditResults = configuration.getShort(CONFIG_MAX_AUDIT_RESULTS, DEFAULT_MAX_AUDIT_RESULTS);
}
......@@ -243,6 +244,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
typeDefinition = ParamChecker.notEmpty(typeDefinition, "type definition");
TypesDef typesDef = validateTypeDefinition(typeDefinition);
try {
final TypeSystem.TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(typesDef, isUpdate);
final Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
......@@ -334,40 +336,13 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
return guids;
}
private ITypedReferenceableInstance[] deserializeClassInstances(String entityInstanceDefinition)
throws AtlasException {
try {
JSONArray referableInstances = new JSONArray(entityInstanceDefinition);
ITypedReferenceableInstance[] instances = new ITypedReferenceableInstance[referableInstances.length()];
for (int index = 0; index < referableInstances.length(); index++) {
Referenceable entityInstance =
InstanceSerialization.fromJsonReferenceable(referableInstances.getString(index), true);
ITypedReferenceableInstance typedInstrance = getTypedReferenceableInstance(entityInstance);
instances[index] = typedInstrance;
}
return instances;
} catch(ValueConversionException | TypeNotFoundException e) {
throw e;
} catch (Exception e) { // exception from deserializer
LOG.error("Unable to deserialize json={}", entityInstanceDefinition, e);
throw new IllegalArgumentException("Unable to deserialize json", e);
}
private ITypedReferenceableInstance[] deserializeClassInstances(String entityInstanceDefinition) throws AtlasException {
return GraphHelper.deserializeClassInstances(typeSystem, entityInstanceDefinition);
}
@Override
public ITypedReferenceableInstance getTypedReferenceableInstance(Referenceable entityInstance) throws AtlasException {
final String entityTypeName = ParamChecker.notEmpty(entityInstance.getTypeName(), "Entity type cannot be null");
ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName);
//Both assigned id and values are required for full update
//classtype.convert() will remove values if id is assigned. So, set temp id, convert and
// then replace with original id
Id origId = entityInstance.getId();
entityInstance.replaceWithNewId(new Id(entityInstance.getTypeName()));
ITypedReferenceableInstance typedInstrance = entityType.convert(entityInstance, Multiplicity.REQUIRED);
((ReferenceableInstance)typedInstrance).replaceWithNewId(origId);
return typedInstrance;
return GraphHelper.getTypedReferenceableInstance(typeSystem, entityInstance);
}
/**
......
repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.util;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.SoftDeleteHandler;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Atlas configuration for repository project
*
*/
public class AtlasRepositoryConfiguration {
private static Logger LOG = LoggerFactory.getLogger(AtlasRepositoryConfiguration.class);
public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
@SuppressWarnings("unchecked")
public static Class<? extends TypeCache> getTypeCache() {
// Get the type cache implementation class from Atlas configuration.
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config, TYPE_CACHE_IMPLEMENTATION_PROPERTY,
DefaultTypeCache.class.getName(), TypeCache.class);
} catch (AtlasException e) {
LOG.error("Error loading typecache ", e);
return DefaultTypeCache.class;
}
}
private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
@SuppressWarnings("unchecked")
public static Class<? extends EntityAuditRepository> getAuditRepositoryImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
@SuppressWarnings("unchecked")
public static Class<? extends DeleteHandler> getDeleteHandlerImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
private static final String GRAPH_DATABASE_IMPLEMENTATION_PROPERTY = "atlas.graphdb.backend";
private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = "org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase";
@SuppressWarnings("unchecked")
public static Class<? extends GraphDatabase> getGraphDatabaseImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
GRAPH_DATABASE_IMPLEMENTATION_PROPERTY, DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS, GraphDatabase.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
}
repository/src/main/scala/org/apache/atlas/query/ClosureQuery.scala 100755 → 100644
......@@ -20,7 +20,7 @@ package org.apache.atlas.query
import java.util
import com.thinkaurelius.titan.core.TitanGraph
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.query.Expressions._
import org.apache.atlas.typesystem.ITypedStruct
import org.apache.atlas.typesystem.json.{InstanceSerialization, Serialization}
......@@ -116,7 +116,7 @@ trait ClosureQuery {
def withPath : Boolean
def persistenceStrategy: GraphPersistenceStrategies
def g: TitanGraph
def g: AtlasGraph[_,_]
def pathExpr : Expressions.Expression = {
closureRelation.tail.foldLeft(closureRelation.head.toExpr)((b,a) => b.field(a.toFieldName))
......@@ -184,8 +184,8 @@ trait ClosureQuery {
* foreach resultRow
* for each Path entry
* add an entry in the edges Map
* add an entry for the Src Vertex to the vertex Map
* add an entry for the Dest Vertex to the vertex Map
* add an entry for the Src AtlasVertex to the vertex Map
* add an entry for the Dest AtlasVertex to the vertex Map
*/
res.rows.map(_.asInstanceOf[StructInstance]).foreach { r =>
......@@ -207,7 +207,7 @@ trait ClosureQuery {
}
currVertex = nextVertex
}
val vertex = r.get(TypeUtils.ResultWithPathStruct.resultAttrName)
val AtlasVertex = r.get(TypeUtils.ResultWithPathStruct.resultAttrName)
vertices.put(id(srcVertex), vertexStruct(srcVertex,
r.get(TypeUtils.ResultWithPathStruct.resultAttrName).asInstanceOf[ITypedStruct],
s"${SRC_PREFIX}_"))
......@@ -242,6 +242,7 @@ trait SingleInstanceClosureQuery[T] extends ClosureQuery {
}
}
import scala.language.existentials;
/**
* A ClosureQuery to compute '''Lineage''' for Hive tables. Assumes the Lineage relation is captured in a ''CTAS''
* type, and the table relations are captured as attributes from a CTAS instance to Table instances.
......@@ -266,7 +267,7 @@ case class InputLineageClosureQuery(tableTypeName : String,
selectAttributes : Option[List[String]],
withPath : Boolean,
persistenceStrategy: GraphPersistenceStrategies,
g: TitanGraph
g: AtlasGraph[_,_]
) extends SingleInstanceClosureQuery[String] {
val closureType : String = tableTypeName
......@@ -306,7 +307,7 @@ case class OutputLineageClosureQuery(tableTypeName : String,
selectAttributes : Option[List[String]],
withPath : Boolean,
persistenceStrategy: GraphPersistenceStrategies,
g: TitanGraph
g: AtlasGraph[_,_]
) extends SingleInstanceClosureQuery[String] {
val closureType : String = tableTypeName
......
repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala
......@@ -18,10 +18,9 @@
package org.apache.atlas.query
import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
import org.apache.atlas.query.Expressions._
import com.thinkaurelius.titan.core.TitanGraph
import com.tinkerpop.pipes.util.structures.Row
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.query.TypeUtils.ResultWithPathStruct
import org.apache.atlas.typesystem.json._
import org.apache.atlas.typesystem.types._
......@@ -40,81 +39,109 @@ case class GremlinQueryResult(query: String,
def toJson = JsonHelper.toJson(this)
}
class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: TitanGraph) {
val manager: ScriptEngineManager = new ScriptEngineManager
val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
val bindings: Bindings = engine.createBindings
bindings.put("g", g)
class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: AtlasGraph[_,_]) {
/**
/**
*
* @param gResultObj is the object returned from gremlin. This must be a List
* @param qryResultObj is the object constructed for the output w/o the Path.
* @return a ResultWithPathStruct
*/
def addPathStruct(gResultObj : AnyRef, qryResultObj : Any) : Any = {
if ( !qry.isPathExpresion) {
qryResultObj
} else {
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
val iPaths = gResultObj.asInstanceOf[java.util.List[AnyRef]].init
val oPaths = iPaths.map { p =>
persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, p)
}.toList.asJava
val sType = qry.expr.dataType.asInstanceOf[StructType]
val sInstance = sType.createInstance()
sInstance.set(ResultWithPathStruct.pathAttrName, oPaths)
sInstance.set(ResultWithPathStruct.resultAttrName, qryResultObj)
sInstance
}
def addPathStruct(gResultObj: AnyRef, qryResultObj: Any): Any = {
if (!qry.isPathExpression) {
qryResultObj
} else {
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
val iPaths = gResultObj.asInstanceOf[java.util.List[AnyRef]].init
val oPaths = iPaths.map { value =>
persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, value)
}.toList.asJava
val sType = qry.expr.dataType.asInstanceOf[StructType]
val sInstance = sType.createInstance()
sInstance.set(ResultWithPathStruct.pathAttrName, oPaths)
sInstance.set(ResultWithPathStruct.resultAttrName, qryResultObj)
sInstance
}
}
def instanceObject(v : AnyRef) : AnyRef = {
if ( qry.isPathExpresion ) {
import scala.collection.JavaConversions._
v.asInstanceOf[java.util.List[AnyRef]].last
} else {
v
}
def instanceObject(v: AnyRef): AnyRef = {
if (qry.isPathExpression) {
import scala.collection.JavaConversions._
v.asInstanceOf[java.util.List[AnyRef]].last
} else {
v
}
}
def evaluate(): GremlinQueryResult = {
import scala.collection.JavaConversions._
val debug:Boolean = false
val rType = qry.expr.dataType
val oType = if (qry.isPathExpresion) qry.expr.children(0).dataType else rType
val rawRes = engine.eval(qry.queryStr, bindings)
val oType = if (qry.isPathExpression) {
qry.expr.children(0).dataType
}
else {
rType
}
val rawRes = g.executeGremlinScript(qry.queryStr, qry.isPathExpression);
if(debug) {
println(" rawRes " +rawRes)
}
if (!qry.hasSelectList) {
val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { v =>
val iV = instanceObject(v)
val o = persistenceStrategy.constructInstance(oType, iV)
addPathStruct(v, o)
val instObj = instanceObject(v)
val o = persistenceStrategy.constructInstance(oType, instObj)
addPathStruct(v, o)
}
GremlinQueryResult(qry.expr.toString, rType, rows.toList)
} else {
val sType = oType.asInstanceOf[StructType]
val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { r =>
val rV = instanceObject(r).asInstanceOf[Row[java.util.List[AnyRef]]]
val rV = instanceObject(r)
val sInstance = sType.createInstance()
val selObj = SelectExpressionHelper.extractSelectExpression(qry.expr)
if (selObj.isDefined)
{
if (selObj.isDefined) {
val selExpr = selObj.get.asInstanceOf[Expressions.SelectExpression]
selExpr.selectListWithAlias.foreach { aE =>
selExpr.selectListWithAlias.foreach { aE =>
val cName = aE.alias
val (src, idx) = qry.resultMaping(cName)
val v = rV.getColumn(src).get(idx)
val v = getColumnValue(rV, src, idx)
sInstance.set(cName, persistenceStrategy.constructInstance(aE.dataType, v))
}
}
addPathStruct(r, sInstance)
}
addPathStruct(r, sInstance)
}
GremlinQueryResult(qry.expr.toString, rType, rows.toList)
}
}
private def getColumnValue(rowValue: AnyRef, colName: String, idx: Integer) : AnyRef = {
var rawColumnValue: AnyRef = null;
if(rowValue.isInstanceOf[java.util.Map[_,_]]) {
val columnsMap = rowValue.asInstanceOf[java.util.Map[String,AnyRef]];
rawColumnValue = columnsMap.get(colName);
}
else {
//when there is only one column, result does not come back as a map
rawColumnValue = rowValue;
}
var value : AnyRef = null;
if(rawColumnValue.isInstanceOf[java.util.List[_]] && idx >= 0) {
val arr = rawColumnValue.asInstanceOf[java.util.List[AnyRef]];
value = arr.get(idx);
}
else {
value = rawColumnValue;
}
return value;
}
}
object JsonHelper {
......
repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala
......@@ -18,21 +18,27 @@
package org.apache.atlas.query
import com.thinkaurelius.titan.core.TitanGraph
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.query.Expressions._
import org.slf4j.{Logger, LoggerFactory}
object QueryProcessor {
val LOG : Logger = LoggerFactory.getLogger("org.apache.atlas.query.QueryProcessor")
def evaluate(e: Expression, g: TitanGraph, gP : GraphPersistenceStrategies = GraphPersistenceStrategy1):
def evaluate(e: Expression, g: AtlasGraph[_,_], gP : GraphPersistenceStrategies = null):
GremlinQueryResult = {
var strategy = gP;
if(strategy == null) {
strategy = GraphPersistenceStrategy1(g);
}
val e1 = validate(e)
val q = new GremlinTranslator(e1, gP).translate()
val q = new GremlinTranslator(e1, strategy).translate()
LOG.debug("Query: " + e1)
LOG.debug("Expression Tree:\n" + e1.treeString)
LOG.debug("Gremlin Query: " + q.queryStr)
new GremlinEvaluator(q, gP, g).evaluate()
new GremlinEvaluator(q, strategy, g).evaluate()
}
def validate(e: Expression): Expression = {
......
repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
......@@ -17,15 +17,15 @@
*/
package org.apache.atlas;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
......@@ -45,11 +45,9 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.annotations.Guice;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
/**
* Base Class to set up hive types and instances for tests
......@@ -63,29 +61,21 @@ public class BaseRepositoryTest {
@Inject
protected MetadataRepository repository;
@Inject
protected GraphProvider<TitanGraph> graphProvider;
protected void setUp() throws Exception {
//force graph initialization / built in type registration
TestUtils.getGraph();
setUpTypes();
new GraphBackedSearchIndexer(graphProvider);
RequestContext.createContext();
new GraphBackedSearchIndexer();
TestUtils.resetRequestContext();
setupInstances();
TestUtils.dumpGraph(graphProvider.get());
TestUtils.dumpGraph(TestUtils.getGraph());
}
protected void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
private void setUpTypes() throws Exception {
......
repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
......@@ -18,14 +18,11 @@
package org.apache.atlas;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.testng.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
/**
* Unit test for Guice injector service loading
*
......@@ -35,12 +32,8 @@ import javax.inject.Inject;
@Guice(modules = RepositoryMetadataModule.class)
public class RepositoryServiceLoadingTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Test
public void testGetGraphService() throws Exception {
Assert.assertNotNull(graphProvider);
Assert.assertNotNull(graphProvider.get());
Assert.assertNotNull(AtlasGraphProvider.getGraphInstance());
}
}
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -18,12 +18,36 @@
package org.apache.atlas;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
......@@ -40,25 +64,18 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
import java.io.File;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
/**
* Test utility class.
......@@ -73,16 +90,30 @@ public final class TestUtils {
/**
* Dumps the graph in GSON format in the path returned.
*
* @param titanGraph handle to graph
* @param graph handle to graph
* @return path to the dump file
* @throws Exception
*/
public static String dumpGraph(TitanGraph titanGraph) throws Exception {
public static String dumpGraph(AtlasGraph<?,?> graph) throws Exception {
File tempFile = File.createTempFile("graph", ".gson");
System.out.println("tempFile.getPath() = " + tempFile.getPath());
GraphSONWriter.outputGraph(titanGraph, tempFile.getPath());
GraphHelper.dumpToLog(graph);
FileOutputStream os = null;
try {
os = new FileOutputStream(tempFile);
graph.exportToGson(os);
}
finally {
if(os != null) {
try {
os.close();
}
catch(IOException e) {
e.printStackTrace();
}
}
}
GraphHelper.dumpToLog(titanGraph);
return tempFile.getPath();
}
......@@ -469,4 +500,44 @@ public final class TestUtils {
}
return null;
}
public static void resetRequestContext() {
RequestContext.createContext();
}
public static void setupGraphProvider(MetadataRepository repo) throws AtlasException {
TypeCache typeCache = null;
try {
typeCache = AtlasRepositoryConfiguration.getTypeCache().newInstance();
}
catch(Throwable t) {
typeCache = new DefaultTypeCache();
}
final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer();
Provider<TypesChangeListener> indexerProvider = new Provider<TypesChangeListener>() {
@Override
public TypesChangeListener get() {
return indexer;
}
};
Configuration config = ApplicationProperties.get();
ITypeStore typeStore = new GraphBackedTypeStore();
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo,
typeStore,
new ReservedTypesRegistrar(),
Collections.singletonList(indexerProvider),
new ArrayList<Provider<EntityChangeListener>>(), TypeSystem.getInstance(), config, typeCache);
//commit the created types
getGraph().commit();
}
public static AtlasGraph getGraph() {
return AtlasGraphProvider.getGraphInstance();
}
}
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -18,7 +18,23 @@
package org.apache.atlas.discovery;
import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.BaseRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
......@@ -28,8 +44,8 @@ import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -49,21 +65,7 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import com.google.common.collect.ImmutableSet;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
......@@ -75,6 +77,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
private GraphBackedDiscoveryService discoveryService;
private QueryParams queryParams = new QueryParams(40, 0);
@Override
@BeforeClass
public void setUp() throws Exception {
super.setUp();
......@@ -112,11 +115,11 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
}
}
TitanGraphProvider provider = new TitanGraphProvider();
//We need to commit the transaction before creating the indices to release the locks held by the transaction.
//otherwise, the index commit will fail while waiting for the those locks to be released.
provider.get().commit();
GraphBackedSearchIndexer idx = new GraphBackedSearchIndexer(provider);
AtlasGraphProvider.getGraphInstance().commit();
GraphBackedSearchIndexer idx = new GraphBackedSearchIndexer();
idx.onAdd(newTypes);
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -99,7 +99,7 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo
}
@Override
protected void assertVerticesDeleted(List<Vertex> vertices) {
protected void assertVerticesDeleted(List<AtlasVertex> vertices) {
assertEquals(vertices.size(), 0);
}
......@@ -171,10 +171,10 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo
assertNull(mapOwnerInstance.get("map"));
assertNull(mapOwnerInstance.get("biMap"));
Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
Object object = mapOwnerVertex.getProperty("MapOwner.map.value1");
AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
Object object = mapOwnerVertex.getProperty("MapOwner.map.value1", String.class);
assertNull(object);
object = mapOwnerVertex.getProperty("MapOwner.biMap.value1");
object = mapOwnerVertex.getProperty("MapOwner.biMap.value1", String.class);
assertNull(object);
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -116,9 +116,9 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo
}
@Override
protected void assertVerticesDeleted(List<Vertex> vertices) {
for (Vertex vertex : vertices) {
assertEquals(vertex.getProperty(Constants.STATE_PROPERTY_KEY), Id.EntityState.DELETED.name());
protected void assertVerticesDeleted(List<AtlasVertex> vertices) {
for (AtlasVertex vertex : vertices) {
assertEquals(GraphHelper.getSingleValuedProperty(vertex, Constants.STATE_PROPERTY_KEY, String.class), Id.EntityState.DELETED.name());
}
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerMockTest.java
......@@ -18,36 +18,33 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import org.apache.atlas.AtlasException;
import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.commons.configuration.Configuration;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
public class GraphBackedSearchIndexerMockTest {
public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider {
@Mock
private Configuration configuration;
@Mock
private GraphProvider<TitanGraph> graphProvider;
private AtlasGraph graph;
@Mock
private TitanGraph titanGraph;
@Mock
private TitanManagement titanManagement;
private AtlasGraphManagement management;
@BeforeMethod
public void setup() {
......@@ -56,26 +53,24 @@ public class GraphBackedSearchIndexerMockTest {
@Test
public void testSearchIndicesAreInitializedOnConstructionWhenHAIsDisabled() throws IndexException, RepositoryException {
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false);
when(graphProvider.get()).thenReturn(titanGraph);
when(titanGraph.getManagementSystem()).thenReturn(titanManagement);
when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false);
when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(this, configuration);
verify(titanManagement).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
verify(management).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
}
@Test
public void testSearchIndicesAreNotInitializedOnConstructionWhenHAIsEnabled() throws IndexException, RepositoryException {
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(graphProvider.get()).thenReturn(titanGraph);
when(titanGraph.getManagementSystem()).thenReturn(titanManagement);
when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
new GraphBackedSearchIndexer(graphProvider, configuration);
verifyZeroInteractions(titanManagement);
new GraphBackedSearchIndexer(this, configuration);
verifyZeroInteractions(management);
}
......@@ -83,13 +78,18 @@ public class GraphBackedSearchIndexerMockTest {
public void testIndicesAreReinitializedWhenServerBecomesActive() throws AtlasException {
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(graphProvider.get()).thenReturn(titanGraph);
when(titanGraph.getManagementSystem()).thenReturn(titanManagement);
when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(this, configuration);
graphBackedSearchIndexer.instanceIsActive();
verify(titanManagement).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
verify(management).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
}
@Override
public AtlasGraph get() {
return graph;
}
}
repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperMockTest.java
......@@ -17,51 +17,51 @@
*/
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import java.util.Iterator;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.Iterator;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
public class GraphHelperMockTest {
private GraphHelper graphHelperInstance;
private TitanGraph graph;
private AtlasGraph graph;
@BeforeClass
public void setup() {
MockitoAnnotations.initMocks(this);
graph = mock(TitanGraph.class);
graph = mock(AtlasGraph.class);
graphHelperInstance = GraphHelper.getInstance(graph);
}
@Test(expectedExceptions = RepositoryException.class)
public void testGetOrCreateEdgeLabelWithMaxRetries() throws Exception {
final String edgeLabel = "testLabel";
TitanVertex v1 = mock(TitanVertex.class);
TitanVertex v2 = mock(TitanVertex.class);
AtlasVertex v1 = mock(AtlasVertex.class);
AtlasVertex v2 = mock(AtlasVertex.class);
Iterable noEdgesIterable = new Iterable<Edge>() {
Iterable noEdgesIterable = new Iterable<AtlasEdge>() {
@Override
public Iterator<Edge> iterator() {
return new Iterator<Edge>() {
public Iterator<AtlasEdge> iterator() {
return new Iterator<AtlasEdge>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public Edge next() {
public AtlasEdge next() {
return null;
}
......@@ -71,33 +71,33 @@ public class GraphHelperMockTest {
};
}
};
when(v2.getEdges(Direction.IN)).thenReturn(noEdgesIterable);
when(v1.getEdges(Direction.OUT)).thenReturn(noEdgesIterable);
when(v2.getEdges(AtlasEdgeDirection.IN)).thenReturn(noEdgesIterable);
when(v1.getEdges(AtlasEdgeDirection.OUT)).thenReturn(noEdgesIterable);
when(v1.getId()).thenReturn(new String("1234"));
when(v2.getId()).thenReturn(new String("5678"));
when(graph.addEdge(null, v1, v2, edgeLabel)).thenThrow(new RuntimeException("Unique property constraint violated"));
when(graph.addEdge(v1, v2, edgeLabel)).thenThrow(new RuntimeException("Unique property constraint violated"));
graphHelperInstance.getOrCreateEdge(v1, v2, edgeLabel);
}
@Test
public void testGetOrCreateEdgeLabelWithRetries() throws Exception {
final String edgeLabel = "testLabel";
TitanVertex v1 = mock(TitanVertex.class);
TitanVertex v2 = mock(TitanVertex.class);
Edge edge = mock(Edge.class);
AtlasVertex v1 = mock(AtlasVertex.class);
AtlasVertex v2 = mock(AtlasVertex.class);
AtlasEdge edge = mock(AtlasEdge.class);
Iterable noEdgesIterable = new Iterable<Edge>() {
Iterable noEdgesIterable = new Iterable<AtlasEdge>() {
@Override
public Iterator<Edge> iterator() {
return new Iterator<Edge>() {
public Iterator<AtlasEdge> iterator() {
return new Iterator<AtlasEdge>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public Edge next() {
public AtlasEdge next() {
return null;
}
......@@ -107,15 +107,15 @@ public class GraphHelperMockTest {
};
}
};
when(v2.getEdges(Direction.IN)).thenReturn(noEdgesIterable);
when(v1.getEdges(Direction.OUT)).thenReturn(noEdgesIterable);
when(v2.getEdges(AtlasEdgeDirection.IN)).thenReturn(noEdgesIterable);
when(v1.getEdges(AtlasEdgeDirection.OUT)).thenReturn(noEdgesIterable);
when(v1.getId()).thenReturn(new String("v1"));
when(v2.getId()).thenReturn(new String("v2"));
when(edge.getId()).thenReturn(new String("edge"));
when(graph.addEdge(null, v1, v2, edgeLabel))
when(graph.addEdge(v1, v2, edgeLabel))
.thenThrow(new RuntimeException("Unique property constraint violated")).thenReturn(edge);
Edge redge = graphHelperInstance.getOrCreateEdge(v1, v2, edgeLabel);
AtlasEdge redge = graphHelperInstance.getOrCreateEdge(v1, v2, edgeLabel);
assertEquals(edge, redge);
}
}
repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java
......@@ -25,8 +25,8 @@ import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import java.util.HashMap;
import java.util.Iterator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
......@@ -35,6 +35,9 @@ import javax.inject.Inject;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphHelper.VertexInfo;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
......@@ -44,16 +47,9 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphHelperTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@DataProvider(name = "encodeDecodeTestData")
private Object[][] createTestData() {
......@@ -81,25 +77,14 @@ public class GraphHelperTest {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
new GraphBackedSearchIndexer(graphProvider);
new GraphBackedSearchIndexer();
TestUtils.defineDeptEmployeeTypes(typeSystem);
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
public void tearDown() {
AtlasGraphProvider.cleanup();
}
@Test
......@@ -116,7 +101,7 @@ public class GraphHelperTest {
deptGuid = guid;
}
}
Vertex deptVertex = GraphHelper.getInstance().getVertexForGUID(deptGuid);
AtlasVertex deptVertex = GraphHelper.getInstance().getVertexForGUID(deptGuid);
Set<VertexInfo> compositeVertices = GraphHelper.getInstance().getCompositeVertices(deptVertex);
HashMap<String, VertexInfo> verticesByGuid = new HashMap<>();
for (VertexInfo vertexInfo: compositeVertices) {
......@@ -141,14 +126,13 @@ public class GraphHelperTest {
@Test
public void testGetOutgoingEdgesByLabel() throws Exception {
TitanGraph graph = graphProvider.get();
TitanVertex v1 = graph.addVertex();
TitanVertex v2 = graph.addVertex();
v1.addEdge("l1", v2);
v1.addEdge("l2", v2);
AtlasGraph graph = TestUtils.getGraph();
AtlasVertex v1 = graph.addVertex();
AtlasVertex v2 = graph.addVertex();
graph.addEdge(v1, v2, "l1");
graph.addEdge(v1, v2, "l2");
Iterator<Edge> iterator = GraphHelper.getInstance().getOutGoingEdgesByLabel(v1, "l1");
Iterator<AtlasEdge> iterator = GraphHelper.getInstance().getOutGoingEdgesByLabel(v1, "l1");
assertTrue(iterator.hasNext());
assertTrue(iterator.hasNext());
assertNotNull(iterator.next());
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -18,18 +18,23 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Predicate;
import com.tinkerpop.blueprints.Vertex;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import javax.inject.Inject;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
......@@ -45,11 +50,6 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
@Test
@Guice(modules = RepositoryMetadataModule.class)
public class GraphRepoMapperScaleTest {
......@@ -58,9 +58,6 @@ public class GraphRepoMapperScaleTest {
private static final String TABLE_NAME = "bar";
@Inject
GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
@Inject
......@@ -73,6 +70,9 @@ public class GraphRepoMapperScaleTest {
@BeforeClass
@GraphTransaction
public void setUp() throws Exception {
//force up front graph initialization
TestUtils.getGraph();
searchIndexer = new GraphBackedSearchIndexer(new AtlasGraphProvider(), ApplicationProperties.get());
//Make sure we can cleanup the index directory
Collection<IDataType> typesAdded = TestUtils.createHiveTypes(typeSystem);
searchIndexer.onAdd(typesAdded);
......@@ -80,23 +80,13 @@ public class GraphRepoMapperScaleTest {
@BeforeMethod
public void setupContext() {
RequestContext.createContext();
TestUtils.resetRequestContext();
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
@Test
......@@ -130,7 +120,7 @@ public class GraphRepoMapperScaleTest {
searchWithOutIndex("hive_table.name", "bar-999");
searchWithIndex("hive_table.name", "bar-999");
searchWithIndex("hive_table.created", Compare.GREATER_THAN_EQUAL, TestUtils.TEST_DATE_IN_LONG, 1000);
searchWithIndex("hive_table.created", ComparisionOperator.GREATER_THAN_EQUAL, TestUtils.TEST_DATE_IN_LONG, 1000);
for (int index = 500; index < 600; index++) {
searchWithIndex("hive_table.name", "bar-" + index);
......@@ -140,12 +130,13 @@ public class GraphRepoMapperScaleTest {
}
private void searchWithOutIndex(String key, String value) {
TitanGraph graph = graphProvider.get();
AtlasGraph graph = TestUtils.getGraph();
long start = System.currentTimeMillis();
int count = 0;
try {
GraphQuery query = graph.query().has(key, Compare.EQUAL, value);
for (Vertex ignored : query.vertices()) {
AtlasGraphQuery query = graph.query().has(key, ComparisionOperator.EQUAL, value);
Iterable<AtlasVertex> result = query.vertices();
for (AtlasVertex ignored : result) {
count++;
}
} finally {
......@@ -154,29 +145,33 @@ public class GraphRepoMapperScaleTest {
}
}
private void searchWithIndex(String key, String value) {
TitanGraph graph = graphProvider.get();
AtlasGraph graph = TestUtils.getGraph();
long start = System.currentTimeMillis();
int count = 0;
try {
String queryString = "v.\"" + key + "\":(" + value + ")";
TitanIndexQuery query = graph.indexQuery(Constants.VERTEX_INDEX, queryString);
for (TitanIndexQuery.Result<Vertex> ignored : query.vertices()) {
AtlasIndexQuery query = graph.indexQuery(Constants.VERTEX_INDEX, queryString);
Iterator<AtlasIndexQuery.Result> result = query.vertices();
while(result.hasNext()) {
result.next();
count++;
}
} finally {
System.out.println("Search on [" + key + "=" + value + "] returned results: " + count + ", took " + (
System.currentTimeMillis() - start) + " ms");
System.currentTimeMillis() - start) + " ms");
}
}
private void searchWithIndex(String key, Predicate searchPredicate, Object value, int expectedResults) {
TitanGraph graph = graphProvider.get();
private void searchWithIndex(String key, ComparisionOperator op, Object value, int expectedResults) {
AtlasGraph graph = TestUtils.getGraph();
long start = System.currentTimeMillis();
int count = 0;
try {
GraphQuery query = graph.query().has(key, searchPredicate, value);
for (Vertex ignored : query.vertices()) {
AtlasGraphQuery query = graph.query().has(key, op, value);
Iterable<AtlasVertex> itrble = query.vertices();
for (AtlasVertex ignored : itrble) {
count++;
}
} finally {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment