Skip to content

A
atlas
Commit d2d6ff7d by Suma Shivaprasad
Options

ATLAS-694 Update Atlas code to use graph abstraction layer (jnhagelb via sumasai)

parent 3e4f28f5
Showing with 3534 additions and 2560 deletions
.gitignore
......@@ -18,7 +18,7 @@
target
dependency-reduced-pom.xml
core*.dmp
pom.xml.releaseBackup
# IntelliJ
*.iml
......
addons/falcon-bridge/pom.xml
......@@ -96,6 +96,13 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
......
addons/hdfs-model/pom.xml
......@@ -73,6 +73,13 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
<scope>test</scope>
</dependency>
......
addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
......@@ -18,10 +18,11 @@
package org.apache.atlas.fs.model;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import javax.inject.Inject;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
......@@ -33,11 +34,10 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import scala.Enumeration;
import scala.collection.Iterator;
import javax.inject.Inject;
@Test
@Guice(modules = RepositoryMetadataModule.class)
public class HDFSModelTest {
......@@ -48,9 +48,6 @@ public class HDFSModelTest {
@Inject
private MetadataService metadataService;
@Inject
private GraphProvider<TitanGraph> graphProvider;
@BeforeClass
public void setUp() throws Exception {
}
......@@ -58,17 +55,7 @@ public class HDFSModelTest {
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
@Test
......
addons/hive-bridge/pom.xml
......@@ -149,6 +149,13 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
......@@ -18,8 +18,10 @@
package org.apache.atlas.hive.bridge;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientResponse;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
......@@ -57,9 +59,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientResponse;
/**
* A Bridge Utility that imports metadata from the Hive Meta Store
......
addons/sqoop-bridge/pom.xml
......@@ -151,6 +151,13 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
......
addons/storm-bridge/pom.xml
......@@ -117,6 +117,14 @@
</dependency>
<!-- to bring up atlas server for integration tests -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
......
catalog/pom.xml
......@@ -60,6 +60,18 @@
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-api</artifactId>
</dependency>
<!-- for now, catalog can only be used with Titan 0.5.4 -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
......
catalog/src/main/java/org/apache/atlas/catalog/query/BaseQuery.java
......@@ -18,12 +18,11 @@
package org.apache.atlas.catalog.query;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.gremlin.java.GremlinPipeline;
import com.tinkerpop.pipes.Pipe;
import com.tinkerpop.pipes.filter.PropertyFilterPipe;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.atlas.catalog.Request;
import org.apache.atlas.catalog.VertexWrapper;
import org.apache.atlas.catalog.definition.ResourceDefinition;
......@@ -31,13 +30,15 @@ import org.apache.atlas.catalog.exception.ResourceNotFoundException;
import org.apache.atlas.catalog.projection.Projection;
import org.apache.atlas.catalog.projection.ProjectionResult;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase;
import org.apache.atlas.typesystem.persistence.Id;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.gremlin.java.GremlinPipeline;
import com.tinkerpop.pipes.Pipe;
import com.tinkerpop.pipes.filter.PropertyFilterPipe;
/**
* Base Query implementation.
......@@ -167,7 +168,7 @@ public abstract class BaseQuery implements AtlasQuery {
//todo: abstract
// Underlying method is synchronized and caches the graph in a static field
protected TitanGraph getGraph() {
return TitanGraphProvider.getGraphInstance();
return Titan0GraphDatabase.getGraphInstance();
}
protected VertexWrapper wrapVertex(Vertex v) {
......
distro/src/conf/atlas-application.properties
......@@ -17,6 +17,12 @@
#
######### Graph Database Configs #########
# Graph Database
#Configures the graph database to use. Defaults to Titan 0.5.4.
#atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase
# Graph Storage
atlas.graph.storage.backend=${titan.storage.backend}
atlas.graph.storage.hbase.table=apache_atlas_titan
......
repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java → graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasCardinality.java 100755 → 100644
......@@ -15,36 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.graphdb.titan0.Titan0Database;
import org.apache.commons.configuration.Configuration;
import com.thinkaurelius.titan.core.TitanGraph;
package org.apache.atlas.repository.graphdb;
/**
* Temporary TitanGraphProvider to use until the graph database abstraction
* layer is fully in place. Delegates to the Titan 0.5.4 implementation. This
* will be removed once the abstraction layer is being used.
*/
public class TitanGraphProvider implements GraphProvider<TitanGraph> {
/* (non-Javadoc)
* @see org.apache.atlas.repository.graph.GraphProvider#get()
* Enumeration of vertex property cardinalities.
*
*/
@Override
public TitanGraph get() {
return Titan0Database.getGraphInstance();
public enum AtlasCardinality {
SINGLE(false),
LIST(true),
SET(true);
private boolean isMany;
AtlasCardinality(boolean isMany) {
this.isMany = isMany;
}
public static TitanGraph getGraphInstance() {
return Titan0Database.getGraphInstance();
public boolean isMany() {
return isMany;
}
public static Configuration getConfiguration() throws AtlasException {
return Titan0Database.getConfiguration();
}
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdge.java
......@@ -19,7 +19,7 @@
package org.apache.atlas.repository.graphdb;
/**
* Represent an edge in the graph
* Represent an edge in the graph.
*
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
......@@ -27,14 +27,14 @@ package org.apache.atlas.repository.graphdb;
public interface AtlasEdge<V, E> extends AtlasElement {
/**
* Gets the incoming vertex for this edge
* Gets the incoming vertex for this edge.
* @param in
* @return
*/
AtlasVertex<V, E> getInVertex();
/**
* Gets the outgoing vertex for this edge
* Gets the outgoing vertex for this edge.
*
* @param in
* @return
......@@ -55,6 +55,6 @@ public interface AtlasEdge<V, E> extends AtlasElement {
*
* @return
*/
public E getE();
E getE();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java
......@@ -123,9 +123,10 @@ public interface AtlasElement {
/**
* Creates a Jettison JSONObject from this Element
* Creates a Jettison JSONObject from this Element.
*
* @param propertyKeys The property keys at the root of the element to serialize. If null, then all keys are serialized.
* @param propertyKeys The property keys at the root of the element to serialize.
* If null, then all keys are serialized.
*/
JSONObject toJson(Set<String> propertyKeys) throws JSONException;
......@@ -156,7 +157,7 @@ public interface AtlasElement {
*
* @return
*/
public String getIdForDisplay();
String getIdForDisplay();
/**
* Whether or not an id has been assigned yet for this Element. This can happen if the element has been created
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
......@@ -19,7 +19,6 @@ package org.apache.atlas.repository.graphdb;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import java.util.Set;
import javax.script.ScriptException;
......@@ -27,7 +26,7 @@ import javax.script.ScriptException;
import org.apache.atlas.typesystem.types.IDataType;
/**
* Represents a graph
* Represents a graph.
*
* @param <V> vertex implementation class
* @param <E> edge implementation class
......@@ -35,7 +34,7 @@ import org.apache.atlas.typesystem.types.IDataType;
public interface AtlasGraph<V, E> {
/**
* Adds an edge to the graph
* Adds an edge to the graph.
*
* @param outVertex
* @param inVertex
......@@ -45,14 +44,14 @@ public interface AtlasGraph<V, E> {
AtlasEdge<V, E> addEdge(AtlasVertex<V, E> outVertex, AtlasVertex<V, E> inVertex, String label);
/**
* Adds a vertex to the graph
* Adds a vertex to the graph.
*
* @return
*/
AtlasVertex<V, E> addVertex();
/**
* Removes the specified edge from the graph
* Removes the specified edge from the graph.
*
* @param edge
*/
......@@ -136,18 +135,21 @@ public interface AtlasGraph<V, E> {
Iterable<AtlasVertex<V, E>> getVertices(String key, Object value);
/**
* Creates a graph query
* Creates a graph query.
*
* @return
*/
AtlasGraphQuery<V, E> query();
/**
* Creates an index query
* Creates an index query.
*
* @param indexName index name
* @param queryString the query
*
* @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html">Elastic Search Reference</a> for query syntax
* @see <a
* href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html">
* Elastic Search Reference</a> for query syntax
*/
AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString);
......@@ -183,7 +185,7 @@ public interface AtlasGraph<V, E> {
void clear();
/**
* Converts the graph to gson and writes it to the specified stream
* Converts the graph to gson and writes it to the specified stream.
*
* @param os
* @throws IOException
......@@ -193,59 +195,6 @@ public interface AtlasGraph<V, E> {
//the following methods insulate Atlas from the details
//of the interaction with Gremlin
/**
*
* When we construct Gremlin select queries, the information we request
* is grouped by the vertex the information is coming from. Each vertex
* is assigned a column name which uniquely identifies it. The queries
* are specially formatted so that the value associated with each of
* these column names is an array with the various things we need
* about that particular vertex. The query evaluator creates a mapping
* that knows what index each bit of information is stored at within
* this array.
* <p/>
* When we execute a Gremlin query, the exact java objects we get
* back vary depending on whether Gremlin 2 or Gremlin 3 is being used.
* This method takes as input a raw row result that was obtained by
* executing a Gremlin query and extracts the value that was found
* at the given index in the array for the given column name.
* <p/>
* If the value found is a vertex or edge, it is automatically converted
* to an AtlasVertex/AtlasEdge.
*
* @param rowValue the raw row value that was returned by Gremin
* @param colName the column name to use
* @param idx the index of the value within the column to retrieve.
*
*/
Object getGremlinColumnValue(Object rowValue, String colName, int idx);
/**
* When Gremlin queries are executed, they return
* Vertex and Edge objects that are specific to the underlying
* graph database. This method provides a way to convert these
* objects back into the AtlasVertex/AtlasEdge objects that
* Atlas requires.
*
* @param rawValue the value that was obtained from Gremlin
* @return either an AtlasEdge, an AtlasVertex, or the original
* value depending on whether the rawValue represents an edge,
* vertex, or something else.
*
*/
Object convertGremlinValue(Object rawValue);
/**
* Gremlin 2 and Gremlin 3 represent the results of "path"
* queries differently. This method takes as input the
* path from Gremlin and returns the list of objects in the path.
*
* @param rawValue
* @return
*/
List<Object> convertPathQueryResultToList(Object rawValue);
/**
* This method is used in the generation of queries. It is used to
* convert property values from the value that is stored in the graph
......@@ -303,12 +252,25 @@ public interface AtlasGraph<V, E> {
String getOutputTransformationPredicate(boolean isSelect, boolean isPath);
/**
* Executes a gremlin query, returns an object with the raw
* result.
* Executes a Gremlin script, returns an object with the result.
*
* @param gremlinQuery
* @param isPath whether this is a path query
*
* @return the result from executing the script
*
* @throws ScriptException
*/
Object executeGremlinScript(String query, boolean isPath) throws ScriptException;
/**
* Convenience method to check whether the given property is
* a multi-property.
*
* @param name
* @return
*/
Object executeGremlinScript(String gremlinQuery) throws ScriptException;
boolean isMultiProperty(String name);
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndex.java
......@@ -21,7 +21,7 @@ package org.apache.atlas.repository.graphdb;
import java.util.Set;
/**
* Represents a graph index on the database
* Represents a graph index on the database.
*/
public interface AtlasGraphIndex {
......@@ -39,14 +39,14 @@ public interface AtlasGraphIndex {
boolean isCompositeIndex();
/**
* Indicates if the index applies to edges
* Indicates if the index applies to edges.
*
* @return
*/
boolean isEdgeIndex();
/**
* Indicates if the index applies to vertices
* Indicates if the index applies to vertices.
*
* @return
*/
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
......@@ -18,28 +18,14 @@
package org.apache.atlas.repository.graphdb;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.types.Multiplicity;
import java.util.List;
/**
* Management interface for a graph
* Management interface for a graph.
*
*/
public interface AtlasGraphManagement {
public static final Set<String> MULTIPLICITY_MANY_PROPERTY_KEYS =
Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(
Constants.SUPER_TYPES_PROPERTY_KEY,
Constants.TRAIT_NAMES_PROPERTY_KEY )));
/**
* Checks whether a property with the given key has been defined in the graph schema.
*
......@@ -49,25 +35,7 @@ public interface AtlasGraphManagement {
boolean containsPropertyKey(String key);
/**
* Creates a mixed Vertex index for the graph
*
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
void buildMixedVertexIndex(String index, String backingIndex);
/**
* Creates a mixed Edge index for the graph
*
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
void buildMixedEdgeIndex(String index, String backingIndex);
/**
* Creates a full text index for the given property
* Creates a full text index for the given property.
*
* @param indexName the name of the index to create
* @param propertyKey full text property to index
......@@ -92,7 +60,7 @@ public interface AtlasGraphManagement {
* @param cardinality
* @return
*/
AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, Multiplicity multiplicity);
AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality);
/**
* @param propertyKey
......@@ -107,7 +75,24 @@ public interface AtlasGraphManagement {
* @param propertyKey
* @param isUnique
*/
void createCompositeIndex(String propertyName, AtlasPropertyKey propertyKey, boolean isUnique);
void createExactMatchIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys);
/**
* Looks up the index with the specified name in the graph. Returns null if
* there is no index with the given name.
*
* @param edgeIndex
* @return
*/
AtlasGraphIndex getGraphIndex(String indexName);
/**
* Creates a mixed Vertex index for the graph.
*
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
void createVertexIndex(String name, String backingIndex, List<AtlasPropertyKey> propertyKeys);
/**
* Adds a property key to the given index in the graph.
......@@ -115,16 +100,14 @@ public interface AtlasGraphManagement {
* @param vertexIndex
* @param propertyKey
*/
void addIndexKey(String vertexIndex, AtlasPropertyKey propertyKey);
void addVertexIndexKey(String vertexIndex, AtlasPropertyKey propertyKey);
/**
* Looks up the index with the specified name in the graph. Returns null if
* there is no index with the given name.
* Creates a mixed Edge index for the graph.
*
* @param edgeIndex
* @return
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
AtlasGraphIndex getGraphIndex(String indexName);
void createEdgeIndex(String index, String backingIndex);
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java
......@@ -21,8 +21,6 @@ package org.apache.atlas.repository.graphdb;
import java.util.Collection;
import java.util.List;
import org.apache.atlas.AtlasException;
/**
* Represents a query against the graph within the context of the
* current transaction.
......@@ -30,7 +28,7 @@ import org.apache.atlas.AtlasException;
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
*/
public interface AtlasGraphQuery<V,E> {
public interface AtlasGraphQuery<V, E> {
/**
* Adds a predicate that the returned vertices must have the specified
......@@ -41,7 +39,7 @@ public interface AtlasGraphQuery<V,E> {
* @param value
* @return
*/
AtlasGraphQuery<V,E> has(String propertyKey, Object value);
AtlasGraphQuery<V, E> has(String propertyKey, Object value);
/**
* Adds a predicate that the returned vertices must have the specified
......@@ -52,7 +50,7 @@ public interface AtlasGraphQuery<V,E> {
* @param value
* @return
*/
AtlasGraphQuery<V,E> in(String propertyKey, Collection<? extends Object> values);
AtlasGraphQuery<V, E> in(String propertyKey, Collection<? extends Object> values);
/**
......@@ -71,7 +69,7 @@ public interface AtlasGraphQuery<V,E> {
* @param value
* @return
*/
AtlasGraphQuery<V,E> has(String propertyKey, ComparisionOperator compMethod, Object values);
AtlasGraphQuery<V, E> has(String propertyKey, ComparisionOperator compMethod, Object values);
/**
* Adds a predicate that the vertices returned must satisfy the
......@@ -80,17 +78,20 @@ public interface AtlasGraphQuery<V,E> {
* @param childQueries
* @return
*/
AtlasGraphQuery<V,E> or(List<AtlasGraphQuery<V,E>> childQueries);
AtlasGraphQuery<V, E> or(List<AtlasGraphQuery<V, E>> childQueries);
/**
* Creates a child query that can be used to add "or" conditions
* Creates a child query that can be used to add "or" conditions.
*
* @return
*/
AtlasGraphQuery<V,E> createChildQuery();
AtlasGraphQuery<V, E> createChildQuery();
public static enum ComparisionOperator {
/**
* Comparison operators that can be used in an AtlasGraphQuery.
*/
enum ComparisionOperator {
GREATER_THAN_EQUAL,
EQUAL,
LESS_THAN_EQUAL,
......@@ -106,7 +107,7 @@ public interface AtlasGraphQuery<V,E> {
AtlasGraphQuery<V, E> addConditionsFrom(AtlasGraphQuery<V, E> otherQuery);
/**
* Whether or not this is a child query
* Whether or not this is a child query.
*
* @return
*/
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
......@@ -44,7 +44,7 @@ public interface AtlasIndexQuery<V, E> {
public interface Result<V, E> {
/**
* Gets the vertex for this result
* Gets the vertex for this result.
*/
AtlasVertex<V, E> getVertex();
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasPropertyKey.java
......@@ -19,12 +19,12 @@
package org.apache.atlas.repository.graphdb;
/**
* Represent a property key
* Represent a property key.
*
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
*/
public interface AtlasPropertyKey {
String getName();
AtlasCardinality getCardinality();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graphdb;
/**
* Represents a Vertex
* Represents a Vertex.
*
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java
......@@ -27,7 +27,7 @@ package org.apache.atlas.repository.graphdb;
public interface AtlasVertexQuery<V, E> {
/**
* Specifies the edge direction that should be query
* Specifies the edge direction that should be query.
*
* @param queryDirection
* @return
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graphdb;
/**
* Represents a graph database
* Represents a graph database.
*
* @param <V> vertex class used by the graph database
* @param <E> edge class used by the graph database
......@@ -32,7 +32,7 @@ public interface GraphDatabase<V, E> {
boolean isGraphLoaded();
/**
* Gets the graph, loading it if it has not been loaded already
* Gets the graph, loading it if it has not been loaded already.
* @return
*/
AtlasGraph<V, E> getGraph();
......@@ -46,5 +46,5 @@ public interface GraphDatabase<V, E> {
/**
* Removes the test graph that was created.
*/
void removeTestGraph();
void cleanup();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GremlinVersion.java
......@@ -19,7 +19,7 @@
package org.apache.atlas.repository.graphdb;
/**
* Enumeration of the supported versions of Gremlin
* Enumeration of the supported versions of Gremlin.
*
*
*/
......
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java
......@@ -31,17 +31,17 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
* @param <V>
* @param <E>
*/
public interface NativeTitanGraphQuery<V,E> {
public interface NativeTitanGraphQuery<V, E> {
/**
* Executes the graph query
* Executes the graph query.
* @return
*/
Iterable<AtlasVertex<V,E>> vertices();
Iterable<AtlasVertex<V, E>> vertices();
/**
* Adds an in condition to the query
* Adds an in condition to the query.
*
* @param propertyName
* @param values
......@@ -49,7 +49,7 @@ public interface NativeTitanGraphQuery<V,E> {
void in(String propertyName, Collection<? extends Object> values);
/**
* Adds a has condition to the query
* Adds a has condition to the query.
*
* @param propertyName
* @param op
......
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanQueryFactory.java
......@@ -24,11 +24,11 @@ package org.apache.atlas.repository.graphdb.titan.query;
* @param <V>
* @param <E>
*/
public interface NativeTitanQueryFactory<V,E> {
public interface NativeTitanQueryFactory<V, E> {
/**
* Creates a NativeTitanGraphQuery
* Creates a NativeTitanGraphQuery.
* @return
*/
NativeTitanGraphQuery<V,E> createNativeTitanQuery();
NativeTitanGraphQuery<V, E> createNativeTitanQuery();
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java
......@@ -84,51 +84,51 @@ import org.slf4j.LoggerFactory;
*
*
*/
public abstract class TitanGraphQuery<V,E> implements AtlasGraphQuery<V, E> {
public abstract class TitanGraphQuery<V, E> implements AtlasGraphQuery<V, E> {
private final Logger LOG = LoggerFactory.getLogger(TitanGraphQuery.class);
protected final AtlasGraph<V,E> graph_;
private final OrCondition queryCondition_ = new OrCondition();
private final boolean isChildQuery_;
private static final Logger LOG = LoggerFactory.getLogger(TitanGraphQuery.class);
protected final AtlasGraph<V, E> graph;
private final OrCondition queryCondition = new OrCondition();
private final boolean isChildQuery;
protected abstract NativeTitanQueryFactory<V, E> getQueryFactory();
/**
* Creates a TitanGraphQuery
* Creates a TitanGraphQuery.
*
* @param graph
*/
public TitanGraphQuery(AtlasGraph<V,E> graph) {
graph_ = graph;
isChildQuery_ = false;
public TitanGraphQuery(AtlasGraph<V, E> graph) {
this.graph = graph;
this.isChildQuery = false;
}
/**
* Creates a TitanGraphQuery
* Creates a TitanGraphQuery.
*
* @param graph
* @param isChildQuery
*/
public TitanGraphQuery(AtlasGraph<V,E> graph, boolean isChildQuery) {
graph_ = graph;
isChildQuery_ = isChildQuery;
public TitanGraphQuery(AtlasGraph<V, E> graph, boolean isChildQuery) {
this.graph = graph;
this.isChildQuery = isChildQuery;
}
@Override
public AtlasGraphQuery<V, E> has(String propertyKey, Object value) {
queryCondition_.andWith(new HasPredicate(propertyKey, ComparisionOperator.EQUAL, value));
queryCondition.andWith(new HasPredicate(propertyKey, ComparisionOperator.EQUAL, value));
return this;
}
@Override
public Iterable<AtlasVertex<V, E>> vertices() {
LOG.debug("Executing: " );
LOG.debug(queryCondition_.toString());
LOG.debug("Executing: ");
LOG.debug(queryCondition.toString());
//compute the overall result by unioning the results from all of the
//AndConditions together.
Set<AtlasVertex<V, E>> result = new HashSet<>();
for(AndCondition andExpr : queryCondition_.getAndTerms()) {
NativeTitanGraphQuery<V,E> andQuery = andExpr.create(getQueryFactory());
for(AtlasVertex<V,E> vertex : andQuery.vertices()) {
for(AndCondition andExpr : queryCondition.getAndTerms()) {
NativeTitanGraphQuery<V, E> andQuery = andExpr.create(getQueryFactory());
for(AtlasVertex<V, E> vertex : andQuery.vertices()) {
result.add(vertex);
}
}
......@@ -138,14 +138,14 @@ public abstract class TitanGraphQuery<V,E> implements AtlasGraphQuery<V, E> {
@Override
public AtlasGraphQuery<V, E> has(String propertyKey, ComparisionOperator operator,
Object value) {
queryCondition_.andWith(new HasPredicate(propertyKey, operator, value));
queryCondition.andWith(new HasPredicate(propertyKey, operator, value));
return this;
}
@Override
public AtlasGraphQuery<V, E> in(String propertyKey, Collection<? extends Object> values) {
queryCondition_.andWith(new InPredicate(propertyKey, values));
queryCondition.andWith(new InPredicate(propertyKey, values));
return this;
}
......@@ -159,31 +159,31 @@ public abstract class TitanGraphQuery<V,E> implements AtlasGraphQuery<V, E> {
OrCondition overallChildQuery = new OrCondition(false);
for(AtlasGraphQuery<V, E> atlasChildQuery : childQueries) {
if(! atlasChildQuery.isChildQuery()) {
if (!atlasChildQuery.isChildQuery()) {
throw new IllegalArgumentException(atlasChildQuery + " is not a child query");
}
TitanGraphQuery<V,E> childQuery = (TitanGraphQuery<V,E>)atlasChildQuery;
TitanGraphQuery<V, E> childQuery = (TitanGraphQuery<V, E>)atlasChildQuery;
overallChildQuery.orWith(childQuery.getOrCondition());
}
queryCondition_.andWith(overallChildQuery);
queryCondition.andWith(overallChildQuery);
return this;
}
private OrCondition getOrCondition() {
return queryCondition_;
return queryCondition;
}
@Override
public AtlasGraphQuery<V, E> addConditionsFrom(AtlasGraphQuery<V, E> otherQuery) {
TitanGraphQuery<V, E> childQuery = (TitanGraphQuery<V, E>)otherQuery;
queryCondition_.andWith(childQuery.getOrCondition());
queryCondition.andWith(childQuery.getOrCondition());
return this;
}
@Override
public boolean isChildQuery() {
return isChildQuery_;
return isChildQuery;
}
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java
......@@ -32,27 +32,27 @@ import org.apache.atlas.repository.graphdb.titan.query.NativeTitanQueryFactory;
*/
public class AndCondition {
private List<QueryPredicate> children_ = new ArrayList<>();
private List<QueryPredicate> children = new ArrayList<>();
public AndCondition() {
}
/**
* Adds a query predicate that must be met by vertices
* Adds a query predicate that must be met by vertices.
* @param predicate
*/
public void andWith(QueryPredicate predicate) {
children_.add(predicate);
children.add(predicate);
}
/**
* Adds multiple predicates that much be met by the vertices
* Adds multiple predicates that much be met by the vertices.
*
* @param predicates
*/
public void andWith(List<QueryPredicate> predicates) {
children_.addAll(predicates);
children.addAll(predicates);
}
/**
......@@ -62,17 +62,17 @@ public class AndCondition {
*/
public AndCondition copy() {
AndCondition builder = new AndCondition();
builder.children_.addAll(children_);
builder.children.addAll(children);
return builder;
}
/**
* Gets the query predicates
* Gets the query predicates.
*
* @return
*/
public List<QueryPredicate> getTerms() {
return children_;
return children;
}
/**
......@@ -81,9 +81,9 @@ public class AndCondition {
* @param graph
* @return
*/
public <V,E> NativeTitanGraphQuery<V, E> create(NativeTitanQueryFactory<V,E> factory) {
public <V, E> NativeTitanGraphQuery<V, E> create(NativeTitanQueryFactory<V, E> factory) {
NativeTitanGraphQuery<V, E> query = factory.createNativeTitanQuery();
for (QueryPredicate predicate : children_) {
for (QueryPredicate predicate : children) {
predicate.addTo(query);
}
return query;
......@@ -91,6 +91,6 @@ public class AndCondition {
@Override
public String toString() {
return "AndExpr [predicates=" + children_ + "]";
return "AndExpr [predicates=" + children + "]";
}
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java
......@@ -26,25 +26,24 @@ import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
*/
public class HasPredicate implements QueryPredicate {
private String propertyName_;
private ComparisionOperator op_;
private Object value_;
private String propertyName;
private ComparisionOperator op;
private Object value;
public HasPredicate(String propertyName, ComparisionOperator op, Object value) {
super();
propertyName_ = propertyName;
op_ = op;
value_ = value;
this.propertyName = propertyName;
this.op = op;
this.value = value;
}
@Override
public void addTo(NativeTitanGraphQuery query) {
query.has(propertyName_, op_, value_);
query.has(propertyName, op, value);
}
@Override
public String toString() {
return "HasTerm [propertyName_=" + propertyName_ + ", op_=" + op_ + ", value_=" + value_ + "]";
return "HasTerm [propertyName=" + propertyName + ", op=" + op + ", value=" + value + "]";
}
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/InPredicate.java
......@@ -27,23 +27,23 @@ import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
*/
public class InPredicate implements QueryPredicate {
private String propertyName_;
private Collection<? extends Object> values_;
private String propertyName;
private Collection<? extends Object> values;
public InPredicate(String propertyName, Collection<? extends Object> values) {
super();
propertyName_ = propertyName;
values_ = values;
this.propertyName = propertyName;
this.values = values;
}
@Override
public void addTo(NativeTitanGraphQuery query) {
query.in(propertyName_, values_);
query.in(propertyName, values);
}
@Override
public String toString() {
return "InPredicate [propertyName_=" + propertyName_ + ", values_=" + values_ + "]";
return "InPredicate [propertyName=" + propertyName + ", values=" + values + "]";
}
}
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/OrCondition.java
......@@ -32,20 +32,20 @@ import java.util.List;
*/
public class OrCondition {
private List<AndCondition> children_;
private List<AndCondition> children;
public OrCondition() {
this(true);
}
private OrCondition(List<AndCondition> children) {
children_ = children;
this.children = children;
}
public OrCondition(boolean addInitialTerm) {
children_ = new ArrayList<AndCondition>();
this.children = new ArrayList<AndCondition>();
if (addInitialTerm) {
children_.add(new AndCondition());
children.add(new AndCondition());
}
}
......@@ -58,18 +58,18 @@ public class OrCondition {
*/
public void andWith(QueryPredicate predicate) {
for (AndCondition child : children_) {
for (AndCondition child : children) {
child.andWith(predicate);
}
}
public List<AndCondition> getAndTerms() {
return children_;
return children;
}
/**
* Updates this OrCondition in place so that it matches vertices that satisfy the current
* OrCondition AND that satisfy the provided OrCondition
* OrCondition AND that satisfy the provided OrCondition.
*
* @param other
*/
......@@ -98,30 +98,30 @@ public class OrCondition {
List<AndCondition> expandedExpressionChildren = new ArrayList<AndCondition>();
for (AndCondition otherExprTerm : other.getAndTerms()) {
for (AndCondition currentExpr : children_) {
for (AndCondition currentExpr : children) {
AndCondition currentAndConditionCopy = currentExpr.copy();
currentAndConditionCopy.andWith(otherExprTerm.getTerms());
expandedExpressionChildren.add(currentAndConditionCopy);
}
}
children_ = expandedExpressionChildren;
children = expandedExpressionChildren;
}
/**
* Updates this OrCondition in place so that it matches vertices that satisfy the current
* OrCondition OR that satisfy the provided OrCondition
* OrCondition OR that satisfy the provided OrCondition.
*
* @param other
*/
public void orWith(OrCondition other) {
children_.addAll(other.getAndTerms());
children.addAll(other.getAndTerms());
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("OrCondition [andExprs=");
Iterator<AndCondition> it = children_.iterator();
Iterator<AndCondition> it = children.iterator();
while (it.hasNext()) {
AndCondition andExpr = it.next();
builder.append(andExpr.toString());
......
graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/QueryPredicate.java
......@@ -20,7 +20,7 @@ package org.apache.atlas.repository.graphdb.titan.query.expr;
import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
/**
* Represents a predicate in an AndExpression
* Represents a predicate in an AndExpression.
*/
public interface QueryPredicate {
......
graphdb/graphdb-impls/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb</artifactId>
<version>0.8-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>atlas-graphdb-impls</artifactId>
<!-- Convenience dependency project that allows
the dependency on the correct implementation class
to be configured through profiles. In implementation
specific profiles, the dependency on this project
should be configured to exclude all but the
proper dependency
-->
<description>Apache Atlas Graph Database Implementation Dependencies</description>
<name>Apache Atlas Graph Database Implementation Dependencies</name>
<packaging>pom</packaging>
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Additional dependencies should be added here
as more implementations are introduced. In the
profile corresponding to the implementation in
the root pom.xml, all dependencies except for
that one should be excluded. -->
</dependencies>
</project>
graphdb/pom.xml
......@@ -29,10 +29,14 @@
<description>Apache Atlas Graph Database Projects</description>
<name>Apache Atlas Graph Database Projects</name>
<packaging>pom</packaging>
<properties>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
</properties>
<modules>
<module>api</module>
<module>titan0</module>
<module>common</module>
<module>graphdb-impls</module>
</modules>
</project>
graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java
......@@ -31,8 +31,6 @@ import com.thinkaurelius.titan.graphdb.query.condition.*;
import com.thinkaurelius.titan.graphdb.transaction.StandardTitanTx;
import com.thinkaurelius.titan.graphdb.types.*;
import com.thinkaurelius.titan.graphdb.types.system.ImplicitKey;
import com.thinkaurelius.titan.util.datastructures.Interval;
import com.thinkaurelius.titan.util.datastructures.PointInterval;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.slf4j.Logger;
......@@ -151,13 +149,13 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
@Override
public GraphCentricQueryBuilder orderBy(String key, Order order) {
Preconditions.checkArgument(tx.containsPropertyKey(key),"Provided key does not exist: %s",key);
Preconditions.checkArgument(tx.containsPropertyKey(key), "Provided key does not exist: %s", key);
return orderBy(tx.getPropertyKey(key), order);
}
@Override
public GraphCentricQueryBuilder orderBy(PropertyKey key, Order order) {
Preconditions.checkArgument(key!=null && order!=null,"Need to specify and key and an order");
Preconditions.checkArgument(key!=null && order!=null, "Need to specify and key and an order");
Preconditions.checkArgument(Comparable.class.isAssignableFrom(key.getDataType()),
"Can only order on keys with comparable data type. [%s] has datatype [%s]", key.getName(), key.getDataType());
Preconditions.checkArgument(key.getCardinality()== Cardinality.SINGLE, "Ordering is undefined on multi-valued key [%s]", key.getName());
......@@ -190,7 +188,7 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
}
private QueryDescription describe(ElementCategory category) {
return new StandardQueryDescription(1,constructQuery(category));
return new StandardQueryDescription(1, constructQuery(category));
}
public QueryDescription describeForVertices() {
......@@ -235,13 +233,13 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
//Compile all indexes that cover at least one of the query conditions
final Set<IndexType> indexCandidates = new HashSet<IndexType>();
ConditionUtil.traversal(conditions,new Predicate<Condition<TitanElement>>() {
ConditionUtil.traversal(conditions, new Predicate<Condition<TitanElement>>() {
@Override
public boolean apply(@Nullable Condition<TitanElement> condition) {
if (condition instanceof PredicateCondition) {
RelationType type = ((PredicateCondition<RelationType,TitanElement>)condition).getKey();
RelationType type = ((PredicateCondition<RelationType, TitanElement>)condition).getKey();
Preconditions.checkArgument(type!=null && type.isPropertyKey());
Iterables.addAll(indexCandidates,Iterables.filter(((InternalRelationType) type).getKeyIndexes(), new Predicate<IndexType>() {
Iterables.addAll(indexCandidates, Iterables.filter(((InternalRelationType) type).getKeyIndexes(), new Predicate<IndexType>() {
@Override
public boolean apply(@Nullable IndexType indexType) {
return indexType.getElement()==resultType;
......@@ -275,12 +273,12 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
//Check that this index actually applies in case of a schema constraint
if (index.hasSchemaTypeConstraint()) {
TitanSchemaType type = index.getSchemaTypeConstraint();
Map.Entry<Condition,Collection<Object>> equalCon = getEqualityConditionValues(conditions,ImplicitKey.LABEL);
Map.Entry<Condition, Collection<Object>> equalCon = getEqualityConditionValues(conditions, ImplicitKey.LABEL);
if (equalCon==null) continue;
Collection<Object> labels = equalCon.getValue();
assert labels.size()>=1;
if (labels.size()>1) {
log.warn("The query optimizer currently does not support multiple label constraints in query: {}",this);
log.warn("The query optimizer currently does not support multiple label constraints in query: {}", this);
continue;
}
if (!type.getName().equals((String)Iterables.getOnlyElement(labels))) continue;
......@@ -288,11 +286,11 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
}
if (index.isCompositeIndex()) {
subcondition = indexCover((CompositeIndexType) index,conditions,subcover);
subcondition = indexCover((CompositeIndexType) index, conditions, subcover);
} else {
subcondition = indexCover((MixedIndexType) index,conditions,serializer,subcover);
subcondition = indexCover((MixedIndexType) index, conditions, serializer, subcover);
if (coveredClauses.isEmpty() && !supportsSort
&& indexCoversOrder((MixedIndexType)index,orders)) supportsSort=true;
&& indexCoversOrder((MixedIndexType)index, orders)) supportsSort=true;
}
if (subcondition==null) continue;
assert !subcover.isEmpty();
......@@ -324,10 +322,10 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
log.debug("Index chosen for query {} {} " , bestCandidate.isCompositeIndex() ? "COMPOSITE" : "MIXED", coveredClauses);
if (bestCandidate.isCompositeIndex()) {
jointQuery.add((CompositeIndexType)bestCandidate,
serializer.getQuery((CompositeIndexType)bestCandidate,(List<Object[]>)candidateSubcondition));
serializer.getQuery((CompositeIndexType)bestCandidate, (List<Object[]>)candidateSubcondition));
} else {
jointQuery.add((MixedIndexType)bestCandidate,
serializer.getQuery((MixedIndexType)bestCandidate,(Condition)candidateSubcondition,orders));
serializer.getQuery((MixedIndexType)bestCandidate, (Condition)candidateSubcondition, orders));
}
} else {
break;
......@@ -371,7 +369,7 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
Set<Condition> coveredClauses = new HashSet<Condition>(fields.length);
List<Object[]> indexCovers = new ArrayList<Object[]>(4);
constructIndexCover(indexValues,0,fields,condition,indexCovers,coveredClauses);
constructIndexCover(indexValues, 0, fields, condition, indexCovers, coveredClauses);
if (!indexCovers.isEmpty()) {
covered.addAll(coveredClauses);
return indexCovers;
......@@ -385,31 +383,31 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
indexCovers.add(indexValues);
} else {
IndexField field = fields[position];
Map.Entry<Condition,Collection<Object>> equalCon = getEqualityConditionValues(condition,field.getFieldKey());
Map.Entry<Condition, Collection<Object>> equalCon = getEqualityConditionValues(condition, field.getFieldKey());
if (equalCon!=null) {
coveredClauses.add(equalCon.getKey());
assert equalCon.getValue().size()>0;
for (Object value : equalCon.getValue()) {
Object[] newValues = Arrays.copyOf(indexValues,fields.length);
Object[] newValues = Arrays.copyOf(indexValues, fields.length);
newValues[position]=value;
constructIndexCover(newValues,position+1,fields,condition,indexCovers,coveredClauses);
constructIndexCover(newValues, position+1, fields, condition, indexCovers, coveredClauses);
}
} else return;
}
}
private static final Map.Entry<Condition,Collection<Object>> getEqualityConditionValues(Condition<TitanElement> condition, RelationType type) {
private static final Map.Entry<Condition, Collection<Object>> getEqualityConditionValues(Condition<TitanElement> condition, RelationType type) {
for (Condition c : condition.getChildren()) {
if (c instanceof Or) {
Map.Entry<RelationType,Collection> orEqual = QueryUtil.extractOrCondition((Or)c);
Map.Entry<RelationType, Collection> orEqual = QueryUtil.extractOrCondition((Or)c);
if (orEqual!=null && orEqual.getKey().equals(type) && !orEqual.getValue().isEmpty()) {
return new AbstractMap.SimpleImmutableEntry(c,orEqual.getValue());
return new AbstractMap.SimpleImmutableEntry(c, orEqual.getValue());
}
} else if (c instanceof PredicateCondition) {
PredicateCondition<RelationType, TitanRelation> atom = (PredicateCondition)c;
if (atom.getKey().equals(type) && atom.getPredicate()==Cmp.EQUAL && atom.getValue()!=null) {
return new AbstractMap.SimpleImmutableEntry(c,ImmutableList.of(atom.getValue()));
return new AbstractMap.SimpleImmutableEntry(c, ImmutableList.of(atom.getValue()));
}
}
......@@ -423,7 +421,7 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
assert condition instanceof And;
And<TitanElement> subcondition = new And<TitanElement>(condition.numChildren());
for (Condition<TitanElement> subclause : condition.getChildren()) {
if (coversAll(index,subclause,indexInfo)) {
if (coversAll(index, subclause, indexInfo)) {
subcondition.add(subclause);
covered.add(subclause);
}
......@@ -446,10 +444,10 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue
if (fields[i].getFieldKey().equals(key)) match = fields[i];
}
if (match==null) return false;
return indexInfo.supports(index,match,atom.getPredicate());
return indexInfo.supports(index, match, atom.getPredicate());
} else {
for (Condition<TitanElement> child : condition.getChildren()) {
if (!coversAll(index,child,indexInfo)) return false;
if (!coversAll(index, child, indexInfo)) return false;
}
return true;
}
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
......@@ -18,9 +18,11 @@
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.tinkerpop.blueprints.Edge;
......@@ -39,8 +41,8 @@ public final class GraphDbObjectFactory {
/**
* Creates a Titan0Edge that corresponds to the given Gremlin Edge.
*
* @param source
* @return
* @param graph The graph the edge should be created in
* @param source The gremlin edge
*/
public static Titan0Edge createEdge(Titan0Graph graph, Edge source) {
......@@ -52,9 +54,8 @@ public final class GraphDbObjectFactory {
/**
* Creates a Titan0GraphQuery that corresponds to the given GraphQuery.
* @param source
*
* @return
* @param graph the graph that is being quried
*/
public static Titan0GraphQuery createQuery(Titan0Graph graph) {
......@@ -64,8 +65,8 @@ public final class GraphDbObjectFactory {
/**
* Creates a Titan0Vertex that corresponds to the given Gremlin Vertex.
*
* @param source
* @return
* @param graph The graph that contains the vertex
* @param source the Gremlin vertex
*/
public static Titan0Vertex createVertex(Titan0Graph graph, Vertex source) {
......@@ -76,8 +77,8 @@ public final class GraphDbObjectFactory {
}
/**
* @param propertyKey
* @return
* @param propertyKey The Gremlin propertyKey.
*
*/
public static Titan0PropertyKey createPropertyKey(PropertyKey propertyKey) {
if (propertyKey == null) {
......@@ -87,7 +88,7 @@ public final class GraphDbObjectFactory {
}
/**
* @param index
* @param index The gremlin index.
* @return
*/
public static AtlasGraphIndex createGraphIndex(TitanGraphIndex index) {
......@@ -97,4 +98,20 @@ public final class GraphDbObjectFactory {
return new Titan0GraphIndex(index);
}
/**
* Converts a Multiplicity to a Cardinality.
*
* @param cardinality
* @return
*/
public static AtlasCardinality createCardinality(Cardinality cardinality) {
if (cardinality == Cardinality.SINGLE) {
return AtlasCardinality.SINGLE;
} else if (cardinality == Cardinality.LIST) {
return AtlasCardinality.LIST;
}
return AtlasCardinality.SET;
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Element.java
......@@ -72,7 +72,20 @@ public class Titan0Element<T extends Element> implements AtlasElement {
@Override
public <U> U getProperty(String propertyName, Class<U> clazz) {
return (U)convert(wrappedElement.getProperty(propertyName), clazz);
Object rawValue = wrappedElement.getProperty(propertyName);
if (rawValue == null) {
return null;
}
if (AtlasEdge.class == clazz) {
return (U)graph.getEdge(rawValue.toString());
}
if (AtlasVertex.class == clazz) {
return (U)graph.getVertex(rawValue.toString());
}
return (U)rawValue;
}
/**
......@@ -136,7 +149,7 @@ public class Titan0Element<T extends Element> implements AtlasElement {
@Override
public boolean equals(Object other) {
if(other == null) {
if (other == null) {
return false;
}
if (other.getClass() != getClass()) {
......@@ -154,9 +167,8 @@ public class Titan0Element<T extends Element> implements AtlasElement {
@Override
public boolean exists() {
try {
return ! ((TitanElement)wrappedElement).isRemoved();
}
catch(IllegalStateException e) {
return !((TitanElement)wrappedElement).isRemoved();
} catch(IllegalStateException e) {
return false;
}
......@@ -192,18 +204,6 @@ public class Titan0Element<T extends Element> implements AtlasElement {
return getId().toString();
}
private <T> T convert(Object propertyValue, Class<T> clazz) {
if(propertyValue == null) {
return null;
}
if(AtlasEdge.class.isAssignableFrom(clazz)) {
return (T)graph.getEdge(propertyValue.toString());
}
if(AtlasVertex.class.isAssignableFrom(clazz)) {
return (T)graph.getVertex(propertyValue.toString());
}
return (T)propertyValue;
}
@Override
......@@ -211,14 +211,13 @@ public class Titan0Element<T extends Element> implements AtlasElement {
List<String> value = getListProperty(propertyName);
if(value == null) {
if (value == null) {
return null;
}
if(AtlasEdge.class.isAssignableFrom(elementType)) {
if (AtlasEdge.class == elementType) {
return (List<V>)Lists.transform(value, new Function<String,AtlasEdge>(){
return (List<V>)Lists.transform(value, new Function<String, AtlasEdge>(){
@Override
public AtlasEdge apply(String input) {
......@@ -227,9 +226,9 @@ public class Titan0Element<T extends Element> implements AtlasElement {
});
}
if(AtlasVertex.class.isAssignableFrom(elementType)) {
if (AtlasVertex.class == elementType) {
return (List<V>)Lists.transform(value, new Function<String,AtlasVertex>(){
return (List<V>)Lists.transform(value, new Function<String, AtlasVertex>(){
@Override
public AtlasVertex apply(String input) {
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
......@@ -19,8 +19,14 @@ package org.apache.atlas.repository.graphdb.titan0;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.script.Bindings;
......@@ -42,9 +48,13 @@ import org.apache.atlas.utils.IteratorToIterableAdapter;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
......@@ -58,8 +68,25 @@ import com.tinkerpop.pipes.util.structures.Row;
*/
public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
public Titan0Graph() {
private final Set<String> multiProperties;
public Titan0Graph() {
//determine multi-properties once at startup
TitanManagement mgmt = null;
try {
mgmt = Titan0GraphDatabase.getGraphInstance().getManagementSystem();
Iterable<PropertyKey> keys = mgmt.getRelationTypes(PropertyKey.class);
multiProperties = Collections.synchronizedSet(new HashSet<String>());
for(PropertyKey key : keys) {
if (key.getCardinality() != Cardinality.SINGLE) {
multiProperties.add(key.getName());
}
}
} finally {
if (mgmt != null) {
mgmt.rollback();
}
}
}
@Override
......@@ -74,6 +101,7 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
}
}
@Override
public AtlasGraphQuery<Titan0Vertex, Titan0Edge> query() {
......@@ -134,7 +162,7 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
@Override
public AtlasGraphManagement getManagementSystem() {
return new Titan0DatabaseManager(getGraph().getManagementSystem());
return new Titan0GraphManagement(this, getGraph().getManagementSystem());
}
@Override
......@@ -170,20 +198,31 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
return wrapVertices(result);
}
@Override
public Object getGremlinColumnValue(Object rowValue, String colName, int idx) {
Row<List> rV = (Row<List>) rowValue;
Object value = rV.getColumn(colName).get(idx);
return convertGremlinValue(value);
}
private Object convertGremlinValue(Object rawValue) {
@Override
public Object convertGremlinValue(Object rawValue) {
if (rawValue instanceof Vertex) {
return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue);
}
if (rawValue instanceof Edge) {
} else if (rawValue instanceof Edge) {
return GraphDbObjectFactory.createEdge(this, (Edge) rawValue);
} else if (rawValue instanceof Row) {
Row rowValue = (Row)rawValue;
Map<String, Object> result = new HashMap<>(rowValue.size());
List<String> columnNames = rowValue.getColumnNames();
for(int i = 0; i < rowValue.size(); i++) {
String key = columnNames.get(i);
Object value = convertGremlinValue(rowValue.get(i));
result.put(key, value);
}
return result;
} else if (rawValue instanceof List) {
return Lists.transform((List)rawValue, new Function<Object, Object>() {
@Override
public Object apply(Object input) {
return convertGremlinValue(input);
}
});
} else if (rawValue instanceof Collection) {
throw new UnsupportedOperationException("Unhandled collection type: " + rawValue.getClass());
}
return rawValue;
}
......@@ -194,11 +233,11 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
return GremlinVersion.TWO;
}
@Override
public List<Object> convertPathQueryResultToList(Object rawValue) {
private List<Object> convertPathQueryResultToList(Object rawValue) {
return (List<Object>) rawValue;
}
@Override
public void clear() {
TitanGraph graph = getGraph();
......@@ -211,7 +250,7 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
private TitanGraph getGraph() {
// return the singleton instance of the graph in the plugin
return Titan0Database.getGraphInstance();
return Titan0GraphDatabase.getGraphInstance();
}
@Override
......@@ -219,15 +258,24 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
GraphSONWriter.outputGraph(getGraph(), os);
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraph#executeGremlinScript(java.
* lang.String)
*/
@Override
public Object executeGremlinScript(String gremlinQuery) throws ScriptException {
public Object executeGremlinScript(String query, boolean isPath) throws ScriptException {
Object result = executeGremlinScript(query);
if (isPath) {
List<Object> path = convertPathQueryResultToList(result);
List<Object> convertedResult = new ArrayList<>(path.size());
for(Object o : path) {
convertedResult.add(convertGremlinValue(o));
}
return convertedResult;
} else {
return convertGremlinValue(result);
}
}
private Object executeGremlinScript(String gremlinQuery) throws ScriptException {
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
......@@ -298,4 +346,14 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
}
});
}
@Override
public boolean isMultiProperty(String propertyName) {
return multiProperties.contains(propertyName);
}
public void addMultiProperties(Set<String> names) {
multiProperties.addAll(names);
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Database.java → graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphDatabase.java
......@@ -42,9 +42,9 @@ import com.thinkaurelius.titan.diskstorage.solr.Solr5Index;
/**
* Titan 0.5.4 implementation of GraphDatabase.
*/
public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
public class Titan0GraphDatabase implements GraphDatabase<Titan0Vertex, Titan0Edge> {
private static final Logger LOG = LoggerFactory.getLogger(Titan0Database.class);
private static final Logger LOG = LoggerFactory.getLogger(Titan0GraphDatabase.class);
/**
* Constant for the configuration property that indicates the prefix.
......@@ -57,7 +57,8 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
public static final String INDEX_BACKEND_ES = "elasticsearch";
private static volatile TitanGraph graphInstance;
private static volatile Titan0Graph atlasGraphInstance = null;
private static volatile TitanGraph graphInstance = null;
public static Configuration getConfiguration() throws AtlasException {
Configuration configProperties = ApplicationProperties.get();
......@@ -102,7 +103,7 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
public static TitanGraph getGraphInstance() {
if (graphInstance == null) {
synchronized (Titan0Database.class) {
synchronized (Titan0GraphDatabase.class) {
if (graphInstance == null) {
Configuration config;
try {
......@@ -112,6 +113,7 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
}
graphInstance = TitanFactory.open(config);
atlasGraphInstance = new Titan0Graph();
validateIndexBackend(config);
}
}
......@@ -121,7 +123,7 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
public static void unload() {
synchronized (Titan0Database.class) {
synchronized (Titan0GraphDatabase.class) {
if (graphInstance == null) {
return;
}
......@@ -167,7 +169,7 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
// force graph loading up front to avoid bootstrapping
// issues
getGraphInstance();
return new Titan0Graph();
return atlasGraphInstance;
}
@Override
......@@ -183,19 +185,17 @@ public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
}
@Override
public void removeTestGraph() {
public void cleanup() {
try {
getGraphInstance().shutdown();
}
catch(Throwable t) {
} catch(Throwable t) {
LOG.warn("Could not shutdown test TitanGraph", t);
t.printStackTrace();
}
try {
TitanCleanup.clear(getGraphInstance());
}
catch(Throwable t) {
} catch(Throwable t) {
LOG.warn("Could not clear test TitanGraph", t);
t.printStackTrace();
}
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseManager.java → graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java
......@@ -17,10 +17,14 @@
*/
package org.apache.atlas.repository.graphdb.titan0;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -37,24 +41,23 @@ import com.tinkerpop.blueprints.Vertex;
/**
* Titan 0.5.4 implementation of AtlasGraphManagement.
*/
public class Titan0DatabaseManager implements AtlasGraphManagement {
public class Titan0GraphManagement implements AtlasGraphManagement {
private static final Logger LOG = LoggerFactory.getLogger(Titan0DatabaseManager.class);
private static final Logger LOG = LoggerFactory.getLogger(Titan0GraphManagement.class);
private TitanManagement management;
private final Titan0Graph graph;
private final TitanManagement management;
public Titan0DatabaseManager(TitanManagement managementSystem) {
private Set<String> newMultProperties = new HashSet<>();
management = managementSystem;
}
public Titan0GraphManagement(Titan0Graph graph, TitanManagement managementSystem) {
@Override
public void buildMixedVertexIndex(String index, String backingIndex) {
buildMixedIndex(index, Vertex.class, backingIndex);
this.graph = graph;
management = managementSystem;
}
@Override
public void buildMixedEdgeIndex(String index, String backingIndex) {
public void createEdgeIndex(String index, String backingIndex) {
buildMixedIndex(index, Edge.class, backingIndex);
}
......@@ -86,69 +89,63 @@ public class Titan0DatabaseManager implements AtlasGraphManagement {
@Override
public void commit() {
graph.addMultiProperties(newMultProperties);
newMultProperties.clear();
management.commit();
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraphManagement#makePropertyKey(
* java.lang.String, java.lang.Class,
* org.apache.atlas.typesystem.types.Multiplicity)
*/
@Override
public AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, Multiplicity multiplicity) {
public AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality) {
if (cardinality.isMany()) {
newMultProperties.add(propertyName);
}
PropertyKeyMaker propertyKeyBuilder = management.makePropertyKey(propertyName).dataType(propertyClass);
if (multiplicity != null) {
Cardinality cardinality = TitanObjectFactory.createCardinality(multiplicity);
propertyKeyBuilder.cardinality(cardinality);
if (cardinality != null) {
Cardinality titanCardinality = TitanObjectFactory.createCardinality(cardinality);
propertyKeyBuilder.cardinality(titanCardinality);
}
PropertyKey propertyKey = propertyKeyBuilder.make();
return GraphDbObjectFactory.createPropertyKey(propertyKey);
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraphManagement#getPropertyKey(
* java.lang.String)
*/
@Override
public AtlasPropertyKey getPropertyKey(String propertyName) {
return GraphDbObjectFactory.createPropertyKey(management.getPropertyKey(propertyName));
}
/*
* (non-Javadoc)
*
* @see org.apache.atlas.repository.graphdb.AtlasGraphManagement#
* createCompositeIndex(java.lang.String,
* org.apache.atlas.repository.graphdb.AtlasPropertyKey, boolean)
*/
@Override
public void createCompositeIndex(String propertyName, AtlasPropertyKey propertyKey, boolean enforceUniqueness) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class).addKey(titanKey);
public void createExactMatchIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for(AtlasPropertyKey key : propertyKeys) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
indexBuilder.addKey(titanKey);
}
if (enforceUniqueness) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
/*
* (non-Javadoc)
*
* @see
* org.apache.atlas.repository.graphdb.AtlasGraphManagement#addIndexKey(java
* .lang.String, org.apache.atlas.repository.graphdb.AtlasPropertyKey)
*/
@Override
public void addIndexKey(String indexName, AtlasPropertyKey propertyKey) {
public void createVertexIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for(AtlasPropertyKey key : propertyKeys) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
indexBuilder.addKey(titanKey);
}
indexBuilder.buildMixedIndex(backingIndex);
}
@Override
public void addVertexIndexKey(String indexName, AtlasPropertyKey propertyKey) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
TitanGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, titanKey);
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java
......@@ -59,7 +59,7 @@ public class Titan0IndexQuery implements AtlasIndexQuery<Titan0Vertex, Titan0Edg
private final class ResultImpl implements AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> {
private TitanIndexQuery.Result<Vertex> wrappedResult;
public ResultImpl(TitanIndexQuery.Result<Vertex> source) {
ResultImpl(TitanIndexQuery.Result<Vertex> source) {
wrappedResult = source;
}
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0PropertyKey.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import com.thinkaurelius.titan.core.PropertyKey;
......@@ -51,6 +52,11 @@ public class Titan0PropertyKey implements AtlasPropertyKey {
}
@Override
public AtlasCardinality getCardinality() {
return GraphDbObjectFactory.createCardinality(wrappedPropertyKey.getCardinality());
}
@Override
public boolean equals(Object other) {
if (!(other instanceof Titan0PropertyKey)) {
return false;
......@@ -66,4 +72,11 @@ public class Titan0PropertyKey implements AtlasPropertyKey {
return result;
}
@Override
public String toString() {
return wrappedPropertyKey.getName();
}
}
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Vertex.java
......@@ -22,7 +22,6 @@ import java.util.Collection;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
......@@ -62,7 +61,7 @@ public class Titan0Vertex extends Titan0Element<Vertex> implements AtlasVertex<T
@Override
public <T> T getProperty(String propertyName, Class<T> clazz) {
if (AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS.contains(propertyName)) {
if (graph.isMultiProperty(propertyName)) {
// throw exception in this case to be consistent with Titan 1.0.0
// behavior.
throw new IllegalStateException();
......@@ -78,7 +77,7 @@ public class Titan0Vertex extends Titan0Element<Vertex> implements AtlasVertex<T
// For consistency with Titan 1.0.0, treat sets of multiplicity many
// properties as adds. Handle this here since this is an uncommon
// occurrence.
if (AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS.contains(propertyName)) {
if (graph.isMultiProperty(propertyName)) {
addProperty(propertyName, value);
} else {
throw e;
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/TitanObjectFactory.java
......@@ -17,9 +17,9 @@
*/
package org.apache.atlas.repository.graphdb.titan0;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.typesystem.types.Multiplicity;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
......@@ -63,17 +63,18 @@ public final class TitanObjectFactory {
* @param multiplicity
* @return
*/
public static Cardinality createCardinality(Multiplicity multiplicity) {
public static Cardinality createCardinality(AtlasCardinality cardinality) {
switch(cardinality) {
if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
case SINGLE:
return Cardinality.SINGLE;
} else if (multiplicity == Multiplicity.COLLECTION) {
case LIST:
return Cardinality.LIST;
} else if (multiplicity == Multiplicity.SET) {
case SET:
return Cardinality.SET;
default:
throw new IllegalStateException("Unrecognized cardinality: " + cardinality);
}
// default to LIST as this is the most forgiving
return Cardinality.LIST;
}
public static PropertyKey createPropertyKey(AtlasPropertyKey key) {
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java
......@@ -22,7 +22,7 @@ import java.util.Collection;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
import org.apache.atlas.repository.graphdb.titan0.Titan0Database;
import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase;
import org.apache.atlas.repository.graphdb.titan0.Titan0Edge;
import org.apache.atlas.repository.graphdb.titan0.Titan0Graph;
import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex;
......@@ -38,26 +38,26 @@ import com.tinkerpop.blueprints.Compare;
* @author jeff
*
*/
public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Vertex,Titan0Edge> {
public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Vertex, Titan0Edge> {
private Titan0Graph graph_;
private TitanGraphQuery<?> query_;
private Titan0Graph graph;
private TitanGraphQuery<?> query;
public NativeTitan0GraphQuery(Titan0Graph graph) {
query_ = Titan0Database.getGraphInstance().query();
graph_ = graph;
query = Titan0GraphDatabase.getGraphInstance().query();
this.graph = graph;
}
@Override
public Iterable<AtlasVertex<Titan0Vertex,Titan0Edge>> vertices() {
Iterable it = query_.vertices();
return graph_.wrapVertices(it);
public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices() {
Iterable it = query.vertices();
return graph.wrapVertices(it);
}
@Override
public void in(String propertyName, Collection<? extends Object> values) {
query_.has(propertyName, Contain.IN, values);
query.has(propertyName, Contain.IN, values);
}
......@@ -66,7 +66,7 @@ public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Verte
Compare c = getGremlinPredicate(op);
TitanPredicate pred = TitanPredicate.Converter.convert(c);
query_.has(propertyName, pred, value);
query.has(propertyName, pred, value);
}
private Compare getGremlinPredicate(ComparisionOperator op) {
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/Titan0GraphQuery.java
......@@ -28,7 +28,8 @@ import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex;
/**
* Titan 0.5.4 implementation of AtlasGraphQuery.
*/
public class Titan0GraphQuery extends TitanGraphQuery<Titan0Vertex,Titan0Edge> implements NativeTitanQueryFactory<Titan0Vertex,Titan0Edge> {
public class Titan0GraphQuery extends TitanGraphQuery<Titan0Vertex, Titan0Edge>
implements NativeTitanQueryFactory<Titan0Vertex, Titan0Edge> {
public Titan0GraphQuery(Titan0Graph graph, boolean isChildQuery) {
super(graph, isChildQuery);
......@@ -40,17 +41,17 @@ public class Titan0GraphQuery extends TitanGraphQuery<Titan0Vertex,Titan0Edge> i
@Override
public AtlasGraphQuery<Titan0Vertex, Titan0Edge> createChildQuery() {
return new Titan0GraphQuery((Titan0Graph)graph_, true);
return new Titan0GraphQuery((Titan0Graph)graph, true);
}
@Override
protected NativeTitanQueryFactory<Titan0Vertex,Titan0Edge> getQueryFactory() {
protected NativeTitanQueryFactory<Titan0Vertex, Titan0Edge> getQueryFactory() {
return this;
}
@Override
public NativeTitanGraphQuery<Titan0Vertex, Titan0Edge> createNativeTitanQuery() {
return new NativeTitan0GraphQuery((Titan0Graph)graph_);
return new NativeTitan0GraphQuery((Titan0Graph)graph);
}
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java
......@@ -19,14 +19,15 @@
package org.apache.atlas.repository.graphdb.titan0;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
......@@ -36,75 +37,49 @@ import org.testng.annotations.BeforeClass;
*/
public abstract class AbstractGraphDatabaseTest {
private static class RunnableWrapper implements Runnable {
private final Runnable r;
private Throwable exceptionThrown_ = null;
private RunnableWrapper(Runnable r) {
this.r = r;
}
@Override
public void run() {
try {
r.run();
}
catch(Throwable e) {
exceptionThrown_ = e;
}
}
public Throwable getExceptionThrown() {
return exceptionThrown_;
}
}
protected static final String WEIGHT_PROPERTY = "weight";
protected static final String TRAIT_NAMES = Constants.TRAIT_NAMES_PROPERTY_KEY;
protected static final String typeProperty = "__type";
protected static final String typeSystem = "typeSystem";
protected static final String TYPE_PROPERTY_NAME = "__type";
protected static final String TYPESYSTEM = "TYPESYSTEM";
/**
*
*/
private static final String BACKING_INDEX_NAME = "backing";
private AtlasGraph<?,?> graph = null;
private AtlasGraph<?, ?> graph = null;
@AfterMethod
public void commitGraph() {
//force any pending actions to be committed so we can be sure they don't cause errors.
pushChangesAndFlushCache();
graph.commit();
}
protected <V, E> void pushChangesAndFlushCache() {
AtlasGraph<V, E> graph = getGraph();
graph.commit();
}
public AbstractGraphDatabaseTest() {
super();
}
@BeforeClass
public static void createIndices() {
Titan0Database db = new Titan0Database();
Titan0GraphDatabase db = new Titan0GraphDatabase();
AtlasGraphManagement mgmt = db.getGraph().getManagementSystem();
if(mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
mgmt.buildMixedVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX);
if (mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
mgmt.createVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX,
Collections.<AtlasPropertyKey>emptyList());
}
mgmt.makePropertyKey("age13",Integer.class, Multiplicity.OPTIONAL);
mgmt.makePropertyKey("age13", Integer.class, AtlasCardinality.SINGLE);
createIndices(mgmt, "name", String.class, false, Multiplicity.REQUIRED);
createIndices(mgmt, WEIGHT_PROPERTY, Integer.class, false, Multiplicity.OPTIONAL);
createIndices(mgmt, "size15", String.class, false, Multiplicity.REQUIRED);
createIndices(mgmt, "typeName", String.class, false, Multiplicity.REQUIRED);
createIndices(mgmt, "__type", String.class, false, Multiplicity.REQUIRED);
createIndices(mgmt, Constants.GUID_PROPERTY_KEY, String.class, true, Multiplicity.REQUIRED);
createIndices(mgmt, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Multiplicity.SET);
createIndices(mgmt, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Multiplicity.SET);
createIndices(mgmt, "name", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, WEIGHT_PROPERTY, Integer.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "size15", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "typeName", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, "__type", String.class, false, AtlasCardinality.SINGLE);
createIndices(mgmt, Constants.GUID_PROPERTY_KEY, String.class, true, AtlasCardinality.SINGLE);
createIndices(mgmt, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET);
createIndices(mgmt, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET);
mgmt.commit();
}
@AfterMethod
public void commitGraph() {
//force any pending actions to be committed so we can be sure they don't cause errors.
pushChangesAndFlushCache();
getGraph().commit();
}
@AfterClass
public static void cleanUp() {
Titan0Graph graph = new Titan0Graph();
......@@ -112,32 +87,31 @@ public abstract class AbstractGraphDatabaseTest {
}
private static void createIndices(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, Multiplicity cardinality) {
protected <V, E> void pushChangesAndFlushCache() {
getGraph().commit();
}
private static void createIndices(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality) {
if(management.containsPropertyKey(propertyName)) {
if (management.containsPropertyKey(propertyName)) {
//index was already created
return;
}
AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
try {
if(propertyClass != Integer.class) {
management.addIndexKey(BACKING_INDEX_NAME, key);
if (propertyClass != Integer.class) {
management.addVertexIndexKey(BACKING_INDEX_NAME, key);
}
}
catch(Throwable t) {
} catch(Throwable t) {
//ok
t.printStackTrace();
}
try {
//if(propertyClass != Integer.class) {
management.createCompositeIndex(propertyName, key, isUnique);
//}
}
catch(Throwable t) {
management.createExactMatchIndex(propertyName, isUnique, Collections.singletonList(key));
} catch(Throwable t) {
//ok
t.printStackTrace();
}
......@@ -145,19 +119,14 @@ public abstract class AbstractGraphDatabaseTest {
}
/**
*
*/
public AbstractGraphDatabaseTest() {
super();
}
protected final <V,E> AtlasGraph<V, E> getGraph() {
if(graph == null) {
protected final <V, E> AtlasGraph<V, E> getGraph() {
if (graph == null) {
graph = new Titan0Graph();
}
return (AtlasGraph<V,E>)graph;
return (AtlasGraph<V, E>)graph;
}
protected Titan0Graph getTitan0Graph() {
......@@ -166,23 +135,23 @@ public abstract class AbstractGraphDatabaseTest {
}
protected List<AtlasVertex> newVertices_ = new ArrayList<>();
protected List<AtlasVertex> newVertices = new ArrayList<>();
protected final <V, E> AtlasVertex<V, E> createVertex(AtlasGraph<V, E> graph) {
AtlasVertex<V,E> vertex = graph.addVertex();
newVertices_.add(vertex);
protected final <V, E> AtlasVertex<V, E> createVertex(AtlasGraph<V, E> theGraph) {
AtlasVertex<V, E> vertex = theGraph.addVertex();
newVertices.add(vertex);
return vertex;
}
@AfterMethod
public void removeVertices() {
for(AtlasVertex vertex : newVertices_) {
if(vertex.exists()) {
for(AtlasVertex vertex : newVertices) {
if (vertex.exists()) {
getGraph().removeVertex(vertex);
}
}
getGraph().commit();
newVertices_.clear();
newVertices.clear();
}
protected void runSynchronouslyInNewThread(final Runnable r) throws Throwable {
......@@ -191,10 +160,32 @@ public abstract class AbstractGraphDatabaseTest {
th.start();
th.join();
Throwable ex = wrapper.getExceptionThrown();
if(ex != null) {
if (ex != null) {
throw ex;
}
}
private static final class RunnableWrapper implements Runnable {
private final Runnable r;
private Throwable exceptionThrown = null;
private RunnableWrapper(Runnable r) {
this.r = r;
}
@Override
public void run() {
try {
r.run();
} catch(Throwable e) {
exceptionThrown = e;
}
}
public Throwable getExceptionThrown() {
return exceptionThrown;
}
}
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/GraphQueryTest.java
......@@ -41,35 +41,35 @@ import com.google.common.collect.Collections2;
/**
* Tests for Titan0GraphQuery
* Tests for Titan0GraphQuery.
*/
@Test
public class GraphQueryTest extends AbstractGraphDatabaseTest {
@Test
public <V,E> void testQueryThatCannotRunInMemory() throws AtlasException {
AtlasGraph<V,E> graph = getGraph();
AtlasVertex<V,E> v1 = createVertex(graph);
public <V, E> void testQueryThatCannotRunInMemory() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<V,E> v2 = createVertex(graph);
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<V,E> v3 = createVertex(graph);
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
graph.commit();
AtlasVertex<V,E> v4 = createVertex(graph);
AtlasVertex<V, E> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
AtlasGraphQuery q = graph.query();
q.has("name", ComparisionOperator.NOT_EQUAL, "George");
q.has("size15","15");
q.has("size15", "15");
graph.commit();
pause(); //pause to let the index get updated
......@@ -81,49 +81,49 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
public void testCombinationOfAndsAndOrs() throws AtlasException {
Titan0Graph graph = getTitan0Graph();
AtlasVertex<Titan0Vertex,Titan0Edge> v1 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
v1.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v2 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v2 = createVertex(graph);
v2.setProperty("name", "George");
v2.setProperty("size15", "16");
v2.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v3 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v3 = createVertex(graph);
v3.setProperty("name", "Jane");
v3.setProperty("size15", "17");
v3.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v4 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v4 = createVertex(graph);
v4.setProperty("name", "Bob");
v4.setProperty("size15", "18");
v4.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v5 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v5 = createVertex(graph);
v5.setProperty("name", "Julia");
v5.setProperty("size15", "19");
v5.setProperty("typeName", "Manager");
AtlasGraphQuery q = getGraphQuery();
q.has("typeName","Person");
q.has("typeName", "Person");
//initially match
AtlasGraphQuery inner1a = q.createChildQuery();
AtlasGraphQuery inner1b = q.createChildQuery();
inner1a.has("name","Fred");
inner1b.has("name","Jane");
inner1a.has("name", "Fred");
inner1b.has("name", "Jane");
q.or(toList(inner1a, inner1b));
AtlasGraphQuery inner2a = q.createChildQuery();
AtlasGraphQuery inner2b = q.createChildQuery();
AtlasGraphQuery inner2c = q.createChildQuery();
inner2a.has("size15","18");
inner2b.has("size15","15");
inner2a.has("size15", "18");
inner2b.has("size15", "15");
inner2c.has("size15", "16");
q.or(toList(inner2a, inner2b, inner2c));
......@@ -137,36 +137,36 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
public void testWithinStep() throws AtlasException {
Titan0Graph graph = getTitan0Graph();
AtlasVertex<Titan0Vertex,Titan0Edge> v1 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
v1.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v2 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v2 = createVertex(graph);
v2.setProperty("name", "George");
v2.setProperty("size15", "16");
v2.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v3 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v3 = createVertex(graph);
v3.setProperty("name", "Jane");
v3.setProperty("size15", "17");
v3.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v4 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v4 = createVertex(graph);
v4.setProperty("name", "Bob");
v4.setProperty("size15", "18");
v4.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v5 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v5 = createVertex(graph);
v5.setProperty("name", "Julia");
v5.setProperty("size15", "19");
v5.setProperty("typeName", "Manager");
AtlasGraphQuery q = getGraphQuery();
q.has("typeName","Person");
q.has("typeName", "Person");
//initially match
q.in("name", toList("Fred", "Jane"));
q.in("size15", toList("18", "15", "16"));
......@@ -181,43 +181,44 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
public void testWithinStepWhereGraphIsStale() throws AtlasException {
Titan0Graph graph = getTitan0Graph();
AtlasVertex<Titan0Vertex,Titan0Edge> v1 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
v1.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v2 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v2 = createVertex(graph);
v2.setProperty("name", "George");
v2.setProperty("size15", "16");
v2.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v3 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v3 = createVertex(graph);
v3.setProperty("name", "Jane");
v3.setProperty("size15", "17");
v3.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v4 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v4 = createVertex(graph);
v4.setProperty("name", "Bob");
v4.setProperty("size15", "18");
v4.setProperty("typeName", "Person");
AtlasVertex<Titan0Vertex,Titan0Edge> v5 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v5 = createVertex(graph);
v5.setProperty("name", "Julia");
v5.setProperty("size15", "19");
v5.setProperty("typeName", "Manager");
AtlasGraphQuery q = getGraphQuery();
q.has("typeName","Person");
q.has("typeName", "Person");
//initially match
q.in("name", toList("Fred", "Jane"));
graph.commit();
pause(); //let the index update
assertQueryMatches(q, v1, v3);
v3.setProperty("name", "Janet"); //make v3 no longer match the query. Within step should filter out the vertex since it no longer matches.
//make v3 no longer match the query. Within step should filter out the vertex since it no longer matches.
v3.setProperty("name", "Janet");
assertQueryMatches(q, v1);
}
......@@ -226,24 +227,24 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
Titan0Graph graph = getTitan0Graph();
AtlasVertex<Titan0Vertex,Titan0Edge> v1 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<Titan0Vertex,Titan0Edge> v2 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<Titan0Vertex,Titan0Edge> v3 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v3 = createVertex(graph);
v3.setProperty("size15", "15");
graph.commit();
AtlasVertex<Titan0Vertex,Titan0Edge> v4 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
AtlasVertex<Titan0Vertex,Titan0Edge> v5 = createVertex(graph);
AtlasVertex<Titan0Vertex, Titan0Edge> v5 = createVertex(graph);
v5.setProperty("name", "George");
v5.setProperty("size15", "16");
......@@ -261,29 +262,29 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
@Test
public <V,E> void testQueryMatchesAddedVertices() throws AtlasException {
AtlasGraph<V,E> graph = getGraph();
public <V, E> void testQueryMatchesAddedVertices() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V,E> v1 = createVertex(graph);
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<V,E> v2 = createVertex(graph);
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<V,E> v3 = createVertex(graph);
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
graph.commit();
AtlasVertex<V,E> v4 = createVertex(graph);
AtlasVertex<V, E> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
AtlasGraphQuery q = getGraphQuery();
q.has("name", "Fred");
q.has("size15","15");
q.has("size15", "15");
assertQueryMatches(q, v1, v4);
graph.commit();
......@@ -293,21 +294,21 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
@Test
public <V,E> void testQueryDoesNotMatchRemovedVertices() throws AtlasException {
AtlasGraph<V,E> graph = getGraph();
public <V, E> void testQueryDoesNotMatchRemovedVertices() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V,E> v1 = createVertex(graph);
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<V,E> v2 = createVertex(graph);
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<V,E> v3 = createVertex(graph);
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
AtlasVertex<V,E> v4 = createVertex(graph);
AtlasVertex<V, E> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
......@@ -317,7 +318,7 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
AtlasGraphQuery q = getGraphQuery();
q.has("name", "Fred");
q.has("size15","15");
q.has("size15", "15");
assertQueryMatches(q, v4);
graph.commit();
......@@ -326,28 +327,28 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
}
@Test
public <V,E> void testQueryDoesNotMatchUncommittedAddedAndRemovedVertices() throws AtlasException {
AtlasGraph<V,E> graph = getGraph();
public <V, E> void testQueryDoesNotMatchUncommittedAddedAndRemovedVertices() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V,E> v1 = createVertex(graph);
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
AtlasVertex<V,E> v2 = createVertex(graph);
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
AtlasVertex<V,E> v3 = createVertex(graph);
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
AtlasVertex<V,E> v4 = createVertex(graph);
AtlasVertex<V, E> v4 = createVertex(graph);
v4.setProperty("name", "Fred");
v4.setProperty("size15", "15");
AtlasGraphQuery q = getGraphQuery();
q.has("name", "Fred");
q.has("size15","15");
q.has("size15", "15");
assertQueryMatches(q, v1, v4);
......@@ -362,20 +363,20 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
@Test
public <V,E> void testQueryResultsReflectPropertyAdd() throws AtlasException {
AtlasGraph<V,E> graph = getGraph();
public <V, E> void testQueryResultsReflectPropertyAdd() throws AtlasException {
AtlasGraph<V, E> graph = getGraph();
AtlasVertex<V,E> v1 = createVertex(graph);
AtlasVertex<V, E> v1 = createVertex(graph);
v1.setProperty("name", "Fred");
v1.setProperty("size15", "15");
v1.addProperty(TRAIT_NAMES, "trait1");
v1.addProperty(TRAIT_NAMES, "trait2");
AtlasVertex<V,E> v2 = createVertex(graph);
AtlasVertex<V, E> v2 = createVertex(graph);
v2.setProperty("name", "Fred");
v2.addProperty(TRAIT_NAMES, "trait1");
AtlasVertex<V,E> v3 = createVertex(graph);
AtlasVertex<V, E> v3 = createVertex(graph);
v3.setProperty("size15", "15");
v3.addProperty(TRAIT_NAMES, "trait2");
......@@ -409,22 +410,25 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
}
private <V,E >void assertQueryMatches(AtlasGraphQuery expr, AtlasVertex... expectedResults) throws AtlasException {
private <V, E> void assertQueryMatches(AtlasGraphQuery expr, AtlasVertex... expectedResults) throws AtlasException {
//getGraph().commit();
Collection<AtlasVertex<Titan0Vertex, Titan0Edge>> temp = toList(expr.vertices());
//filter out vertices from previous test executions
Collection<AtlasVertex<Titan0Vertex, Titan0Edge>> result = Collections2.filter(temp, new Predicate<AtlasVertex<Titan0Vertex, Titan0Edge>>() {
Collection<AtlasVertex<Titan0Vertex, Titan0Edge>> result =
Collections2.filter(temp, new Predicate<AtlasVertex<Titan0Vertex, Titan0Edge>>() {
@Override
public boolean apply(AtlasVertex<Titan0Vertex, Titan0Edge> input) {
return newVertices_.contains(input);
return newVertices.contains(input);
}
});
assertEquals("Expected/found result sizes differ. Expected: " + Arrays.asList(expectedResults).toString() +", found: " + result, expectedResults.length, result.size());
String errorMessage = "Expected/found result sizes differ. Expected: "
+ Arrays.asList(expectedResults).toString() +", found: " + result;
assertEquals(errorMessage, expectedResults.length, result.size());
for(AtlasVertex<V,E> v : expectedResults) {
for(AtlasVertex<V, E> v : expectedResults) {
assertTrue(result.contains(v));
}
}
......@@ -440,8 +444,8 @@ public class GraphQueryTest extends AbstractGraphDatabaseTest {
private void pause() {
try {
Thread.sleep(5000);
} catch(InterruptedException e) {
//ignore
}
catch(InterruptedException e)
{}
}
}
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java
......@@ -30,10 +30,12 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
......@@ -43,7 +45,6 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Test;
......@@ -57,16 +58,16 @@ public class Titan0DatabaseTest {
private <V, E> AtlasGraph<V, E> getGraph() {
if (atlasGraph == null) {
Titan0Database db = new Titan0Database();
Titan0GraphDatabase db = new Titan0GraphDatabase();
atlasGraph = db.getGraph();
AtlasGraphManagement mgmt = atlasGraph.getManagementSystem();
// create the index (which defines these properties as being mult
// many)
for (String propertyName : AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS) {
for (String propertyName : new String[]{"__superTypeNames", "__traitNames"}) {
AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = mgmt.makePropertyKey(propertyName, String.class, Multiplicity.SET);
mgmt.createCompositeIndex(propertyName, propertyKey, false);
propertyKey = mgmt.makePropertyKey(propertyName, String.class, AtlasCardinality.SET);
mgmt.createExactMatchIndex(propertyName, false, Collections.singletonList(propertyKey));
}
}
mgmt.commit();
......
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseValidationTest.java
......@@ -37,7 +37,7 @@ public class Titan0DatabaseValidationTest {
// First get Instance
graph = new Titan0Graph();
configuration = ApplicationProperties.getSubsetConfiguration(ApplicationProperties.get(),
Titan0Database.GRAPH_PREFIX);
Titan0GraphDatabase.GRAPH_PREFIX);
}
@AfterClass
......@@ -58,15 +58,15 @@ public class Titan0DatabaseValidationTest {
@Test
public void testValidate() throws AtlasException {
try {
Titan0Database.validateIndexBackend(configuration);
Titan0GraphDatabase.validateIndexBackend(configuration);
} catch (Exception e) {
Assert.fail("Unexpected exception ", e);
}
// Change backend
configuration.setProperty(Titan0Database.INDEX_BACKEND_CONF, Titan0Database.INDEX_BACKEND_LUCENE);
configuration.setProperty(Titan0GraphDatabase.INDEX_BACKEND_CONF, Titan0GraphDatabase.INDEX_BACKEND_LUCENE);
try {
Titan0Database.validateIndexBackend(configuration);
Titan0GraphDatabase.validateIndexBackend(configuration);
Assert.fail("Expected exception");
} catch (Exception e) {
Assert.assertEquals(e.getMessage(),
......
graphdb/titan0/src/test/resources/atlas-application.properties
......@@ -17,7 +17,7 @@
#
######### Graph Database to Use #########
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan0.Titan0Database
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase
######### Atlas Server Configs #########
atlas.rest.address=http://localhost:31000
......
pom.xml 100755 → 100644
......@@ -455,6 +455,7 @@
<titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.index.backend>elasticsearch</titan.index.backend>
<entity.repository.impl>org.apache.atlas.repository.audit.InMemoryEntityAuditRepository</entity.repository.impl>
<graphdb.backend.impl>org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase</graphdb.backend.impl>
<atlas.surefire.options></atlas.surefire.options>
</properties>
......@@ -900,6 +901,57 @@
</dependency>
<!-- Graph DB -->
<dependency>
<groupId>com.tinkerpop.blueprints</groupId>
<artifactId>blueprints-core</artifactId>
<version>${tinkerpop.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId>
<version>${titan.version}</version>
<exclusions>
<!-- rexster does not work with servlet-api -->
<exclusion>
<groupId>com.tinkerpop.rexster</groupId>
<artifactId>rexster-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.tinkerpop.rexster</groupId>
<artifactId>rexster-server</artifactId>
</exclusion>
<!-- asm 4.0 does not work with jersey asm 3.1 -->
<exclusion>
<groupId>com.tinkerpop</groupId>
<artifactId>frames</artifactId>
</exclusion>
<exclusion>
<groupId>com.esotericsoftware.reflectasm</groupId>
<artifactId>reflectasm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion> <!-- GPL license imported from ganglia -->
<groupId>org.acplt</groupId>
<artifactId>oncrpc</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-hbase</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
......@@ -954,6 +1006,7 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.vividsolutions</groupId>
<artifactId>jts</artifactId>
......@@ -1027,12 +1080,6 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-server-api</artifactId>
<version>${project.version}</version>
</dependency>
......@@ -1083,6 +1130,17 @@
<type>war</type>
</dependency>
<!-- use titan 0.5.4 by default -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<version>${project.version}</version>
<type>pom</type>
<!-- exclusions should be added here for all of the non-titan0
implementations -->
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>hive-bridge</artifactId>
......@@ -1653,9 +1711,12 @@
<excludeSubProjects>true</excludeSubProjects>
<excludes>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>**/javax.script.ScriptEngineFactory</exclude>
<exclude>.reviewboardrc</exclude>
<exclude>3party-licenses/**</exclude>
<exclude>**/.cache</exclude>
<exclude>**/.cache-main</exclude>
<exclude>**/.cache-tests</exclude>
<exclude>**/.checkstyle</exclude>
<exclude>*.txt</exclude>
<exclude>**/*.json</exclude>
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-694 Update Atlas code to use graph abstraction layer (jnhagelb via sumasai)
ATLAS-1215 Atlas UI not working in firefox due to fix in ATLAS-1199 (kevalbhatt)
ATLAS-1171 Structured, high-level public APIs - Fix JAXB issues with PList, SearchFilter (mneethiraj via sumasai)
ATLAS-1206 Atlas UI not working with IE or Chrome on Windows OS in Kerberos mode (nixonrodrigues via sumasai)
......@@ -50,7 +51,7 @@ ATLAS-1104 Get outgoing edges by label doesn't work in some cases (shwethags)
ATLAS-1106 Fix Build failure due to wrong version in graphdb/common pom (sumasai)
ATLAS-1105 Disable HiveLiteralRewriterTest since its not used currently (sumasai)
ATLAS-1103 : UI: Search type list is not refreshed (Kalyanikashikar via sumasai)
ATLAS-693 Titan 0.5.4 implementation of the graph db abstraction {jnhagelb via dkantor)
ATLAS-693 Titan 0.5.4 implementation of the graph db abstraction (jnhagelb via dkantor)
ATLAS-1099 UI : multiple tag assign button hides wrongly (Kalyanikashikar via sumasai)
ATLAS-1087 Provide an option to turn off persisting entity definition in audits (sumasai, shwethags)
ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags)
......
repository/pom.xml
......@@ -52,11 +52,15 @@
<artifactId>atlas-graphdb-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
......
repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
......@@ -17,11 +17,10 @@
package org.apache.atlas;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import org.aopalliance.intercept.MethodInterceptor;
import org.aopalliance.intercept.MethodInvocation;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.SchemaNotFoundException;
import org.slf4j.Logger;
......@@ -29,30 +28,27 @@ import org.slf4j.LoggerFactory;
public class GraphTransactionInterceptor implements MethodInterceptor {
private static final Logger LOG = LoggerFactory.getLogger(GraphTransactionInterceptor.class);
private TitanGraph titanGraph;
@Inject
GraphProvider<TitanGraph> graphProvider;
private AtlasGraph graph;
@Override
public Object invoke(MethodInvocation invocation) throws Throwable {
if (titanGraph == null) {
titanGraph = graphProvider.get();
if (graph == null) {
graph = AtlasGraphProvider.getGraphInstance();
}
try {
Object response = invocation.proceed();
titanGraph.commit();
graph.commit();
LOG.info("graph commit");
return response;
} catch (Throwable t) {
titanGraph.rollback();
if (logException(t)) {
LOG.error("graph rollback due to exception ", t);
} else {
LOG.error("graph rollback due to exception " + t.getClass().getSimpleName() + ":" + t.getMessage());
}
graph.rollback();
throw t;
}
}
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -18,16 +18,9 @@
package org.apache.atlas;
import com.google.inject.Binder;
import com.google.inject.Singleton;
import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.DataSetLineageService;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.LineageService;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.listener.EntityChangeListener;
......@@ -35,13 +28,9 @@ import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.audit.EntityAuditListener;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.SoftDeleteHandler;
import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.service.Service;
......@@ -51,10 +40,15 @@ import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeSystemProvider;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.configuration.Configuration;
import com.google.inject.Binder;
import com.google.inject.Singleton;
import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
/**
* Guice module for Repository module.
*/
......@@ -62,9 +56,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
@Override
protected void configure() {
// special wiring for Titan Graph
ThrowingProviderBinder.create(binder()).bind(GraphProvider.class, TitanGraph.class).to(TitanGraphProvider.class)
.asEagerSingleton();
// allow for dynamic binding of the metadata repo & graph service
// bind the MetadataRepositoryService interface to an implementation
......@@ -95,9 +86,9 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
Configuration configuration = getConfiguration();
bindAuditRepository(binder(), configuration);
bind(DeleteHandler.class).to(getDeleteHandlerImpl(configuration)).asEagerSingleton();
bind(DeleteHandler.class).to((Class<? extends DeleteHandler>) AtlasRepositoryConfiguration.getDeleteHandlerImpl()).asEagerSingleton();
bind(TypeCache.class).to(getTypeCache(configuration)).asEagerSingleton();
bind(TypeCache.class).to((Class<? extends TypeCache>) AtlasRepositoryConfiguration.getTypeCache()).asEagerSingleton();
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
......@@ -119,7 +110,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
protected void bindAuditRepository(Binder binder, Configuration configuration) {
Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(getConfiguration());
Class<? extends EntityAuditRepository> auditRepoImpl = AtlasRepositoryConfiguration.getAuditRepositoryImpl();
//Map EntityAuditRepository interface to configured implementation
binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton();
......@@ -132,40 +123,4 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
}
}
private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
private Class<? extends EntityAuditRepository> getAuditRepositoryImpl(Configuration configuration) {
try {
return ApplicationProperties.getClass(configuration,
AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
private Class<? extends DeleteHandler> getDeleteHandlerImpl(Configuration configuration) {
try {
return ApplicationProperties.getClass(configuration,
DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
protected Class<? extends TypeCache> getTypeCache(Configuration configuration) {
// Get the type cache implementation class from Atlas configuration.
try {
return ApplicationProperties.getClass(configuration, TYPE_CACHE_IMPLEMENTATION_PROPERTY,
DefaultTypeCache.class.getName(), TypeCache.class);
} catch (AtlasException e) {
throw new RuntimeException("Error getting TypeCache implementation class", e);
}
}
}
repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
......@@ -18,7 +18,9 @@
package org.apache.atlas.discovery;
import com.thinkaurelius.titan.core.TitanGraph;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -31,7 +33,8 @@ import org.apache.atlas.query.InputLineageClosureQuery;
import org.apache.atlas.query.OutputLineageClosureQuery;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.SchemaNotFoundException;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
......@@ -39,13 +42,11 @@ import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Option;
import scala.Some;
import scala.collection.immutable.List;
import javax.inject.Inject;
import javax.inject.Singleton;
/**
* Hive implementation of Lineage service interface.
*/
......@@ -80,14 +81,14 @@ public class DataSetLineageService implements LineageService {
}
private final TitanGraph titanGraph;
private final AtlasGraph graph;
private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
private final GraphBackedDiscoveryService discoveryService;
@Inject
DataSetLineageService(GraphProvider<TitanGraph> graphProvider, MetadataRepository metadataRepository,
DataSetLineageService(MetadataRepository metadataRepository,
GraphBackedDiscoveryService discoveryService) throws DiscoveryException {
this.titanGraph = graphProvider.get();
this.graph = AtlasGraphProvider.getGraphInstance();
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
this.discoveryService = discoveryService;
}
......@@ -136,7 +137,7 @@ public class DataSetLineageService implements LineageService {
inputsQuery = new InputLineageClosureQuery(AtlasClient.DATA_SET_SUPER_TYPE, SELECT_INSTANCE_GUID,
guid, HIVE_PROCESS_TYPE_NAME,
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, titanGraph);
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
return inputsQuery.graph().toInstanceJson();
}
......@@ -153,7 +154,7 @@ public class DataSetLineageService implements LineageService {
OutputLineageClosureQuery outputsQuery =
new OutputLineageClosureQuery(AtlasClient.DATA_SET_SUPER_TYPE, SELECT_INSTANCE_GUID, guid, HIVE_PROCESS_TYPE_NAME,
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, titanGraph);
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
return outputsQuery.graph().toInstanceJson();
}
......
repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
......@@ -18,9 +18,10 @@
package org.apache.atlas.discovery.graph;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanVertex;
import java.util.List;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.query.Expressions;
import org.apache.atlas.query.GraphPersistenceStrategies;
......@@ -29,8 +30,12 @@ import org.apache.atlas.query.IntSequence;
import org.apache.atlas.query.TypeUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -44,8 +49,8 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.util.List;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
/**
* Default implementation of GraphPersistenceStrategy.
......@@ -95,8 +100,8 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
}
@Override
public List<String> traitNames(TitanVertex vertex) {
return GraphHelper.getTraitNames(vertex);
public List<String> traitNames(AtlasVertex AtlasVertex) {
return GraphHelper.getTraitNames(AtlasVertex);
}
@Override
......@@ -105,7 +110,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
}
@Override
public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
public Id getIdFromVertex(String dataTypeName, AtlasVertex vertex) {
return GraphHelper.getIdFromVertex(dataTypeName, vertex);
}
......@@ -133,16 +138,16 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
break;
case STRUCT:
TitanVertex structVertex = (TitanVertex) value;
AtlasVertex structVertex = (AtlasVertex) value;
StructType structType = (StructType) dataType;
ITypedStruct structInstance = structType.createInstance();
TypeSystem.IdType idType = TypeSystem.getInstance().getIdType();
if (dataType.getName().equals(idType.getName())) {
structInstance.set(idType.typeNameAttrName(), GraphHelper.getProperty(structVertex, typeAttributeName()));
structInstance.set(idType.idAttrName(), GraphHelper.getProperty(structVertex, idAttributeName()));
String stateValue = GraphHelper.getProperty(structVertex, stateAttributeName());
structInstance.set(idType.typeNameAttrName(), GraphHelper.getSingleValuedProperty(structVertex, typeAttributeName(), String.class));
structInstance.set(idType.idAttrName(), GraphHelper.getSingleValuedProperty(structVertex, idAttributeName(), String.class));
String stateValue = GraphHelper.getSingleValuedProperty(structVertex, stateAttributeName(), String.class);
if (stateValue != null) {
structInstance.set(idType.stateAttrName(), stateValue);
}
......@@ -153,7 +158,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
return dataType.convert(structInstance, Multiplicity.OPTIONAL);
case TRAIT:
TitanVertex traitVertex = (TitanVertex) value;
AtlasVertex traitVertex = (AtlasVertex) value;
TraitType traitType = (TraitType) dataType;
ITypedStruct traitInstance = traitType.createInstance();
// todo - this is not right, we should load the Instance associated with this
......@@ -165,9 +170,9 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
break;
case CLASS:
TitanVertex classVertex = (TitanVertex) value;
AtlasVertex classVertex = (AtlasVertex) value;
ITypedReferenceableInstance classInstance = metadataRepository.getGraphToInstanceMapper()
.mapGraphToTypedInstance(classVertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
.mapGraphToTypedInstance(GraphHelper.getSingleValuedProperty(classVertex, Constants.GUID_PROPERTY_KEY, String.class),
classVertex);
return dataType.convert(classInstance, Multiplicity.OPTIONAL);
......@@ -212,6 +217,11 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
}
@Override
public String gremlinPrimitiveOp(Expressions.ComparisonExpression op) {
return GraphPersistenceStrategies$class.gremlinPrimitiveOp(this, op);
}
@Override
public String loopObjectExpression(IDataType<?> dataType) {
return GraphPersistenceStrategies$class.loopObjectExpression(this, dataType);
}
......@@ -251,4 +261,29 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
return GraphPersistenceStrategies$class.addGraphVertexPrefix(this, preStatements);
}
@Override
public GremlinVersion getSupportedGremlinVersion() {
return GraphPersistenceStrategies$class.getSupportedGremlinVersion(this);
}
@Override
public String generatePersisentToLogicalConversionExpression(String expr, IDataType<?> t) {
return GraphPersistenceStrategies$class.generatePersisentToLogicalConversionExpression(this,expr, t);
}
@Override
public String initialQueryCondition() {
return GraphPersistenceStrategies$class.initialQueryCondition(this);
}
@Override
public boolean isPropertyValueConversionNeeded(IDataType<?> t) {
return GraphPersistenceStrategies$class.isPropertyValueConversionNeeded(this, t);
}
@Override
public AtlasGraph getGraph() throws RepositoryException {
return metadataRepository.getGraph();
}
}
repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
......@@ -18,13 +18,17 @@
package org.apache.atlas.discovery.graph;
import com.thinkaurelius.titan.core.TitanEdge;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Vertex;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.script.ScriptException;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.discovery.DiscoveryException;
......@@ -39,29 +43,21 @@ import org.apache.atlas.query.QueryParser;
import org.apache.atlas.query.QueryProcessor;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.util.Either;
import scala.util.parsing.combinator.Parsers;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Graph backed implementation of Search.
*/
......@@ -70,19 +66,19 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class);
private final TitanGraph titanGraph;
private final AtlasGraph graph;
private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
public final static String SCORE = "score";
@Inject
GraphBackedDiscoveryService(GraphProvider<TitanGraph> graphProvider, MetadataRepository metadataRepository)
GraphBackedDiscoveryService(MetadataRepository metadataRepository)
throws DiscoveryException {
this.titanGraph = graphProvider.get();
this.graph = AtlasGraphProvider.getGraphInstance();
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
}
//Refer http://s3.thinkaurelius.com/docs/titan/0.5.4/index-backends.html for indexed query
//For titan 0.5.4, refer to http://s3.thinkaurelius.com/docs/titan/0.5.4/index-backends.html for indexed query
//http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query
// .html#query-string-syntax for query syntax
@Override
......@@ -90,8 +86,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
public String searchByFullText(String query, QueryParams queryParams) throws DiscoveryException {
String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, query);
LOG.debug("Full text query: {}", graphQuery);
Iterator<TitanIndexQuery.Result<Vertex>> results =
titanGraph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery).vertices().iterator();
Iterator<AtlasIndexQuery.Result<?, ?>> results =graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery).vertices();
JSONArray response = new JSONArray();
int index = 0;
......@@ -101,8 +96,9 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
}
while (results.hasNext() && response.length() < queryParams.limit()) {
TitanIndexQuery.Result<Vertex> result = results.next();
Vertex vertex = result.getElement();
AtlasIndexQuery.Result<?,?> result = results.next();
AtlasVertex<?,?> vertex = result.getVertex();
JSONObject row = new JSONObject();
String guid = GraphHelper.getIdFromVertex(vertex);
......@@ -157,7 +153,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
LOG.debug("Query = {}", validatedExpression);
LOG.debug("Expression Tree = {}", validatedExpression.treeString());
LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr());
return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, titanGraph).evaluate();
return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, graph).evaluate();
}
/**
......@@ -173,18 +169,8 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
@GraphTransaction
public List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException {
LOG.debug("Executing gremlin query={}", gremlinQuery);
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
if(engine == null) {
throw new DiscoveryException("gremlin-groovy: engine not found");
}
Bindings bindings = engine.createBindings();
bindings.put("g", titanGraph);
try {
Object o = engine.eval(gremlinQuery, bindings);
Object o = graph.executeGremlinScript(gremlinQuery, false);
return extractResult(o);
} catch (ScriptException se) {
throw new DiscoveryException(se);
......@@ -195,47 +181,44 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
List<Map<String, String>> result = new ArrayList<>();
if (o instanceof List) {
List l = (List) o;
for (Object r : l) {
for (Object value : l) {
Map<String, String> oRow = new HashMap<>();
if (r instanceof Map) {
@SuppressWarnings("unchecked") Map<Object, Object> iRow = (Map) r;
if (value instanceof Map) {
@SuppressWarnings("unchecked") Map<Object, Object> iRow = (Map) value;
for (Map.Entry e : iRow.entrySet()) {
Object k = e.getKey();
Object v = e.getValue();
oRow.put(k.toString(), v.toString());
}
} else if (r instanceof TitanVertex) {
TitanVertex vertex = (TitanVertex) r;
oRow.put("id", vertex.getId().toString());
Iterable<TitanProperty> ps = vertex.getProperties();
for (TitanProperty tP : ps) {
String pName = tP.getPropertyKey().getName();
Object pValue = vertex.getProperty(pName);
if (pValue != null) {
oRow.put(pName, pValue.toString());
} else if (value instanceof AtlasVertex) {
AtlasVertex<?,?> vertex = (AtlasVertex<?,?>)value;
for (String key : vertex.getPropertyKeys()) {
Object propertyValue = GraphHelper.getProperty(vertex, key);
if (propertyValue != null) {
oRow.put(key, propertyValue.toString());
}
}
} else if (r instanceof String) {
oRow.put("", r.toString());
} else if (r instanceof TitanEdge) {
TitanEdge edge = (TitanEdge) r;
} else if (value instanceof String) {
oRow.put("", value.toString());
} else if(value instanceof AtlasEdge) {
AtlasEdge edge = (AtlasEdge) value;
oRow.put("id", edge.getId().toString());
oRow.put("label", edge.getLabel());
oRow.put("inVertex", edge.getVertex(Direction.IN).getId().toString());
oRow.put("outVertex", edge.getVertex(Direction.OUT).getId().toString());
Set<String> propertyKeys = edge.getPropertyKeys();
for (String propertyKey : propertyKeys) {
oRow.put(propertyKey, edge.getProperty(propertyKey).toString());
oRow.put("inVertex", edge.getInVertex().getId().toString());
oRow.put("outVertex", edge.getOutVertex().getId().toString());
for (String propertyKey : edge.getPropertyKeys()) {
oRow.put(propertyKey, GraphHelper.getProperty(edge, propertyKey).toString());
}
} else {
throw new DiscoveryException(String.format("Cannot process result %s", o.toString()));
throw new DiscoveryException(String.format("Cannot process result %s", String.valueOf(value)));
}
result.add(oRow);
}
} else {
}
else {
result.add(new HashMap<String, String>() {{
put("result", o.toString());
}});
......
repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java 0 → 100755
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import com.google.common.annotations.VisibleForTesting;
/**
* Provides access to the AtlasGraph
*
*/
public class AtlasGraphProvider implements IAtlasGraphProvider {
private static volatile GraphDatabase<?,?> graphDb_;
public static <V, E> AtlasGraph<V, E> getGraphInstance() {
GraphDatabase<?,?> db = getGraphDatabase();
AtlasGraph<?, ?> graph = db.getGraph();
return (AtlasGraph<V, E>) graph;
}
private static <V, E> GraphDatabase<?,?> getGraphDatabase() {
try {
if (graphDb_ == null) {
synchronized(AtlasGraphProvider.class) {
if(graphDb_ == null) {
Class implClass = AtlasRepositoryConfiguration.getGraphDatabaseImpl();
graphDb_ = (GraphDatabase<V, E>) implClass.newInstance();
}
}
}
return graphDb_;
}
catch (IllegalAccessException e) {
throw new RuntimeException("Error initializing graph database", e);
} catch (InstantiationException e) {
throw new RuntimeException("Error initializing graph database", e);
}
}
@VisibleForTesting
public static void cleanup() {
getGraphDatabase().cleanup();
}
@Override
public AtlasGraph get() throws RepositoryException {
return getGraphInstance();
}
}
repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
......@@ -18,14 +18,23 @@
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper.VertexInfo;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.exception.NullRequiredAttributeException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeInfo;
......@@ -38,16 +47,6 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.graph.GraphHelper.string;
public abstract class DeleteHandler {
public static final Logger LOG = LoggerFactory.getLogger(DeleteHandler.class);
......@@ -71,12 +70,12 @@ public abstract class DeleteHandler {
* @param instanceVertices
* @throws AtlasException
*/
public void deleteEntities(List<Vertex> instanceVertices) throws AtlasException {
public void deleteEntities(List<AtlasVertex> instanceVertices) throws AtlasException {
RequestContext requestContext = RequestContext.get();
Set<Vertex> deletionCandidateVertices = new HashSet<>();
Set<AtlasVertex> deletionCandidateVertices = new HashSet<>();
for (Vertex instanceVertex : instanceVertices) {
for (AtlasVertex instanceVertex : instanceVertices) {
String guid = GraphHelper.getIdFromVertex(instanceVertex);
Id.EntityState state = GraphHelper.getState(instanceVertex);
if (requestContext.getDeletedEntityIds().contains(guid) || state == Id.EntityState.DELETED) {
......@@ -96,13 +95,13 @@ public abstract class DeleteHandler {
}
// Delete traits and vertices.
for (Vertex deletionCandidateVertex : deletionCandidateVertices) {
for (AtlasVertex deletionCandidateVertex : deletionCandidateVertices) {
deleteAllTraits(deletionCandidateVertex);
deleteTypeVertex(deletionCandidateVertex, false);
}
}
protected abstract void deleteEdge(Edge edge, boolean force) throws AtlasException;
protected abstract void deleteEdge(AtlasEdge edge, boolean force) throws AtlasException;
/**
* Deletes a type vertex - can be entity(class type) or just vertex(struct/trait type)
......@@ -110,7 +109,7 @@ public abstract class DeleteHandler {
* @param typeCategory
* @throws AtlasException
*/
protected void deleteTypeVertex(Vertex instanceVertex, DataTypes.TypeCategory typeCategory, boolean force) throws AtlasException {
protected void deleteTypeVertex(AtlasVertex instanceVertex, DataTypes.TypeCategory typeCategory, boolean force) throws AtlasException {
switch (typeCategory) {
case STRUCT:
case TRAIT:
......@@ -131,7 +130,7 @@ public abstract class DeleteHandler {
* @param instanceVertex
* @throws AtlasException
*/
protected void deleteTypeVertex(Vertex instanceVertex, boolean force) throws AtlasException {
protected void deleteTypeVertex(AtlasVertex instanceVertex, boolean force) throws AtlasException {
LOG.debug("Deleting {}", string(instanceVertex));
String typeName = GraphHelper.getTypeName(instanceVertex);
IDataType type = typeSystem.getDataType(IDataType.class, typeName);
......@@ -158,10 +157,10 @@ public abstract class DeleteHandler {
DataTypes.TypeCategory elementTypeCategory = elementType.getTypeCategory();
if (elementTypeCategory == DataTypes.TypeCategory.STRUCT ||
elementTypeCategory == DataTypes.TypeCategory.CLASS) {
Iterator<Edge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel);
Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel);
if (edges != null) {
while (edges.hasNext()) {
Edge edge = edges.next();
AtlasEdge edge = edges.next();
deleteEdgeReference(edge, elementType.getTypeCategory(), attributeInfo.isComposite, false);
}
}
......@@ -176,7 +175,7 @@ public abstract class DeleteHandler {
if (valueTypeCategory == DataTypes.TypeCategory.STRUCT ||
valueTypeCategory == DataTypes.TypeCategory.CLASS) {
List<String> keys = GraphHelper.getProperty(instanceVertex, propertyName);
List<String> keys = GraphHelper.getListProperty(instanceVertex, propertyName);
if (keys != null) {
for (String key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key);
......@@ -199,7 +198,7 @@ public abstract class DeleteHandler {
* @return returns true if the edge reference is hard deleted
* @throws AtlasException
*/
public boolean deleteEdgeReference(Edge edge, DataTypes.TypeCategory typeCategory, boolean isComposite,
public boolean deleteEdgeReference(AtlasEdge edge, DataTypes.TypeCategory typeCategory, boolean isComposite,
boolean forceDeleteStructTrait) throws AtlasException {
LOG.debug("Deleting {}", string(edge));
boolean forceDelete =
......@@ -210,14 +209,14 @@ public abstract class DeleteHandler {
//If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities.
//If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled
//through this delete, hence delete the edge and the reference vertex.
Vertex vertexForDelete = edge.getVertex(Direction.IN);
AtlasVertex vertexForDelete = edge.getInVertex();
//If deleting the edge and then the in vertex, reverse attribute shouldn't be updated
deleteEdge(edge, false, forceDelete);
deleteTypeVertex(vertexForDelete, typeCategory, forceDelete);
} else {
//If the vertex is of type class, and its not a composite attributes, the reference vertex' lifecycle is not controlled
//through this delete. Hence just remove the reference edge. Leave the reference vertex as is
//If the vertex is of type class, and its not a composite attributes, the reference AtlasVertex' lifecycle is not controlled
//through this delete. Hence just remove the reference edge. Leave the reference AtlasVertex as is
//If deleting just the edge, reverse attribute should be updated for any references
//For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated
......@@ -226,20 +225,20 @@ public abstract class DeleteHandler {
return !softDelete || forceDelete;
}
public void deleteEdgeReference(Vertex outVertex, String edgeLabel, DataTypes.TypeCategory typeCategory,
public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, DataTypes.TypeCategory typeCategory,
boolean isComposite) throws AtlasException {
Edge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (edge != null) {
deleteEdgeReference(edge, typeCategory, isComposite, false);
}
}
protected void deleteEdge(Edge edge, boolean updateReverseAttribute, boolean force) throws AtlasException {
protected void deleteEdge(AtlasEdge edge, boolean updateReverseAttribute, boolean force) throws AtlasException {
//update reverse attribute
if (updateReverseAttribute) {
AttributeInfo attributeInfo = getAttributeForEdge(edge.getLabel());
if (attributeInfo.reverseAttributeName != null) {
deleteEdgeBetweenVertices(edge.getVertex(Direction.IN), edge.getVertex(Direction.OUT),
deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(),
attributeInfo.reverseAttributeName);
}
}
......@@ -247,25 +246,25 @@ public abstract class DeleteHandler {
deleteEdge(edge, force);
}
protected void deleteVertex(Vertex instanceVertex, boolean force) throws AtlasException {
protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasException {
//Update external references(incoming edges) to this vertex
LOG.debug("Setting the external references to {} to null(removing edges)", string(instanceVertex));
Iterator<Edge> edges = instanceVertex.getEdges(Direction.IN).iterator();
Iterator<AtlasEdge> edges = instanceVertex.getEdges(AtlasEdgeDirection.IN).iterator();
while(edges.hasNext()) {
Edge edge = edges.next();
AtlasEdge edge = edges.next();
Id.EntityState edgeState = GraphHelper.getState(edge);
if (edgeState == Id.EntityState.ACTIVE) {
//Delete only the active edge references
AttributeInfo attribute = getAttributeForEdge(edge.getLabel());
//TODO use delete edge instead??
deleteEdgeBetweenVertices(edge.getVertex(Direction.OUT), edge.getVertex(Direction.IN), attribute.name);
deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute.name);
}
}
_deleteVertex(instanceVertex, force);
}
protected abstract void _deleteVertex(Vertex instanceVertex, boolean force);
protected abstract void _deleteVertex(AtlasVertex instanceVertex, boolean force);
/**
* Deletes the edge between outvertex and inVertex. The edge is for attribute attributeName of outVertex
......@@ -274,7 +273,7 @@ public abstract class DeleteHandler {
* @param attributeName
* @throws AtlasException
*/
protected void deleteEdgeBetweenVertices(Vertex outVertex, Vertex inVertex, String attributeName) throws AtlasException {
protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVertex, String attributeName) throws AtlasException {
LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex),
attributeName);
String typeName = GraphHelper.getTypeName(outVertex);
......@@ -289,7 +288,7 @@ public abstract class DeleteHandler {
AttributeInfo attributeInfo = getFieldMapping(type).fields.get(attributeName);
String propertyName = GraphHelper.getQualifiedFieldName(type, attributeName);
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
Edge edge = null;
AtlasEdge edge = null;
switch (attributeInfo.dataType().getTypeCategory()) {
case CLASS:
......@@ -308,17 +307,17 @@ public abstract class DeleteHandler {
case ARRAY:
//If its array attribute, find the right edge between the two vertices and update array property
List<String> elements = GraphHelper.getProperty(outVertex, propertyName);
List<String> elements = GraphHelper.getListProperty(outVertex, propertyName);
if (elements != null) {
elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty()
for (String elementEdgeId : elements) {
Edge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
if (elementEdge == null) {
continue;
}
Vertex elementVertex = elementEdge.getVertex(Direction.IN);
if (elementVertex.getId().toString().equals(inVertex.getId().toString())) {
AtlasVertex elementVertex = elementEdge.getInVertex();
if (elementVertex.equals(inVertex)) {
edge = elementEdge;
//TODO element.size includes deleted items as well. should exclude
......@@ -349,15 +348,15 @@ public abstract class DeleteHandler {
case MAP:
//If its map attribute, find the right edge between two vertices and update map property
List<String> keys = GraphHelper.getProperty(outVertex, propertyName);
List<String> keys = GraphHelper.getListProperty(outVertex, propertyName);
if (keys != null) {
keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty()
for (String key : keys) {
String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
String mapEdgeId = GraphHelper.getProperty(outVertex, keyPropertyName);
Edge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class);
AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
if(mapEdge != null) {
Vertex mapVertex = mapEdge.getVertex(Direction.IN);
AtlasVertex mapVertex = mapEdge.getInVertex();
if (mapVertex.getId().toString().equals(inVertex.getId().toString())) {
//TODO keys.size includes deleted items as well. should exclude
if (attributeInfo.multiplicity.nullAllowed() || keys.size() > attributeInfo.multiplicity.lower) {
......@@ -427,7 +426,7 @@ public abstract class DeleteHandler {
* @param instanceVertex
* @throws AtlasException
*/
private void deleteAllTraits(Vertex instanceVertex) throws AtlasException {
private void deleteAllTraits(AtlasVertex instanceVertex) throws AtlasException {
List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
LOG.debug("Deleting traits {} for {}", traitNames, string(instanceVertex));
String typeName = GraphHelper.getTypeName(instanceVertex);
......
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
......@@ -17,7 +17,7 @@
*/
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -49,7 +49,7 @@ public class FullTextMapper {
instanceCache = new HashMap<>();
}
public String mapRecursive(Vertex instanceVertex, boolean followReferences) throws AtlasException {
public String mapRecursive(AtlasVertex instanceVertex, boolean followReferences) throws AtlasException {
String guid = GraphHelper.getIdFromVertex(instanceVertex);
ITypedReferenceableInstance typedReference;
if (instanceCache.containsKey(guid)) {
......@@ -121,7 +121,7 @@ public class FullTextMapper {
case CLASS:
if (followReferences) {
String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
Vertex refVertex = graphHelper.getVertexForGUID(refGuid);
AtlasVertex refVertex = graphHelper.getVertexForGUID(refGuid);
return mapRecursive(refVertex, false);
}
break;
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -18,13 +18,12 @@
package org.apache.atlas.repository.graph;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
......@@ -32,6 +31,10 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.exception.EntityExistsException;
......@@ -46,10 +49,9 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
/**
* An implementation backed by a Graph database provided
......@@ -64,16 +66,16 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private final TitanGraph titanGraph;
private final AtlasGraph graph;
private DeleteHandler deleteHandler;
private GraphToTypedInstanceMapper graphToInstanceMapper;
@Inject
public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider, DeleteHandler deleteHandler) {
this.titanGraph = graphProvider.get();
graphToInstanceMapper = new GraphToTypedInstanceMapper(titanGraph);
public GraphBackedMetadataRepository(DeleteHandler deleteHandler) {
this.graph = AtlasGraphProvider.getGraphInstance();
graphToInstanceMapper = new GraphToTypedInstanceMapper(graph);
this.deleteHandler = deleteHandler;
}
......@@ -148,7 +150,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException {
LOG.debug("Retrieving entity with guid={}", guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
try {
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
......@@ -164,7 +166,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.debug("Retrieving entity with type={} and {}={}", entityType, attribute, value);
IDataType type = typeSystem.getDataType(IDataType.class, entityType);
String propertyKey = getFieldNameInVertex(type, attribute);
Vertex instanceVertex = graphHelper.findVertex(propertyKey, value,
AtlasVertex instanceVertex = graphHelper.findVertex(propertyKey, value,
Constants.ENTITY_TYPE_PROPERTY_KEY, entityType,
Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
......@@ -176,15 +178,15 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@GraphTransaction
public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.debug("Retrieving entity list for type={}", entityType);
GraphQuery query = titanGraph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<Vertex> results = query.vertices().iterator();
AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<AtlasVertex> results = query.vertices().iterator();
if (!results.hasNext()) {
return Collections.emptyList();
}
ArrayList<String> entityList = new ArrayList<>();
while (results.hasNext()) {
Vertex vertex = results.next();
AtlasVertex vertex = results.next();
entityList.add(GraphHelper.getIdFromVertex(vertex));
}
......@@ -202,7 +204,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@GraphTransaction
public List<String> getTraitNames(String guid) throws AtlasException {
LOG.debug("Retrieving trait names for entity={}", guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
return GraphHelper.getTraitNames(instanceVertex);
}
......@@ -222,7 +224,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.debug("Adding a new trait={} for entity={}", traitName, guid);
try {
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
// add the trait instance as a new vertex
final String typeName = GraphHelper.getTypeName(instanceVertex);
......@@ -256,7 +258,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public void deleteTrait(String guid, String traitNameToBeDeleted) throws TraitNotFoundException, EntityNotFoundException, RepositoryException {
LOG.debug("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
if (!traitNames.contains(traitNameToBeDeleted)) {
......@@ -267,7 +269,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
try {
final String entityTypeName = GraphHelper.getTypeName(instanceVertex);
String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted);
Edge edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
AtlasEdge edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
if(edge != null) {
deleteHandler.deleteEdgeReference(edge, DataTypes.TypeCategory.TRAIT, false, true);
......@@ -281,7 +283,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
private void updateTraits(Vertex instanceVertex, List<String> traitNames) {
private void updateTraits(AtlasVertex instanceVertex, List<String> traitNames) {
// remove the key
instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
......@@ -332,14 +334,14 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throw new IllegalArgumentException("guids must be non-null and non-empty");
}
List<Vertex> vertices = new ArrayList<>(guids.size());
List<AtlasVertex> vertices = new ArrayList<>(guids.size());
for (String guid : guids) {
if (guid == null) {
LOG.warn("deleteEntities: Ignoring null guid");
continue;
}
try {
Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
vertices.add(instanceVertex);
} catch (EntityNotFoundException e) {
// Entity does not exist - treat as non-error, since the caller
......@@ -360,4 +362,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return new AtlasClient.EntityResult(requestContext.getCreatedEntityIds(),
requestContext.getUpdatedEntityIds(), requestContext.getDeletedEntityIds());
}
public AtlasGraph getGraph() {
return AtlasGraphProvider.getGraphInstance();
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -18,14 +18,16 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.discovery.SearchIndexer;
......@@ -35,6 +37,11 @@ import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.IndexCreationException;
import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
......@@ -46,13 +53,8 @@ import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
/**
* Adds index for properties of a given type when its added before any instances are added.
......@@ -61,32 +63,45 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
private final TitanGraph titanGraph;
List<Class> MIXED_INDEX_EXCLUSIONS = new ArrayList() {{
private static final List<Class> VERTEX_INDEX_EXCLUSIONS = new ArrayList() {
{
add(Boolean.class);
add(BigDecimal.class);
add(BigInteger.class);
}};
}
};
//allows injection of a dummy graph for testing
private IAtlasGraphProvider provider;
@Inject
public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider) throws AtlasException {
this(graphProvider, ApplicationProperties.get());
public GraphBackedSearchIndexer() throws RepositoryException, AtlasException {
this(new AtlasGraphProvider(), ApplicationProperties.get());
}
GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider, Configuration configuration)
@VisibleForTesting
GraphBackedSearchIndexer( IAtlasGraphProvider provider, Configuration configuration)
throws IndexException, RepositoryException {
this.titanGraph = graphProvider.get();
this.provider = provider;
if (!HAConfiguration.isHAEnabled(configuration)) {
initialize();
initialize(provider.get());
}
}
/**
* Initializes the indices for the graph - create indices for Global Vertex Keys
* Initializes the indices for the graph - create indices for Global AtlasVertex Keys
*/
private void initialize() throws RepositoryException, IndexException {
TitanManagement management = titanGraph.getManagementSystem();
initialize(provider.get());
}
/**
* Initializes the indices for the graph - create indices for Global AtlasVertex Keys
*/
private void initialize(AtlasGraph graph) throws RepositoryException, IndexException {
AtlasGraphManagement management = graph.getManagementSystem();
try {
if (management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)) {
LOG.info("Global indexes already exist for graph");
......@@ -95,37 +110,39 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
/* This is called only once, which is the first time Atlas types are made indexable .*/
LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
management.buildIndex(Constants.VERTEX_INDEX, Vertex.class).buildMixedIndex(Constants.BACKING_INDEX);
management.buildIndex(Constants.EDGE_INDEX, Edge.class).buildMixedIndex(Constants.BACKING_INDEX);
LOG.info("Indexes do not exist, Creating indexes for graph.");
management.createVertexIndex(Constants.VERTEX_INDEX, Constants.BACKING_INDEX, Collections.<AtlasPropertyKey>emptyList());
management.createEdgeIndex(Constants.EDGE_INDEX, Constants.BACKING_INDEX);
// create a composite index for guid as its unique
createIndexes(management, Constants.GUID_PROPERTY_KEY, String.class, true,
Cardinality.SINGLE, true, true);
AtlasCardinality.SINGLE, true, true);
// create a composite index for entity state
createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true, true);
// create a composite index for entity creation timestamp
createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, AtlasCardinality.SINGLE, true, true);
// create a mixed index for entity state. Set systemProperty flag deliberately to false
// so that it doesnt create a composite index which has issues with
// titan 0.5.4 - Refer https://groups.google.com/forum/#!searchin/aureliusgraphs/hemanth/aureliusgraphs/bx7T843mzXU/fjAsclx7GAAJ
createStateMixedIndex(management);
createIndexes(management, Constants.STATE_PROPERTY_KEY, String.class, false, AtlasCardinality.SINGLE, false, false);
// create a composite index for entity state
// create a composite index for entity modification timestamp
createIndexes(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false,
Cardinality.SINGLE, false, false);
AtlasCardinality.SINGLE, false, false);
// create a composite and mixed index for type since it can be combined with other keys
createIndexes(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE,
createIndexes(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, AtlasCardinality.SINGLE,
true, true);
// create a composite and mixed index for type since it can be combined with other keys
createIndexes(management, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET,
createIndexes(management, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET,
true, true);
// create a composite and mixed index for traitNames since it can be combined with other
// keys. Traits must be a set and not a list.
createIndexes(management, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET,
createIndexes(management, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET,
true, true);
// Index for full text search
......@@ -134,6 +151,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
//Indexes for graph backed type system store
createTypeStoreIndexes(management);
commit(management);
LOG.info("Index creation for global keys complete.");
} catch (Throwable t) {
......@@ -142,33 +160,22 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private void createStateMixedIndex(TitanManagement management) {
PropertyKey propertyKey = management.getPropertyKey(Constants.STATE_PROPERTY_KEY);
if (propertyKey == null) {
propertyKey = management.makePropertyKey(Constants.STATE_PROPERTY_KEY).dataType(String.class).cardinality(Cardinality.SINGLE)
.make();
}
enhanceMixedIndex(management, Constants.STATE_PROPERTY_KEY, String.class, Cardinality.SINGLE, propertyKey);
}
private void createFullTextIndex(AtlasGraphManagement management) {
AtlasPropertyKey fullText =
management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY, String.class, AtlasCardinality.SINGLE);
private void createFullTextIndex(TitanManagement management) {
PropertyKey fullText =
management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();
management.createFullTextIndex(Constants.FULLTEXT_INDEX, fullText, Constants.BACKING_INDEX);
management.buildIndex(Constants.FULLTEXT_INDEX, Vertex.class)
.addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
.buildMixedIndex(Constants.BACKING_INDEX);
LOG.info("Created mixed index for {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
}
private void createTypeStoreIndexes(TitanManagement management) {
private void createTypeStoreIndexes(AtlasGraphManagement management) {
//Create unique index on typeName
createIndexes(management, Constants.TYPENAME_PROPERTY_KEY, String.class, true, Cardinality.SINGLE,
createIndexes(management, Constants.TYPENAME_PROPERTY_KEY, String.class, true, AtlasCardinality.SINGLE,
true, true);
//create index on vertex type
createIndexes(management, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE,
createIndexes(management, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, AtlasCardinality.SINGLE,
true, true);
}
......@@ -180,7 +187,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
*/
@Override
public void onAdd(Collection<? extends IDataType> dataTypes) throws AtlasException {
TitanManagement management = titanGraph.getManagementSystem();
AtlasGraphManagement management = provider.get().getManagementSystem();
for (IDataType dataType : dataTypes) {
LOG.debug("Creating indexes for type name={}, definition={}", dataType.getName(), dataType.getClass());
try {
......@@ -203,7 +211,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
onAdd(dataTypes);
}
private void addIndexForType(TitanManagement management, IDataType dataType) {
private void addIndexForType(AtlasGraphManagement management, IDataType dataType) {
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
case ENUM:
......@@ -233,17 +241,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private void createIndexForFields(TitanManagement management, IDataType dataType, Map<String, AttributeInfo> fields) {
private void createIndexForFields(AtlasGraphManagement management, IDataType dataType, Map<String, AttributeInfo> fields) {
for (AttributeInfo field : fields.values()) {
createIndexForAttribute(management, dataType.getName(), field);
}
}
private void createIndexForAttribute(TitanManagement management, String typeName, AttributeInfo field) {
private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AttributeInfo field) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + field.name);
switch (field.dataType().getTypeCategory()) {
case PRIMITIVE:
Cardinality cardinality = getCardinality(field.multiplicity);
AtlasCardinality cardinality = getCardinality(field.multiplicity);
createIndexes(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique,
cardinality, false, field.isIndexable);
break;
......@@ -310,109 +318,117 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
private Cardinality getCardinality(Multiplicity multiplicity) {
private AtlasCardinality getCardinality(Multiplicity multiplicity) {
if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
return Cardinality.SINGLE;
return AtlasCardinality.SINGLE;
} else if (multiplicity == Multiplicity.COLLECTION) {
return Cardinality.LIST;
return AtlasCardinality.LIST;
} else if (multiplicity == Multiplicity.SET) {
return Cardinality.SET;
return AtlasCardinality.SET;
}
// todo - default to LIST as this is the most forgiving
return Cardinality.LIST;
return AtlasCardinality.LIST;
}
private PropertyKey createIndexes(TitanManagement management, String propertyName,
Class propertyClass, boolean isUnique, Cardinality cardinality,
boolean createCompositeForAttribute, boolean createCompositeWithTypeandSuperTypes) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
private AtlasPropertyKey createIndexes(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality, boolean createCompositeForAttribute,
boolean createCompositeWithTypeandSuperTypes) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName).dataType(propertyClass).cardinality(cardinality)
.make();
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
enhanceMixedIndex(management, propertyName, propertyClass, cardinality, propertyKey);
updateVertexIndex(management, propertyName, propertyClass, cardinality, propertyKey);
}
if (createCompositeForAttribute) {
createCompositeIndex(management, propertyName, propertyClass, propertyKey, isUnique);
createExactMatchIndex(management, propertyClass, propertyKey, isUnique);
} else if (createCompositeWithTypeandSuperTypes) {
//Index with typename since typename+property key queries need to speed up
createCompositeIndexWithTypeName(management, propertyName, propertyClass, propertyKey);
createCompositeIndexWithSuperTypeName(management, propertyName, propertyClass, propertyKey);
// Index with typename since typename+property key queries need to
// speed up
createExactMatchIndexWithTypeName(management, propertyClass, propertyKey);
createExactMatchIndexWithSuperTypeName(management, propertyClass, propertyKey);
}
return propertyKey;
}
private void createCompositeIndex(TitanManagement management, String propertyName, Class propertyClass,
PropertyKey propertyKey, boolean enforceUniqueness) {
LOG.debug("Creating composite index for property {} of type {} ", propertyName,
propertyClass.getName());
private void createExactMatchIndex(AtlasGraphManagement management, Class propertyClass,
AtlasPropertyKey propertyKey, boolean enforceUniqueness) {
TitanGraphIndex existingIndex = management.getGraphIndex(propertyName);
if ( existingIndex == null) {
TitanManagement.IndexBuilder indexBuilder =
management.buildIndex(propertyName, Vertex.class).addKey(propertyKey);
String propertyName = propertyKey.getName();
LOG.debug("Creating composite index for property {} of type {} ", propertyName, propertyClass.getName());
AtlasGraphIndex existingIndex = management.getGraphIndex(propertyName);
if (existingIndex == null) {
if (enforceUniqueness) {
LOG.debug("Enabling unique index for property {} of type {} ", propertyName,
propertyClass.getName());
indexBuilder.unique();
LOG.debug("Enabling unique index for property {} of type {} ", propertyName, propertyClass.getName());
}
indexBuilder.buildCompositeIndex();
management.createExactMatchIndex(propertyName, enforceUniqueness, Collections.singletonList(propertyKey));
}
LOG.info("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
}
private void createCompositeIndexWithTypeName(TitanManagement management, String propertyName, Class propertyClass,
PropertyKey propertyKey) {
createCompositeIndexWithSystemProperty(management, propertyName, propertyClass, propertyKey, Constants.ENTITY_TYPE_PROPERTY_KEY, Cardinality.SINGLE);
private void createExactMatchIndexWithTypeName(AtlasGraphManagement management,
Class propertyClass, AtlasPropertyKey propertyKey) {
createExactMatchIndexWithSystemProperty(management, propertyClass, propertyKey,
Constants.ENTITY_TYPE_PROPERTY_KEY, AtlasCardinality.SINGLE);
}
private void createCompositeIndexWithSuperTypeName(TitanManagement management, String propertyName, Class propertyClass,
PropertyKey propertyKey) {
createCompositeIndexWithSystemProperty(management, propertyName, propertyClass, propertyKey, Constants.SUPER_TYPES_PROPERTY_KEY, Cardinality.SET);
private void createExactMatchIndexWithSuperTypeName(AtlasGraphManagement management,
Class propertyClass, AtlasPropertyKey propertyKey) {
createExactMatchIndexWithSystemProperty(management, propertyClass, propertyKey,
Constants.SUPER_TYPES_PROPERTY_KEY, AtlasCardinality.SET);
}
private void createCompositeIndexWithSystemProperty(TitanManagement management, String propertyName, Class propertyClass,
PropertyKey propertyKey, final String systemPropertyKey, Cardinality cardinality) {
LOG.debug("Creating composite index for property {} of type {} and {}", propertyName,
propertyClass.getName(), systemPropertyKey);
PropertyKey typePropertyKey = management.getPropertyKey(systemPropertyKey);
private void createExactMatchIndexWithSystemProperty(AtlasGraphManagement management,
Class propertyClass, AtlasPropertyKey propertyKey, final String systemPropertyKey,
AtlasCardinality cardinality) {
LOG.debug("Creating composite index for property {} of type {} and {}", propertyKey.getName(), propertyClass.getName(),
systemPropertyKey);
AtlasPropertyKey typePropertyKey = management.getPropertyKey(systemPropertyKey);
if (typePropertyKey == null) {
typePropertyKey = management.makePropertyKey(systemPropertyKey).
dataType(String.class).cardinality(cardinality)
.make();
typePropertyKey = management.makePropertyKey(systemPropertyKey, String.class, cardinality);
}
final String indexName = propertyName + systemPropertyKey;
TitanGraphIndex existingIndex = management.getGraphIndex(indexName);
if ( existingIndex == null) {
TitanManagement.IndexBuilder indexBuilder =
management.buildIndex(indexName, Vertex.class).
addKey(propertyKey).addKey(typePropertyKey);
indexBuilder.buildCompositeIndex();
LOG.info("Created composite index for property {} of type {} and {}", propertyName, propertyClass.getName(), systemPropertyKey);
final String indexName = propertyKey.getName() + systemPropertyKey;
AtlasGraphIndex existingIndex = management.getGraphIndex(indexName);
if (existingIndex == null) {
List<AtlasPropertyKey> keys = new ArrayList<AtlasPropertyKey>(2);
keys.add(propertyKey);
keys.add(typePropertyKey);
management.createExactMatchIndex(indexName, false, keys);
LOG.info("Created composite index for property {} of type {} and {}", propertyKey.getName(), propertyClass.getName(),
systemPropertyKey);
}
}
private void enhanceMixedIndex(TitanManagement management, String propertyName, Class propertyClass,
Cardinality cardinality, PropertyKey propertyKey) {
if (checkIfMixedIndexApplicable(propertyClass, cardinality)) {
//Use backing index
private void updateVertexIndex(AtlasGraphManagement management, String propertyName, Class propertyClass,
AtlasCardinality cardinality, AtlasPropertyKey propertyKey) {
if (checkIfVertexIndexApplicable(propertyClass, cardinality)) {
// Use backing index
management.addVertexIndexKey(Constants.VERTEX_INDEX, propertyKey);
LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey);
LOG.info("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
}
}
private boolean checkIfMixedIndexApplicable(Class propertyClass, Cardinality cardinality) {
return !(MIXED_INDEX_EXCLUSIONS.contains(propertyClass) || cardinality == Cardinality.LIST || cardinality ==
Cardinality.SET);
private boolean checkIfVertexIndexApplicable(Class propertyClass, AtlasCardinality cardinality) {
return !(VERTEX_INDEX_EXCLUSIONS.contains(propertyClass) || cardinality.isMany());
}
public void commit(TitanManagement management) throws IndexException {
private void commit(AtlasGraphManagement management) throws IndexException {
try {
management.commit();
} catch (Exception e) {
......@@ -421,7 +437,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
public void rollback(TitanManagement management) throws IndexException {
private void rollback(AtlasGraphManagement management) throws IndexException {
try {
management.rollback();
} catch (Exception e) {
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -18,49 +18,55 @@
package org.apache.atlas.repository.graph;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.UUID;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.lang.StringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.UUID;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
/**
* Utility class for graph operations.
......@@ -77,13 +83,13 @@ public final class GraphHelper {
private static volatile GraphHelper INSTANCE;
private TitanGraph titanGraph;
private AtlasGraph graph;
private static int maxRetries;
public static long retrySleepTimeMillis;
@VisibleForTesting
GraphHelper(TitanGraph titanGraph) {
this.titanGraph = titanGraph;
GraphHelper(AtlasGraph graph) {
this.graph = graph;
try {
maxRetries = ApplicationProperties.get().getInt(RETRY_COUNT, 3);
retrySleepTimeMillis = ApplicationProperties.get().getLong(RETRY_DELAY, 1000);
......@@ -96,7 +102,7 @@ public final class GraphHelper {
if ( INSTANCE == null) {
synchronized (GraphHelper.class) {
if (INSTANCE == null) {
INSTANCE = new GraphHelper(TitanGraphProvider.getGraphInstance());
INSTANCE = new GraphHelper(AtlasGraphProvider.getGraphInstance());
}
}
}
......@@ -104,7 +110,7 @@ public final class GraphHelper {
}
@VisibleForTesting
static GraphHelper getInstance(TitanGraph graph) {
static GraphHelper getInstance(AtlasGraph graph) {
if ( INSTANCE == null) {
synchronized (GraphHelper.class) {
if (INSTANCE == null) {
......@@ -116,10 +122,10 @@ public final class GraphHelper {
}
public Vertex createVertexWithIdentity(ITypedReferenceableInstance typedInstance, Set<String> superTypeNames) {
public AtlasVertex createVertexWithIdentity(ITypedReferenceableInstance typedInstance, Set<String> superTypeNames) {
final String guid = UUID.randomUUID().toString();
final Vertex vertexWithIdentity = createVertexWithoutIdentity(typedInstance.getTypeName(),
final AtlasVertex vertexWithIdentity = createVertexWithoutIdentity(typedInstance.getTypeName(),
new Id(guid, 0, typedInstance.getTypeName()), superTypeNames);
// add identity
......@@ -131,10 +137,10 @@ public final class GraphHelper {
return vertexWithIdentity;
}
public Vertex createVertexWithoutIdentity(String typeName, Id typedInstanceId, Set<String> superTypeNames) {
LOG.debug("Creating vertex for type {} id {}", typeName,
public AtlasVertex createVertexWithoutIdentity(String typeName, Id typedInstanceId, Set<String> superTypeNames) {
LOG.debug("Creating AtlasVertex for type {} id {}", typeName,
typedInstanceId != null ? typedInstanceId._getId() : null);
final Vertex vertexWithoutIdentity = titanGraph.addVertex(null);
final AtlasVertex vertexWithoutIdentity = graph.addVertex();
// add type information
setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
......@@ -156,9 +162,9 @@ public final class GraphHelper {
return vertexWithoutIdentity;
}
private Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
private AtlasEdge addEdge(AtlasVertex fromVertex, AtlasVertex toVertex, String edgeLabel) {
LOG.debug("Adding edge for {} -> label {} -> {}", string(fromVertex), edgeLabel, string(toVertex));
Edge edge = titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
AtlasEdge edge = graph.addEdge(fromVertex, toVertex, edgeLabel);
setProperty(edge, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
setProperty(edge, Constants.TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
......@@ -168,15 +174,15 @@ public final class GraphHelper {
return edge;
}
public Edge getOrCreateEdge(Vertex outVertex, Vertex inVertex, String edgeLabel) throws RepositoryException {
public AtlasEdge getOrCreateEdge(AtlasVertex outVertex, AtlasVertex inVertex, String edgeLabel) throws RepositoryException {
for (int numRetries = 0; numRetries < maxRetries; numRetries++) {
try {
LOG.debug("Running edge creation attempt {}", numRetries);
Iterator<Edge> edges = getAdjacentEdgesByLabel(inVertex, Direction.IN, edgeLabel);
Iterator<AtlasEdge> edges = getAdjacentEdgesByLabel(inVertex, AtlasEdgeDirection.IN, edgeLabel);
while (edges.hasNext()) {
Edge edge = edges.next();
if (edge.getVertex(Direction.OUT).getId().toString().equals(outVertex.getId().toString())) {
AtlasEdge edge = edges.next();
if (edge.getOutVertex().equals(outVertex)) {
Id.EntityState edgeState = getState(edge);
if (edgeState == null || edgeState == Id.EntityState.ACTIVE) {
return edge;
......@@ -206,16 +212,16 @@ public final class GraphHelper {
}
public Edge getEdgeByEdgeId(Vertex outVertex, String edgeLabel, String edgeId) {
public AtlasEdge getEdgeByEdgeId(AtlasVertex outVertex, String edgeLabel, String edgeId) {
if (edgeId == null) {
return null;
}
return titanGraph.getEdge(edgeId);
return graph.getEdge(edgeId);
//TODO get edge id is expensive. Use this logic. But doesn't work for now
/**
Iterable<Edge> edges = outVertex.getEdges(Direction.OUT, edgeLabel);
for (Edge edge : edges) {
Iterable<AtlasEdge> edges = outVertex.getEdges(Direction.OUT, edgeLabel);
for (AtlasEdge edge : edges) {
if (edge.getId().toString().equals(edgeId)) {
return edge;
}
......@@ -226,24 +232,24 @@ public final class GraphHelper {
/**
* Args of the format prop1, key1, prop2, key2...
* Searches for a vertex with prop1=key1 && prop2=key2
* Searches for a AtlasVertex with prop1=key1 && prop2=key2
* @param args
* @return vertex with the given property keys
* @return AtlasVertex with the given property keys
* @throws EntityNotFoundException
*/
public Vertex findVertex(Object... args) throws EntityNotFoundException {
public AtlasVertex findVertex(Object... args) throws EntityNotFoundException {
StringBuilder condition = new StringBuilder();
GraphQuery query = titanGraph.query();
AtlasGraphQuery query = graph.query();
for (int i = 0 ; i < args.length; i+=2) {
query = query.has((String) args[i], args[i+1]);
condition.append(args[i]).append(" = ").append(args[i+1]).append(", ");
}
String conditionStr = condition.toString();
LOG.debug("Finding vertex with {}", conditionStr);
LOG.debug("Finding AtlasVertex with {}", conditionStr);
Iterator<Vertex> results = query.vertices().iterator();
Iterator<AtlasVertex> results = query.vertices().iterator();
// returning one since entityType, qualifiedName should be unique
Vertex vertex = results.hasNext() ? results.next() : null;
AtlasVertex vertex = results.hasNext() ? results.next() : null;
if (vertex == null) {
LOG.debug("Could not find a vertex with {}", condition.toString());
......@@ -257,17 +263,17 @@ public final class GraphHelper {
//In some cases of parallel APIs, the edge is added, but get edge by label doesn't return the edge. ATLAS-1104
//So traversing all the edges
public Iterator<Edge> getAdjacentEdgesByLabel(Vertex instanceVertex, Direction direction, final String edgeLabel) {
public Iterator<AtlasEdge> getAdjacentEdgesByLabel(AtlasVertex instanceVertex, AtlasEdgeDirection direction, final String edgeLabel) {
LOG.debug("Finding edges for {} with label {}", string(instanceVertex), edgeLabel);
if(instanceVertex != null && edgeLabel != null) {
final Iterator<Edge> iterator = instanceVertex.getEdges(direction).iterator();
return new Iterator<Edge>() {
private Edge edge = null;
final Iterator<AtlasEdge> iterator = instanceVertex.getEdges(direction).iterator();
return new Iterator<AtlasEdge>() {
private AtlasEdge edge = null;
@Override
public boolean hasNext() {
while (edge == null && iterator.hasNext()) {
Edge localEdge = iterator.next();
AtlasEdge localEdge = iterator.next();
if (localEdge.getLabel().equals(edgeLabel)) {
edge = localEdge;
}
......@@ -276,9 +282,9 @@ public final class GraphHelper {
}
@Override
public Edge next() {
public AtlasEdge next() {
if (hasNext()) {
Edge localEdge = edge;
AtlasEdge localEdge = edge;
edge = null;
return localEdge;
}
......@@ -294,8 +300,8 @@ public final class GraphHelper {
return null;
}
public Iterator<Edge> getOutGoingEdgesByLabel(Vertex instanceVertex, String edgeLabel) {
return getAdjacentEdgesByLabel(instanceVertex, Direction.OUT, edgeLabel);
public Iterator<AtlasEdge> getOutGoingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel);
}
/**
......@@ -305,19 +311,19 @@ public final class GraphHelper {
* @param edgeLabel
* @return
*/
public Edge getEdgeForLabel(Vertex vertex, String edgeLabel) {
Iterator<Edge> iterator = getAdjacentEdgesByLabel(vertex, Direction.OUT, edgeLabel);
Edge latestDeletedEdge = null;
public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel) {
Iterator<AtlasEdge> iterator = getAdjacentEdgesByLabel(vertex, AtlasEdgeDirection.OUT, edgeLabel);
AtlasEdge latestDeletedEdge = null;
long latestDeletedEdgeTime = Long.MIN_VALUE;
while (iterator != null && iterator.hasNext()) {
Edge edge = iterator.next();
AtlasEdge edge = iterator.next();
Id.EntityState edgeState = getState(edge);
if (edgeState == null || edgeState == Id.EntityState.ACTIVE) {
LOG.debug("Found {}", string(edge));
return edge;
} else {
Long modificationTime = getProperty(edge, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Long modificationTime = edge.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
if (modificationTime != null && modificationTime >= latestDeletedEdgeTime) {
latestDeletedEdgeTime = modificationTime;
latestDeletedEdge = edge;
......@@ -329,25 +335,26 @@ public final class GraphHelper {
return latestDeletedEdge;
}
public static String vertexString(final Vertex vertex) {
public static String vertexString(final AtlasVertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
properties.append(propertyKey).append("=").append(vertex.getProperty(propertyKey).toString()).append(", ");
Collection<?> propertyValues = vertex.getPropertyValues(propertyKey, Object.class);
properties.append(propertyKey).append("=").append(propertyValues.toString()).append(", ");
}
return "v[" + vertex.getId() + "], Properties[" + properties + "]";
return "v[" + vertex.getIdForDisplay() + "], Properties[" + properties + "]";
}
public static String edgeString(final Edge edge) {
return "e[" + edge.getLabel() + "], [" + edge.getVertex(Direction.OUT) + " -> " + edge.getLabel() + " -> "
+ edge.getVertex(Direction.IN) + "]";
public static String edgeString(final AtlasEdge edge) {
return "e[" + edge.getLabel() + "], [" + edge.getOutVertex() + " -> " + edge.getLabel() + " -> "
+ edge.getInVertex() + "]";
}
public static <T extends Element> void setProperty(T element, String propertyName, Object value) {
public static <T extends AtlasElement> void setProperty(T element, String propertyName, Object value) {
String elementStr = string(element);
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
LOG.debug("Setting property {} = \"{}\" to {}", actualPropertyName, value, elementStr);
Object existValue = element.getProperty(actualPropertyName);
Object existValue = element.getProperty(actualPropertyName, Object.class);
if(value == null || (value instanceof Collection && ((Collection) value).isEmpty())) {
if(existValue != null) {
LOG.info("Removing property - {} value from {}", actualPropertyName, elementStr);
......@@ -361,32 +368,66 @@ public final class GraphHelper {
}
}
public static <T extends Element, O> O getProperty(T element, String propertyName) {
/**
* Gets the value of a property that is stored in the graph as a single property value. If
* a multi-property such as {@link Constants#TRAIT_NAMES_PROPERTY_KEY} or {@link Constants#SUPER_TYPES_PROPERTY_KEY}
* is used, an exception will be thrown.
*
* @param element
* @param propertyName
* @param clazz
* @return
*/
public static <T> T getSingleValuedProperty(AtlasElement element, String propertyName, Class<T> clazz) {
String elementStr = string(element);
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
LOG.debug("Reading property {} from {}", actualPropertyName, elementStr);
return element.getProperty(actualPropertyName);
return (T)element.getProperty(actualPropertyName, clazz);
}
public static Iterable<TitanProperty> getProperties(TitanVertex vertex, String propertyName) {
public static Object getProperty(AtlasVertex<?,?> vertex, String propertyName) {
String elementStr = string(vertex);
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
LOG.debug("Reading property {} from {}", actualPropertyName, elementStr);
return vertex.getProperties(actualPropertyName);
if(AtlasGraphProvider.getGraphInstance().isMultiProperty(actualPropertyName)) {
return vertex.getPropertyValues(actualPropertyName, String.class);
}
else {
return vertex.getProperty(actualPropertyName, Object.class);
}
}
private static <T extends Element> String string(T element) {
if (element instanceof Vertex) {
return string((Vertex) element);
} else if (element instanceof Edge) {
return string((Edge)element);
public static Object getProperty(AtlasEdge<?,?> edge, String propertyName) {
String elementStr = string(edge);
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
LOG.debug("Reading property {} from {}", actualPropertyName, elementStr);
return edge.getProperty(actualPropertyName, Object.class);
}
private static <T extends AtlasElement> String string(T element) {
if (element instanceof AtlasVertex) {
return string((AtlasVertex) element);
} else if (element instanceof AtlasEdge) {
return string((AtlasEdge)element);
}
return element.toString();
}
public static void addProperty(Vertex vertex, String propertyName, Object value) {
LOG.debug("Adding property {} = \"{}\" to vertex {}", propertyName, value, string(vertex));
((TitanVertex)vertex).addProperty(propertyName, value);
/**
* Adds an additional value to a multi-property.
*
* @param vertex
* @param propertyName
* @param value
*/
public static void addProperty(AtlasVertex vertex, String propertyName, Object value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
LOG.debug("Adding property {} = \"{}\" to vertex {}", actualPropertyName, value, string(vertex));
vertex.addProperty(actualPropertyName, value);
}
/**
......@@ -394,26 +435,26 @@ public final class GraphHelper {
*
* @param edge
*/
public void removeEdge(Edge edge) {
public void removeEdge(AtlasEdge edge) {
String edgeString = string(edge);
LOG.debug("Removing {}", edgeString);
titanGraph.removeEdge(edge);
graph.removeEdge(edge);
LOG.info("Removed {}", edgeString);
}
/**
* Remove the specified vertex from the graph.
* Remove the specified AtlasVertex from the graph.
*
* @param vertex
* @param AtlasVertex
*/
public void removeVertex(Vertex vertex) {
public void removeVertex(AtlasVertex vertex) {
String vertexString = string(vertex);
LOG.debug("Removing {}", vertexString);
titanGraph.removeVertex(vertex);
graph.removeVertex(vertex);
LOG.info("Removed {}", vertexString);
}
public Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
public AtlasVertex getVertexForGUID(String guid) throws EntityNotFoundException {
return findVertex(Constants.GUID_PROPERTY_KEY, guid);
}
......@@ -436,12 +477,12 @@ public final class GraphHelper {
return attrName;
}
public static List<String> getTraitNames(Vertex entityVertex) {
public static List<String> getTraitNames(AtlasVertex<?,?> entityVertex) {
ArrayList<String> traits = new ArrayList<>();
for (TitanProperty property : ((TitanVertex) entityVertex).getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
traits.add((String) property.getValue());
Collection<String> propertyValues = entityVertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class);
for(String value : propertyValues) {
traits.add(value);
}
return traits;
}
......@@ -454,26 +495,26 @@ public final class GraphHelper {
return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedFieldName(dataType, aInfo.name);
}
public static Id getIdFromVertex(String dataTypeName, Vertex vertex) {
public static Id getIdFromVertex(String dataTypeName, AtlasVertex vertex) {
return new Id(getIdFromVertex(vertex),
vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName, getStateAsString(vertex));
vertex.getProperty(Constants.VERSION_PROPERTY_KEY, Integer.class), dataTypeName, getStateAsString(vertex));
}
public static String getIdFromVertex(Vertex vertex) {
return vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY);
public static String getIdFromVertex(AtlasVertex vertex) {
return vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY, String.class);
}
public static String getTypeName(Vertex instanceVertex) {
return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
public static String getTypeName(AtlasVertex instanceVertex) {
return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
}
public static Id.EntityState getState(Element element) {
public static Id.EntityState getState(AtlasElement element) {
String state = getStateAsString(element);
return state == null ? null : Id.EntityState.valueOf(state);
}
public static String getStateAsString(Element element) {
return element.getProperty(Constants.STATE_PROPERTY_KEY);
public static String getStateAsString(AtlasElement element) {
return element.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
}
/**
......@@ -485,10 +526,10 @@ public final class GraphHelper {
* @return
* @throws AtlasException
*/
public Vertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance)
public AtlasVertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance)
throws AtlasException {
LOG.debug("Checking if there is an instance with the same unique attributes for instance {}", instance.toShortString());
Vertex result = null;
AtlasVertex result = null;
for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
if (attributeInfo.isUnique) {
String propertyKey = getQualifiedFieldName(classType, attributeInfo.name);
......@@ -507,14 +548,14 @@ public final class GraphHelper {
}
/**
* Guid and Vertex combo
* Guid and AtlasVertex combo
*/
public static class VertexInfo {
private String guid;
private Vertex vertex;
private AtlasVertex vertex;
private String typeName;
public VertexInfo(String guid, Vertex vertex, String typeName) {
public VertexInfo(String guid, AtlasVertex vertex, String typeName) {
this.guid = guid;
this.vertex = vertex;
this.typeName = typeName;
......@@ -523,7 +564,7 @@ public final class GraphHelper {
public String getGuid() {
return guid;
}
public Vertex getVertex() {
public AtlasVertex getVertex() {
return vertex;
}
public String getTypeName() {
......@@ -560,19 +601,19 @@ public final class GraphHelper {
}
/**
* Get the GUIDs and vertices for all composite entities owned/contained by the specified root entity vertex.
* Get the GUIDs and vertices for all composite entities owned/contained by the specified root entity AtlasVertex.
* The graph is traversed from the root entity through to the leaf nodes of the containment graph.
*
* @param entityVertex the root entity vertex
* @return set of VertexInfo for all composite entities
* @throws AtlasException
*/
public Set<VertexInfo> getCompositeVertices(Vertex entityVertex) throws AtlasException {
public Set<VertexInfo> getCompositeVertices(AtlasVertex entityVertex) throws AtlasException {
Set<VertexInfo> result = new HashSet<>();
Stack<Vertex> vertices = new Stack<>();
Stack<AtlasVertex> vertices = new Stack<>();
vertices.push(entityVertex);
while (vertices.size() > 0) {
Vertex vertex = vertices.pop();
AtlasVertex vertex = vertices.pop();
String typeName = GraphHelper.getTypeName(vertex);
String guid = GraphHelper.getIdFromVertex(vertex);
Id.EntityState state = GraphHelper.getState(vertex);
......@@ -589,9 +630,9 @@ public final class GraphHelper {
String edgeLabel = GraphHelper.getEdgeLabel(classType, attributeInfo);
switch (attributeInfo.dataType().getTypeCategory()) {
case CLASS:
Edge edge = getEdgeForLabel(vertex, edgeLabel);
AtlasEdge edge = getEdgeForLabel(vertex, edgeLabel);
if (edge != null && GraphHelper.getState(edge) == Id.EntityState.ACTIVE) {
Vertex compositeVertex = edge.getVertex(Direction.IN);
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
}
break;
......@@ -601,12 +642,12 @@ public final class GraphHelper {
if (elementTypeCategory != TypeCategory.CLASS) {
continue;
}
Iterator<Edge> edges = getOutGoingEdgesByLabel(vertex, edgeLabel);
Iterator<AtlasEdge> edges = getOutGoingEdgesByLabel(vertex, edgeLabel);
if (edges != null) {
while (edges.hasNext()) {
edge = edges.next();
if (edge != null && GraphHelper.getState(edge) == Id.EntityState.ACTIVE) {
Vertex compositeVertex = edge.getVertex(Direction.IN);
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
}
}
......@@ -619,13 +660,13 @@ public final class GraphHelper {
continue;
}
String propertyName = GraphHelper.getQualifiedFieldName(classType, attributeInfo.name);
List<String> keys = vertex.getProperty(propertyName);
List<String> keys = vertex.getProperty(propertyName, List.class);
if (keys != null) {
for (String key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key);
edge = getEdgeForLabel(vertex, mapEdgeLabel);
if (edge != null && GraphHelper.getState(edge) == Id.EntityState.ACTIVE) {
Vertex compositeVertex = edge.getVertex(Direction.IN);
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
}
}
......@@ -639,15 +680,99 @@ public final class GraphHelper {
return result;
}
public static void dumpToLog(final Graph graph) {
public static ITypedReferenceableInstance[] deserializeClassInstances(TypeSystem typeSystem, String entityInstanceDefinition)
throws AtlasException {
try {
JSONArray referableInstances = new JSONArray(entityInstanceDefinition);
ITypedReferenceableInstance[] instances = new ITypedReferenceableInstance[referableInstances.length()];
for (int index = 0; index < referableInstances.length(); index++) {
Referenceable entityInstance =
InstanceSerialization.fromJsonReferenceable(referableInstances.getString(index), true);
ITypedReferenceableInstance typedInstrance = getTypedReferenceableInstance(typeSystem, entityInstance);
instances[index] = typedInstrance;
}
return instances;
} catch(ValueConversionException | TypeNotFoundException e) {
throw e;
} catch (Exception e) { // exception from deserializer
LOG.error("Unable to deserialize json={}", entityInstanceDefinition, e);
throw new IllegalArgumentException("Unable to deserialize json", e);
}
}
public static ITypedReferenceableInstance getTypedReferenceableInstance(TypeSystem typeSystem, Referenceable entityInstance)
throws AtlasException {
final String entityTypeName = ParamChecker.notEmpty(entityInstance.getTypeName(), "Entity type cannot be null");
ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName);
//Both assigned id and values are required for full update
//classtype.convert() will remove values if id is assigned. So, set temp id, convert and
// then replace with original id
Id origId = entityInstance.getId();
entityInstance.replaceWithNewId(new Id(entityInstance.getTypeName()));
ITypedReferenceableInstance typedInstrance = entityType.convert(entityInstance, Multiplicity.REQUIRED);
((ReferenceableInstance)typedInstrance).replaceWithNewId(origId);
return typedInstrance;
}
public static boolean isReference(IDataType type) {
return type.getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
type.getTypeCategory() == DataTypes.TypeCategory.CLASS;
}
public static void setArrayElementsProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
setListPropertyFromElementIds(instanceVertex, actualPropertyName, (List)values);
}
else {
setProperty(instanceVertex, actualPropertyName, values);
}
}
public static void setMapValueProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName, Object value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
instanceVertex.setPropertyFromElementId(actualPropertyName, (AtlasEdge)value);
}
else {
instanceVertex.setProperty(actualPropertyName, value);
}
}
public static Object getMapValueProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
return instanceVertex.getProperty(actualPropertyName, AtlasEdge.class);
}
else {
return instanceVertex.getProperty(actualPropertyName, String.class).toString();
}
}
public static List<Object> getArrayElementsProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
return (List)instanceVertex.getListProperty(actualPropertyName, AtlasEdge.class);
}
else {
return (List)instanceVertex.getListProperty(actualPropertyName);
}
}
public static void dumpToLog(final AtlasGraph<?,?> graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
for (Vertex vertex : graph.getVertices()) {
for (AtlasVertex vertex : graph.getVertices()) {
LOG.debug(vertexString(vertex));
}
LOG.debug("Edges of {}", graph);
for (Edge edge : graph.getEdges()) {
for (AtlasEdge edge : graph.getEdges()) {
LOG.debug(edgeString(edge));
}
LOG.debug("*******************Graph Dump****************************");
......@@ -657,41 +782,41 @@ public final class GraphHelper {
return String.format("entity[type=%s guid=%]", instance.getTypeName(), instance.getId()._getId());
}
public static String string(Vertex vertex) {
public static String string(AtlasVertex<?,?> vertex) {
if(vertex == null) {
return "vertex[null]";
} else {
if (LOG.isDebugEnabled()) {
return getVertexDetails(vertex);
} else {
return String.format("vertex[id=%s]", vertex.getId().toString());
return String.format("vertex[id=%s]", vertex.getIdForDisplay());
}
}
}
public static String getVertexDetails(Vertex vertex) {
public static String getVertexDetails(AtlasVertex<?,?> vertex) {
return String.format("vertex[id=%s type=%s guid=%s]", vertex.getId().toString(), getTypeName(vertex),
return String.format("vertex[id=%s type=%s guid=%s]", vertex.getIdForDisplay(), getTypeName(vertex),
getIdFromVertex(vertex));
}
public static String string(Edge edge) {
public static String string(AtlasEdge<?,?> edge) {
if(edge == null) {
return "edge[null]";
} else {
if (LOG.isDebugEnabled()) {
return getEdgeDetails(edge);
} else {
return String.format("edge[id=%s]", edge.getId().toString());
return String.format("edge[id=%s]", edge.getIdForDisplay());
}
}
}
public static String getEdgeDetails(Edge edge) {
public static String getEdgeDetails(AtlasEdge<?,?> edge) {
return String.format("edge[id=%s label=%s from %s -> to %s]", edge.getId().toString(), edge.getLabel(),
string(edge.getVertex(Direction.OUT)), string(edge.getVertex(Direction.IN)));
return String.format("edge[id=%s label=%s from %s -> to %s]", edge.getIdForDisplay(), edge.getLabel(),
string(edge.getOutVertex()), string(edge.getInVertex()));
}
@VisibleForTesting
......@@ -741,4 +866,34 @@ public final class GraphHelper {
return null;
}
public static boolean elementExists(AtlasElement v) {
return v != null && v.exists();
}
public static void setListPropertyFromElementIds(AtlasVertex<?, ?> instanceVertex, String propertyName,
List<AtlasElement> elements) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
instanceVertex.setPropertyFromElementsIds(actualPropertyName, elements);
}
public static void setPropertyFromElementId(AtlasVertex<?, ?> instanceVertex, String propertyName,
AtlasElement value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
instanceVertex.setPropertyFromElementId(actualPropertyName, value);
}
public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) throws AtlasException {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
instanceVertex.setListProperty(actualPropertyName, value);
}
public static List<String> getListProperty(AtlasVertex instanceVertex, String propertyName) throws AtlasException {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
return instanceVertex.getListProperty(actualPropertyName);
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphSchemaInitializer.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.setup.SetupException;
import org.apache.atlas.setup.SetupStep;
......@@ -41,7 +41,7 @@ public class GraphSchemaInitializer implements SetupStep {
LOG.info("Initializing graph schema backend.");
try {
// The implementation of this method internally creates the schema.
TitanGraphProvider.getGraphInstance();
AtlasGraphProvider.getGraphInstance();
LOG.info("Completed initializing graph schema backend.");
} catch (Exception e) {
LOG.error("Could not initialize graph schema backend due to exception, {}", e.getMessage(), e);
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
......@@ -17,13 +17,22 @@
*/
package org.apache.atlas.repository.graph;
import com.google.inject.Singleton;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
......@@ -39,17 +48,7 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.repository.graph.GraphHelper.getIdFromVertex;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import com.google.inject.Singleton;
@Singleton
public final class GraphToTypedInstanceMapper {
......@@ -58,17 +57,17 @@ public final class GraphToTypedInstanceMapper {
private static TypeSystem typeSystem = TypeSystem.getInstance();
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private TitanGraph titanGraph;
private AtlasGraph graph;
public GraphToTypedInstanceMapper(TitanGraph titanGraph) {
this.titanGraph = titanGraph;
public GraphToTypedInstanceMapper(AtlasGraph graph) {
this.graph = graph;
}
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, Vertex instanceVertex)
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, AtlasVertex instanceVertex)
throws AtlasException {
LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid);
String typeName = GraphHelper.getProperty(instanceVertex, Constants.ENTITY_TYPE_PROPERTY_KEY);
String typeName = GraphHelper.getSingleValuedProperty(instanceVertex, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
List<String> traits = GraphHelper.getTraitNames(instanceVertex);
String state = GraphHelper.getStateAsString(instanceVertex);
......@@ -86,7 +85,7 @@ public final class GraphToTypedInstanceMapper {
return typedInstance;
}
private void mapVertexToInstanceTraits(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
private void mapVertexToInstanceTraits(AtlasVertex instanceVertex, ITypedReferenceableInstance typedInstance,
List<String> traits) throws AtlasException {
for (String traitName : traits) {
LOG.debug("mapping trait {} to instance", traitName);
......@@ -95,7 +94,7 @@ public final class GraphToTypedInstanceMapper {
}
}
public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
public void mapVertexToInstance(AtlasVertex instanceVertex, ITypedInstance typedInstance,
Map<String, AttributeInfo> fields) throws AtlasException {
LOG.debug("Mapping vertex {} to instance {} for fields", instanceVertex, typedInstance.getTypeName(),
......@@ -105,7 +104,7 @@ public final class GraphToTypedInstanceMapper {
}
}
private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
private void mapVertexToAttribute(AtlasVertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws AtlasException {
LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
final IDataType dataType = attributeInfo.dataType();
......@@ -145,8 +144,9 @@ public final class GraphToTypedInstanceMapper {
break;
case CLASS:
AtlasEdge nullEdge = null;
Object idOrInstance = mapVertexToClassReference(instanceVertex, attributeInfo, relationshipLabel,
attributeInfo.dataType(), null);
attributeInfo.dataType(), nullEdge);
if (idOrInstance != null) {
typedInstance.set(attributeInfo.name, idOrInstance);
}
......@@ -157,27 +157,30 @@ public final class GraphToTypedInstanceMapper {
}
}
private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException {
private Object mapVertexToClassReference(AtlasVertex instanceVertex, AttributeInfo attributeInfo,
String relationshipLabel, IDataType dataType, AtlasEdge optionalEdge) throws AtlasException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
Edge edge;
if (edgeId == null) {
AtlasEdge edge = null;
if (optionalEdge == null) {
edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
} else {
edge = graphHelper.getEdgeByEdgeId(instanceVertex, relationshipLabel, edgeId);
edge = optionalEdge;
}
if (edge != null) {
final Vertex referenceVertex = edge.getVertex(Direction.IN);
final String guid = GraphHelper.getIdFromVertex(referenceVertex);
if (GraphHelper.elementExists(edge)) {
final AtlasVertex referenceVertex = edge.getInVertex();
final String guid = GraphHelper.getSingleValuedProperty(referenceVertex, Constants.GUID_PROPERTY_KEY, String.class);
LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid);
if (attributeInfo.isComposite) {
//Also, when you retrieve a type's instance, you get the complete object graph of the composites
LOG.debug("Found composite, mapping vertex to instance");
return mapGraphToTypedInstance(guid, referenceVertex);
} else {
Id referenceId = getIdFromVertex(dataType.getName(), referenceVertex);
String state = GraphHelper.getStateAsString(referenceVertex);
Id referenceId =
new Id(guid, GraphHelper.getSingleValuedProperty(referenceVertex, Constants.VERSION_PROPERTY_KEY, Integer.class),
dataType.getName(), state);
LOG.debug("Found non-composite, adding id {} ", referenceId);
return referenceId;
}
......@@ -187,15 +190,18 @@ public final class GraphToTypedInstanceMapper {
}
@SuppressWarnings("unchecked")
private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
private void mapVertexToArrayInstance(AtlasVertex<?,?> instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo, String propertyName) throws AtlasException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
List list = GraphHelper.getProperty(instanceVertex, propertyName);
final DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
final IDataType elementType = arrayType.getElemType();
List<Object> list = GraphHelper.getArrayElementsProperty(elementType, instanceVertex, propertyName);
if (list == null || list.size() == 0) {
return;
}
DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
final IDataType elementType = arrayType.getElemType();
String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
ArrayList values = new ArrayList();
......@@ -209,7 +215,7 @@ public final class GraphToTypedInstanceMapper {
}
}
private Object mapVertexToCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
private Object mapVertexToCollectionEntry(AtlasVertex instanceVertex, AttributeInfo attributeInfo,
IDataType elementType, Object value, String edgeLabel) throws AtlasException {
switch (elementType.getTypeCategory()) {
case PRIMITIVE:
......@@ -223,10 +229,10 @@ public final class GraphToTypedInstanceMapper {
break;
case STRUCT:
return mapVertexToStructInstance(instanceVertex, (StructType) elementType, edgeLabel, (String) value);
return mapVertexToStructInstance(instanceVertex, (StructType) elementType, edgeLabel, (AtlasEdge) value);
case CLASS:
return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType, (AtlasEdge) value);
default:
break;
......@@ -236,21 +242,21 @@ public final class GraphToTypedInstanceMapper {
}
@SuppressWarnings("unchecked")
private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
private void mapVertexToMapInstance(AtlasVertex<?,?> instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo, final String propertyName) throws AtlasException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
List<String> keys = GraphHelper.getProperty(instanceVertex, propertyName);
List<String> keys = GraphHelper.getListProperty(instanceVertex, propertyName);
if (keys == null || keys.size() == 0) {
return;
}
DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
final IDataType valueType = mapType.getValueType();
HashMap values = new HashMap();
HashMap<String,Object> values = new HashMap<>();
for (String key : keys) {
final String keyPropertyName = propertyName + "." + key;
final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + keyPropertyName;
final Object keyValue = GraphHelper.getProperty(instanceVertex, keyPropertyName);
final Object keyValue = GraphHelper.getMapValueProperty(valueType, instanceVertex, keyPropertyName);
Object mapValue = mapVertexToCollectionEntry(instanceVertex, attributeInfo, valueType, keyValue, edgeLabel);
if (mapValue != null) {
values.put(key, mapValue);
......@@ -262,21 +268,21 @@ public final class GraphToTypedInstanceMapper {
}
}
private ITypedStruct mapVertexToStructInstance(Vertex instanceVertex, StructType structType,
String relationshipLabel, String edgeId) throws AtlasException {
private ITypedStruct mapVertexToStructInstance(AtlasVertex instanceVertex, StructType structType,
String relationshipLabel, AtlasEdge optionalEdge) throws AtlasException {
LOG.debug("mapping {} to struct {}", string(instanceVertex), relationshipLabel);
ITypedStruct structInstance = null;
Edge edge;
if (edgeId == null) {
AtlasEdge edge;
if (optionalEdge == null) {
edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
} else {
edge = graphHelper.getEdgeByEdgeId(instanceVertex, relationshipLabel, edgeId);
edge = optionalEdge;
}
if (edge != null) {
if (GraphHelper.elementExists(edge)) {
structInstance = structType.createInstance();
Vertex structInstanceVertex = edge.getVertex(Direction.IN);
AtlasVertex structInstanceVertex = edge.getInVertex();
LOG.debug("Found struct instance {}, mapping to instance {} ", string(structInstanceVertex),
structInstance.getTypeName());
mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
......@@ -285,21 +291,19 @@ public final class GraphToTypedInstanceMapper {
return structInstance;
}
private void mapVertexToTraitInstance(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
private void mapVertexToTraitInstance(AtlasVertex instanceVertex, ITypedReferenceableInstance typedInstance,
String traitName, TraitType traitType) throws AtlasException {
ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
mapVertexToTraitInstance(instanceVertex, typedInstance.getTypeName(), traitName, traitType, traitInstance);
}
private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName, String traitName,
private void mapVertexToTraitInstance(AtlasVertex<?,?> instanceVertex, String typedInstanceTypeName, String traitName,
TraitType traitType, ITypedStruct traitInstance) throws AtlasException {
String relationshipLabel = GraphHelper.getTraitLabel(typedInstanceTypeName, traitName);
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
Iterator<Edge> edgeIterator = graphHelper.getOutGoingEdgesByLabel(instanceVertex, relationshipLabel);
while (edgeIterator.hasNext()) {
Edge edge = edgeIterator.next();
final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
for (AtlasEdge<?,?> edge : instanceVertex.getEdges(AtlasEdgeDirection.OUT, relationshipLabel)) {
final AtlasVertex<?,?> traitInstanceVertex = edge.getInVertex();
if (traitInstanceVertex != null) {
LOG.debug("Found trait instance vertex {}, mapping to instance {} ", traitInstanceVertex,
traitInstance.getTypeName());
......@@ -309,45 +313,46 @@ public final class GraphToTypedInstanceMapper {
}
}
private void mapVertexToPrimitive(Vertex instanceVertex, ITypedInstance typedInstance,
private void mapVertexToPrimitive(AtlasVertex<?,?> instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws AtlasException {
LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
Object propertyValue = GraphHelper.getProperty(instanceVertex, vertexPropertyName);
if (propertyValue == null) {
if (GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Object.class) == null) {
return;
}
if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
typedInstance.setString(attributeInfo.name, (String) propertyValue);
typedInstance.setString(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, String.class));
} else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
typedInstance.setShort(attributeInfo.name, (Short) propertyValue);
typedInstance.setShort(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Short.class));
} else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
typedInstance.setInt(attributeInfo.name, (Integer) propertyValue);
typedInstance.setInt(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Integer.class));
} else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
typedInstance.setBigInt(attributeInfo.name, (BigInteger) propertyValue);
typedInstance.setBigInt(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, BigInteger.class));
} else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
typedInstance.setBoolean(attributeInfo.name, (Boolean) propertyValue);
typedInstance.setBoolean(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Boolean.class));
} else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
typedInstance.setByte(attributeInfo.name, (Byte) propertyValue);
typedInstance.setByte(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Byte.class));
} else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
typedInstance.setLong(attributeInfo.name, (Long) propertyValue);
typedInstance.setLong(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Long.class));
} else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
typedInstance.setFloat(attributeInfo.name, (Float) propertyValue);
typedInstance.setFloat(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Float.class));
} else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
typedInstance.setDouble(attributeInfo.name, (Double) propertyValue);
typedInstance.setDouble(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Double.class));
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
typedInstance.setBigDecimal(attributeInfo.name, (BigDecimal) propertyValue);
typedInstance
.setBigDecimal(attributeInfo.name, GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, BigDecimal.class));
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
final Long dateVal = (Long) propertyValue;
final Long dateVal = GraphHelper.getSingleValuedProperty(instanceVertex, vertexPropertyName, Long.class);
typedInstance.setDate(attributeInfo.name, new Date(dateVal));
}
}
public ITypedInstance getReferredEntity(String edgeId, IDataType<?> referredType) throws AtlasException {
final Edge edge = titanGraph.getEdge(edgeId);
final AtlasEdge edge = graph.getEdge(edgeId);
if (edge != null) {
final Vertex referredVertex = edge.getVertex(Direction.IN);
final AtlasVertex referredVertex = edge.getInVertex();
if (referredVertex != null) {
switch (referredType.getTypeCategory()) {
case STRUCT:
......
repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java
......@@ -19,8 +19,8 @@
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.types.TypeSystem;
......@@ -32,12 +32,12 @@ public class HardDeleteHandler extends DeleteHandler {
}
@Override
protected void _deleteVertex(Vertex instanceVertex, boolean force) {
protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
graphHelper.removeVertex(instanceVertex);
}
@Override
protected void deleteEdge(Edge edge, boolean force) throws AtlasException {
protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasException {
graphHelper.removeEdge(edge);
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java → repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java
......@@ -18,10 +18,15 @@
package org.apache.atlas.repository.graph;
import com.google.inject.throwingproviders.CheckedProvider;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
@Override
T get();
/**
* Provides a mechanism to control what graph is used in various places. This
* allows the graph to be mocked out during unit testing and be initialized
* lazily.
*/
public interface IAtlasGraphProvider {
AtlasGraph get() throws RepositoryException;
}
repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java
......@@ -19,8 +19,8 @@
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -36,7 +36,7 @@ public class SoftDeleteHandler extends DeleteHandler {
}
@Override
protected void _deleteVertex(Vertex instanceVertex, boolean force) {
protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
if (force) {
graphHelper.removeVertex(instanceVertex);
} else {
......@@ -50,7 +50,7 @@ public class SoftDeleteHandler extends DeleteHandler {
}
@Override
protected void deleteEdge(Edge edge, boolean force) throws AtlasException {
protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasException {
if (force) {
graphHelper.removeEdge(edge);
} else {
......
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
......@@ -17,15 +17,26 @@
*/
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -48,23 +59,12 @@ import org.apache.atlas.utils.MD5Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import com.google.inject.Inject;
public final class TypedInstanceToGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class);
private final Map<Id, Vertex> idToVertexMap = new HashMap<>();
private final Map<Id, AtlasVertex> idToVertexMap = new HashMap<>();
private final TypeSystem typeSystem = TypeSystem.getInstance();
private static final GraphHelper graphHelper = GraphHelper.getInstance();
......@@ -143,7 +143,7 @@ public final class TypedInstanceToGraphMapper {
//new vertex, set all the properties
String guid = addOrUpdateAttributesAndTraits(operation, instance);
guids.add(guid);
} catch (SchemaViolationException e) {
} catch (AtlasSchemaViolationException e) {
throw new EntityExistsException(instance, e);
}
}
......@@ -159,7 +159,7 @@ public final class TypedInstanceToGraphMapper {
throw new RepositoryException("id cannot be null");
}
Vertex instanceVertex = idToVertexMap.get(id);
AtlasVertex instanceVertex = idToVertexMap.get(id);
// add the attributes for the instance
ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName());
......@@ -174,7 +174,7 @@ public final class TypedInstanceToGraphMapper {
return getId(typedInstance)._getId();
}
void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
void mapInstanceToVertex(ITypedInstance typedInstance, AtlasVertex instanceVertex,
Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation)
throws AtlasException {
......@@ -189,7 +189,7 @@ public final class TypedInstanceToGraphMapper {
RequestContext.get().getRequestTime());
}
void mapAttributeToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
void mapAttributeToVertex(ITypedInstance typedInstance, AtlasVertex instanceVertex,
AttributeInfo attributeInfo, Operation operation) throws AtlasException {
Object attrValue = typedInstance.get(attributeInfo.name);
LOG.debug("Mapping attribute {} = {}", attributeInfo.name, attrValue);
......@@ -213,11 +213,11 @@ public final class TypedInstanceToGraphMapper {
case CLASS:
String edgeLabel = graphHelper.getEdgeLabel(typedInstance, attributeInfo);
Edge currentEdge = graphHelper.getEdgeForLabel(instanceVertex, edgeLabel);
String newEdgeId = addOrUpdateReference(instanceVertex, attributeInfo, attributeInfo.dataType(),
AtlasEdge currentEdge = graphHelper.getEdgeForLabel(instanceVertex, edgeLabel);
AtlasEdge newEdge = addOrUpdateReference(instanceVertex, attributeInfo, attributeInfo.dataType(),
attrValue, currentEdge, edgeLabel, operation);
if (currentEdge != null && !currentEdge.getId().toString().equals(newEdgeId)) {
if (currentEdge != null && !currentEdge.equals(newEdge)) {
deleteHandler.deleteEdgeReference(currentEdge, attributeInfo.dataType().getTypeCategory(),
attributeInfo.isComposite, true);
}
......@@ -245,7 +245,7 @@ public final class TypedInstanceToGraphMapper {
Id id = instance.getId();
if (!idToVertexMap.containsKey(id)) {
Vertex instanceVertex;
AtlasVertex instanceVertex;
if (id.isAssigned()) { // has a GUID
LOG.debug("Instance has an assigned id {}", instance.getId()._getId());
instanceVertex = graphHelper.getVertexForGUID(id.id);
......@@ -291,13 +291,13 @@ public final class TypedInstanceToGraphMapper {
private void addFullTextProperty(List<ITypedReferenceableInstance> instances, FullTextMapper fulltextMapper) throws AtlasException {
for (ITypedReferenceableInstance typedInstance : instances) { // Traverse
Vertex instanceVertex = getClassVertex(typedInstance);
AtlasVertex instanceVertex = getClassVertex(typedInstance);
String fullText = fulltextMapper.mapRecursive(instanceVertex, true);
GraphHelper.setProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
}
}
private void addTraits(ITypedReferenceableInstance typedInstance, Vertex instanceVertex, ClassType classType)
private void addTraits(ITypedReferenceableInstance typedInstance, AtlasVertex instanceVertex, ClassType classType)
throws AtlasException {
for (String traitName : typedInstance.getTraits()) {
LOG.debug("mapping trait {}", traitName);
......@@ -311,7 +311,7 @@ public final class TypedInstanceToGraphMapper {
/******************************************** ARRAY **************************************************/
private void mapArrayCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
private void mapArrayCollectionToVertex(ITypedInstance typedInstance, AtlasVertex instanceVertex,
AttributeInfo attributeInfo, Operation operation) throws AtlasException {
LOG.debug("Mapping instance {} for array attribute {} vertex {}", typedInstance.toShortString(),
attributeInfo.name, string(instanceVertex));
......@@ -323,16 +323,18 @@ public final class TypedInstanceToGraphMapper {
return;
}
String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
List<String> currentElements = GraphHelper.getProperty(instanceVertex, propertyName);
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
List<Object> currentElements = GraphHelper.getArrayElementsProperty(elementType, instanceVertex, propertyName);
List<Object> newElementsCreated = new ArrayList<>();
if (!newAttributeEmpty) {
if (newElements != null && !newElements.isEmpty()) {
int index = 0;
for (; index < newElements.size(); index++) {
String currentElement = (currentElements != null && index < currentElements.size()) ?
Object currentElement = (currentElements != null && index < currentElements.size()) ?
currentElements.get(index) : null;
LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index,
currentElement, newElements.get(index));
......@@ -343,18 +345,21 @@ public final class TypedInstanceToGraphMapper {
}
}
List<String> additionalEdges = removeUnusedEntries(instanceVertex, propertyName, currentElements,
newElementsCreated, elementType, attributeInfo);
if(GraphHelper.isReference(elementType)) {
List<AtlasEdge> additionalEdges = removeUnusedEntries(instanceVertex, propertyName, (List)currentElements,
(List)newElementsCreated, elementType, attributeInfo);
newElementsCreated.addAll(additionalEdges);
}
// for dereference on way out
GraphHelper.setProperty(instanceVertex, propertyName, newElementsCreated);
GraphHelper.setArrayElementsProperty(elementType, instanceVertex, propertyName, newElementsCreated);
}
//Removes unused edges from the old collection, compared to the new collection
private List<String> removeUnusedEntries(Vertex instanceVertex, String edgeLabel,
Collection<String> currentEntries,
Collection<Object> newEntries,
private List<AtlasEdge> removeUnusedEntries(AtlasVertex instanceVertex, String edgeLabel,
Collection<AtlasEdge> currentEntries,
Collection<AtlasEdge> newEntries,
IDataType entryType, AttributeInfo attributeInfo) throws AtlasException {
if (currentEntries != null && !currentEntries.isEmpty()) {
LOG.debug("Removing unused entries from the old collection");
......@@ -362,20 +367,17 @@ public final class TypedInstanceToGraphMapper {
|| entryType.getTypeCategory() == DataTypes.TypeCategory.CLASS) {
//Remove the edges for (current edges - new edges)
List<String> cloneElements = new ArrayList<>(currentEntries);
List<AtlasEdge> cloneElements = new ArrayList<>(currentEntries);
cloneElements.removeAll(newEntries);
List<String> additionalElements = new ArrayList<>();
List<AtlasEdge> additionalElements = new ArrayList<>();
LOG.debug("Removing unused entries from the old collection - {}", cloneElements);
if (!cloneElements.isEmpty()) {
for (String edgeIdForDelete : cloneElements) {
Edge edge = graphHelper.getEdgeByEdgeId(instanceVertex, edgeLabel, edgeIdForDelete);
if(edge != null) {
for (AtlasEdge edge : cloneElements) {
boolean deleted = deleteHandler.deleteEdgeReference(edge, entryType.getTypeCategory(),
attributeInfo.isComposite, true);
if (!deleted) {
additionalElements.add(edgeIdForDelete);
}
additionalElements.add(edge);
}
}
}
......@@ -387,7 +389,7 @@ public final class TypedInstanceToGraphMapper {
/******************************************** MAP **************************************************/
private void mapMapCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
private void mapMapCollectionToVertex(ITypedInstance typedInstance, AtlasVertex instanceVertex,
AttributeInfo attributeInfo, Operation operation) throws AtlasException {
LOG.debug("Mapping instance {} to vertex {} for attribute {}", typedInstance.toShortString(), string(instanceVertex),
attributeInfo.name);
......@@ -402,20 +404,20 @@ public final class TypedInstanceToGraphMapper {
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
Map<String, String> currentMap = new HashMap<>();
Map<String, Object> currentMap = new HashMap<>();
Map<String, Object> newMap = new HashMap<>();
List<String> currentKeys = GraphHelper.getProperty(instanceVertex, propertyName);
List<String> currentKeys = GraphHelper.getListProperty(instanceVertex, propertyName);
if (currentKeys != null && !currentKeys.isEmpty()) {
for (String key : currentKeys) {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
String propertyValueForKey = GraphHelper.getProperty(instanceVertex, propertyNameForKey).toString();
Object propertyValueForKey = GraphHelper.getMapValueProperty(elementType, instanceVertex, propertyNameForKey);
currentMap.put(key, propertyValueForKey);
}
}
if (!newAttributeEmpty) {
for (Map.Entry entry : newAttribute.entrySet()) {
for (Map.Entry<Object,Object> entry : newAttribute.entrySet()) {
String keyStr = entry.getKey().toString();
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, keyStr);
......@@ -423,53 +425,53 @@ public final class TypedInstanceToGraphMapper {
entry.getValue(), currentMap.get(keyStr), propertyNameForKey, operation);
//Add/Update/Remove property value
GraphHelper.setProperty(instanceVertex, propertyNameForKey, newEntry);
GraphHelper.setMapValueProperty(elementType, instanceVertex, propertyNameForKey, newEntry);
newMap.put(keyStr, newEntry);
}
}
Map<String, String> additionalMap =
Map<String, Object> additionalMap =
removeUnusedMapEntries(instanceVertex, propertyName, currentMap, newMap, elementType, attributeInfo);
Set<String> newKeys = new HashSet<>(newMap.keySet());
newKeys.addAll(additionalMap.keySet());
// for dereference on way out
GraphHelper.setProperty(instanceVertex, propertyName, new ArrayList<>(newKeys));
GraphHelper.setListProperty(instanceVertex, propertyName, new ArrayList<>(newKeys));
}
//Remove unused entries from map
private Map<String, String> removeUnusedMapEntries(Vertex instanceVertex, String propertyName,
Map<String, String> currentMap,
private Map<String, Object> removeUnusedMapEntries(
AtlasVertex instanceVertex, String propertyName,
Map<String, Object> currentMap,
Map<String, Object> newMap, IDataType elementType,
AttributeInfo attributeInfo)
throws AtlasException {
boolean reference = (elementType.getTypeCategory() == DataTypes.TypeCategory.STRUCT
|| elementType.getTypeCategory() == DataTypes.TypeCategory.CLASS);
Map<String, String> additionalMap = new HashMap<>();
Map<String, Object> additionalMap = new HashMap<>();
for (String currentKey : currentMap.keySet()) {
boolean shouldDeleteKey = !newMap.containsKey(currentKey);
if (reference) {
String currentEdge = currentMap.get(currentKey);
if (GraphHelper.isReference(elementType)) {
//Delete the edge reference if its not part of new edges created/updated
AtlasEdge currentEdge = (AtlasEdge)currentMap.get(currentKey);
if (!newMap.values().contains(currentEdge)) {
String edgeLabel = GraphHelper.getQualifiedNameForMapKey(propertyName, currentKey);
Edge edge = graphHelper.getEdgeByEdgeId(instanceVertex, edgeLabel, currentMap.get(currentKey));
if(edge != null) {
boolean deleted =
deleteHandler.deleteEdgeReference(edge, elementType.getTypeCategory(), attributeInfo.isComposite, true);
deleteHandler.deleteEdgeReference(currentEdge, elementType.getTypeCategory(), attributeInfo.isComposite, true);
if (!deleted) {
additionalMap.put(currentKey, currentEdge);
shouldDeleteKey = false;
}
}
}
}
if (shouldDeleteKey) {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, currentKey);
graphHelper.setProperty(instanceVertex, propertyNameForKey, null);
GraphHelper.setProperty(instanceVertex, propertyNameForKey, null);
}
}
return additionalMap;
......@@ -477,8 +479,8 @@ public final class TypedInstanceToGraphMapper {
/******************************************** ARRAY & MAP **************************************************/
private Object addOrUpdateCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
IDataType elementType, Object newAttributeValue, String currentValue,
private Object addOrUpdateCollectionEntry(AtlasVertex instanceVertex, AttributeInfo attributeInfo,
IDataType elementType, Object newAttributeValue, Object currentValue,
String propertyName, Operation operation)
throws AtlasException {
......@@ -496,8 +498,7 @@ public final class TypedInstanceToGraphMapper {
case STRUCT:
case CLASS:
final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
Edge currentEdge = graphHelper.getEdgeByEdgeId(instanceVertex, edgeLabel, currentValue);
return addOrUpdateReference(instanceVertex, attributeInfo, elementType, newAttributeValue, currentEdge,
return addOrUpdateReference(instanceVertex, attributeInfo, elementType, newAttributeValue, (AtlasEdge)currentValue,
edgeLabel, operation);
default:
......@@ -505,8 +506,8 @@ public final class TypedInstanceToGraphMapper {
}
}
private String addOrUpdateReference(Vertex instanceVertex, AttributeInfo attributeInfo,
IDataType attributeType, Object newAttributeValue, Edge currentEdge,
private AtlasEdge addOrUpdateReference(AtlasVertex instanceVertex, AttributeInfo attributeInfo,
IDataType attributeType, Object newAttributeValue, AtlasEdge currentEdge,
String edgeLabel, Operation operation) throws AtlasException {
switch (attributeType.getTypeCategory()) {
case STRUCT:
......@@ -523,26 +524,26 @@ public final class TypedInstanceToGraphMapper {
}
/******************************************** STRUCT **************************************************/
private String addOrUpdateStruct(Vertex instanceVertex, AttributeInfo attributeInfo,
ITypedStruct newAttributeValue, Edge currentEdge,
private AtlasEdge addOrUpdateStruct(AtlasVertex instanceVertex, AttributeInfo attributeInfo,
ITypedStruct newAttributeValue, AtlasEdge currentEdge,
String edgeLabel, Operation operation) throws AtlasException {
String newEdgeId = null;
if (currentEdge != null && newAttributeValue != null) {
AtlasEdge newEdge = null;
if (GraphHelper.elementExists(currentEdge) && newAttributeValue != null) {
//update
updateStructVertex(newAttributeValue, currentEdge, operation);
newEdgeId = currentEdge.getId().toString();
} else if (currentEdge == null && newAttributeValue != null) {
newEdge = currentEdge;
} else if (! GraphHelper.elementExists(currentEdge) && newAttributeValue != null) {
//add
Edge newEdge = addStructVertex(newAttributeValue, instanceVertex, attributeInfo, edgeLabel);
newEdgeId = newEdge.getId().toString();
newEdge = addStructVertex(newAttributeValue, instanceVertex, attributeInfo, edgeLabel);
}
return newEdgeId;
return newEdge;
}
private Edge addStructVertex(ITypedStruct structInstance, Vertex instanceVertex,
private AtlasEdge addStructVertex(ITypedStruct structInstance, AtlasVertex instanceVertex,
AttributeInfo attributeInfo, String edgeLabel) throws AtlasException {
// add a new vertex for the struct or trait instance
Vertex structInstanceVertex = graphHelper.createVertexWithoutIdentity(structInstance.getTypeName(), null,
AtlasVertex structInstanceVertex = graphHelper.createVertexWithoutIdentity(structInstance.getTypeName(), null,
Collections.<String>emptySet()); // no super types for struct type
LOG.debug("created vertex {} for struct {} value {}", string(structInstanceVertex), attributeInfo.name,
structInstance.toShortString());
......@@ -551,22 +552,22 @@ public final class TypedInstanceToGraphMapper {
mapInstanceToVertex(structInstance, structInstanceVertex, structInstance.fieldMapping().fields, false,
Operation.CREATE);
// add an edge to the newly created vertex from the parent
Edge newEdge = graphHelper.getOrCreateEdge(instanceVertex, structInstanceVertex, edgeLabel);
AtlasEdge newEdge = graphHelper.getOrCreateEdge(instanceVertex, structInstanceVertex, edgeLabel);
return newEdge;
}
private void updateStructVertex(ITypedStruct newAttributeValue, Edge currentEdge,
private void updateStructVertex(ITypedStruct newAttributeValue, AtlasEdge currentEdge,
Operation operation) throws AtlasException {
//Already existing vertex. Update
Vertex structInstanceVertex = currentEdge.getVertex(Direction.IN);
AtlasVertex structInstanceVertex = currentEdge.getInVertex();
LOG.debug("Updating struct vertex {} with struct {}", string(structInstanceVertex), newAttributeValue.toShortString());
// Update attributes
final MessageDigest digester = MD5Utils.getDigester();
String newSignature = newAttributeValue.getSignatureHash(digester);
String curSignature = GraphHelper.getProperty(structInstanceVertex, SIGNATURE_HASH_PROPERTY_KEY);
String curSignature = GraphHelper.getSingleValuedProperty(structInstanceVertex, SIGNATURE_HASH_PROPERTY_KEY, String.class);
if (!newSignature.equals(curSignature)) {
//Update struct vertex instance only if there is a change
......@@ -578,33 +579,34 @@ public final class TypedInstanceToGraphMapper {
/******************************************** CLASS **************************************************/
private String addOrUpdateClassVertex(Vertex instanceVertex, Edge currentEdge,
private AtlasEdge addOrUpdateClassVertex(AtlasVertex instanceVertex, AtlasEdge currentEdge,
ITypedReferenceableInstance newAttributeValue, AttributeInfo attributeInfo,
String edgeLabel) throws AtlasException {
Vertex newReferenceVertex = getClassVertex(newAttributeValue);
if(newReferenceVertex == null && newAttributeValue != null) {
AtlasVertex newReferenceVertex = getClassVertex(newAttributeValue);
if( ! GraphHelper.elementExists(newReferenceVertex) && newAttributeValue != null) {
LOG.error("Could not find vertex for Class Reference " + newAttributeValue);
throw new EntityNotFoundException("Could not find vertex for Class Reference " + newAttributeValue);
}
String newEdgeId = null;
if (currentEdge != null && newAttributeValue != null) {
newEdgeId = updateClassEdge(instanceVertex, currentEdge, newAttributeValue, newReferenceVertex,
AtlasEdge newEdge = null;
if (GraphHelper.elementExists(currentEdge) && newAttributeValue != null) {
newEdge = updateClassEdge(instanceVertex, currentEdge, newAttributeValue, newReferenceVertex,
attributeInfo, edgeLabel);
} else if (currentEdge == null && newAttributeValue != null){
Edge newEdge = addClassEdge(instanceVertex, newReferenceVertex, edgeLabel);
newEdgeId = newEdge.getId().toString();
} else if (! GraphHelper.elementExists(currentEdge) && newAttributeValue != null){
newEdge = addClassEdge(instanceVertex, newReferenceVertex, edgeLabel);
}
return newEdgeId;
return newEdge;
}
private Edge addClassEdge(Vertex instanceVertex, Vertex toVertex, String edgeLabel) throws AtlasException {
private AtlasEdge addClassEdge(AtlasVertex instanceVertex, AtlasVertex toVertex, String edgeLabel) throws AtlasException {
// add an edge to the class vertex from the instance
return graphHelper.getOrCreateEdge(instanceVertex, toVertex, edgeLabel);
}
private Vertex getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
Vertex referenceVertex = null;
private AtlasVertex getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
AtlasVertex referenceVertex = null;
Id id = null;
if (typedReference != null) {
id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
......@@ -625,7 +627,7 @@ public final class TypedInstanceToGraphMapper {
Id id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
if (id.isUnassigned()) {
Vertex classVertex = idToVertexMap.get(id);
AtlasVertex classVertex = idToVertexMap.get(id);
String guid = GraphHelper.getIdFromVertex(classVertex);
id = new Id(guid, 0, typedReference.getTypeName());
}
......@@ -633,48 +635,48 @@ public final class TypedInstanceToGraphMapper {
}
private String updateClassEdge(Vertex instanceVertex, Edge currentEdge,
private AtlasEdge updateClassEdge(AtlasVertex instanceVertex, AtlasEdge currentEdge,
ITypedReferenceableInstance newAttributeValue,
Vertex newVertex, AttributeInfo attributeInfo,
AtlasVertex newVertex, AttributeInfo attributeInfo,
String edgeLabel) throws AtlasException {
LOG.debug("Updating {} for reference attribute {}", string(currentEdge), attributeInfo.name);
// Update edge if it exists
Vertex currentVertex = currentEdge.getVertex(Direction.IN);
AtlasVertex currentVertex = currentEdge.getInVertex();
String currentEntityId = GraphHelper.getIdFromVertex(currentVertex);
String newEntityId = getId(newAttributeValue).id;
String newEdgeId = currentEdge.getId().toString();
AtlasEdge newEdge = currentEdge;
if (!currentEntityId.equals(newEntityId)) {
// add an edge to the class vertex from the instance
if (newVertex != null) {
Edge newEdge = graphHelper.getOrCreateEdge(instanceVertex, newVertex, edgeLabel);
newEdgeId = newEdge.getId().toString();
newEdge = graphHelper.getOrCreateEdge(instanceVertex, newVertex, edgeLabel);
}
}
return newEdgeId;
return newEdge;
}
/******************************************** TRAITS ****************************************************/
void mapTraitInstanceToVertex(ITypedStruct traitInstance, IDataType entityType, Vertex parentInstanceVertex)
void mapTraitInstanceToVertex(ITypedStruct traitInstance, IDataType entityType, AtlasVertex parentInstanceVertex)
throws AtlasException {
// add a new vertex for the struct or trait instance
// add a new AtlasVertex for the struct or trait instance
final String traitName = traitInstance.getTypeName();
Vertex traitInstanceVertex = graphHelper.createVertexWithoutIdentity(traitInstance.getTypeName(), null,
AtlasVertex traitInstanceVertex = graphHelper.createVertexWithoutIdentity(traitInstance.getTypeName(), null,
typeSystem.getDataType(TraitType.class, traitName).getAllSuperTypeNames());
LOG.debug("created vertex {} for trait {}", string(traitInstanceVertex), traitName);
// map all the attributes to this newly created vertex
// map all the attributes to this newly created AtlasVertex
mapInstanceToVertex(traitInstance, traitInstanceVertex, traitInstance.fieldMapping().fields, false, Operation.CREATE);
// add an edge to the newly created vertex from the parent
// add an edge to the newly created AtlasVertex from the parent
String relationshipLabel = GraphHelper.getTraitLabel(entityType.getName(), traitName);
graphHelper.getOrCreateEdge(parentInstanceVertex, traitInstanceVertex, relationshipLabel);
}
/******************************************** PRIMITIVES **************************************************/
private void mapPrimitiveOrEnumToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
private void mapPrimitiveOrEnumToVertex(ITypedInstance typedInstance, AtlasVertex instanceVertex,
AttributeInfo attributeInfo) throws AtlasException {
Object attrValue = typedInstance.get(attributeInfo.name);
......
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
......@@ -18,25 +18,29 @@
package org.apache.atlas.repository.typestore;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasConstants;
import static org.apache.atlas.repository.graph.GraphHelper.setProperty;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
......@@ -53,13 +57,10 @@ import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import static org.apache.atlas.repository.graph.GraphHelper.setProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@Singleton
public class GraphBackedTypeStore implements ITypeStore {
......@@ -69,13 +70,13 @@ public class GraphBackedTypeStore implements ITypeStore {
private static Logger LOG = LoggerFactory.getLogger(GraphBackedTypeStore.class);
private final TitanGraph titanGraph;
private final AtlasGraph graph;
private GraphHelper graphHelper = GraphHelper.getInstance();
@Inject
public GraphBackedTypeStore(GraphProvider<TitanGraph> graphProvider) {
titanGraph = graphProvider.get();
public GraphBackedTypeStore() {
graph = AtlasGraphProvider.getGraphInstance();
}
@Override
......@@ -91,14 +92,14 @@ public class GraphBackedTypeStore implements ITypeStore {
case STRUCT:
StructType structType = (StructType) dataType;
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), dataType.getVersion(),
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(),
ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of());
break;
case TRAIT:
case CLASS:
HierarchicalType type = (HierarchicalType) dataType;
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.getVersion(), type.immediateAttrs,
storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.immediateAttrs,
type.superTypes);
break;
......@@ -109,14 +110,14 @@ public class GraphBackedTypeStore implements ITypeStore {
}
private void storeInGraph(EnumType dataType) {
Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), dataType.getVersion());
AtlasVertex AtlasVertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription());
List<String> values = new ArrayList<>(dataType.values().size());
for (EnumValue enumValue : dataType.values()) {
String key = getPropertyKey(dataType.getName(), enumValue.value);
setProperty(vertex, key, enumValue.ordinal);
setProperty(AtlasVertex, key, enumValue.ordinal);
values.add(enumValue.value);
}
setProperty(vertex, getPropertyKey(dataType.getName()), values);
setProperty(AtlasVertex, getPropertyKey(dataType.getName()), values);
}
private String getPropertyKey(String name) {
......@@ -131,9 +132,9 @@ public class GraphBackedTypeStore implements ITypeStore {
return PROPERTY_PREFIX + "edge." + parent + "." + child;
}
private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription, String typeVersion,
private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription,
ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes) throws AtlasException {
Vertex vertex = createVertex(category, typeName, typeDescription, typeVersion);
AtlasVertex vertex = createVertex(category, typeName, typeDescription);
List<String> attrNames = new ArrayList<>();
if (attributes != null) {
for (AttributeInfo attribute : attributes) {
......@@ -153,18 +154,18 @@ public class GraphBackedTypeStore implements ITypeStore {
if (superTypes != null) {
for (String superTypeName : superTypes) {
HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription(), AtlasConstants.DEFAULT_TYPE_VERSION);
AtlasVertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription());
graphHelper.getOrCreateEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL);
}
}
}
private void addReferencesForAttribute(TypeSystem typeSystem, Vertex vertex, AttributeInfo attribute)
private void addReferencesForAttribute(TypeSystem typeSystem, AtlasVertex vertex, AttributeInfo attribute)
throws AtlasException {
ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
List<IDataType> attrDataTypes = new ArrayList<>();
IDataType attrDataType = attribute.dataType();
String vertexTypeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
String vertexTypeName = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, String.class);
switch (attrDataType.getTypeCategory()) {
case ARRAY:
......@@ -201,7 +202,7 @@ public class GraphBackedTypeStore implements ITypeStore {
for (IDataType attrType : attrDataTypes) {
if (!coreTypes.contains(attrType.getName())) {
Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription(), attrType.getVersion());
AtlasVertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription());
String label = getEdgeLabel(vertexTypeName, attribute.name);
graphHelper.getOrCreateEdge(vertex, attrVertex, label);
}
......@@ -213,7 +214,7 @@ public class GraphBackedTypeStore implements ITypeStore {
public TypesDef restore() throws AtlasException {
//Get all vertices for type system
Iterator vertices =
titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
graph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
return getTypesFromVertices(vertices);
}
......@@ -221,25 +222,24 @@ public class GraphBackedTypeStore implements ITypeStore {
@Override
@GraphTransaction
public TypesDef restoreType(String typeName) throws AtlasException {
// Get vertex for the specified type name.
// Get AtlasVertex for the specified type name.
Iterator vertices =
titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
graph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
return getTypesFromVertices(vertices);
}
private TypesDef getTypesFromVertices(Iterator vertices) throws AtlasException {
private TypesDef getTypesFromVertices(Iterator<AtlasVertex> vertices) throws AtlasException {
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
while (vertices.hasNext()) {
Vertex vertex = (Vertex) vertices.next();
DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
String typeVersion = vertex.getProperty(Constants.TYPEVERSION_PROPERTY_KEY);
AtlasVertex vertex = vertices.next();
DataTypes.TypeCategory typeCategory = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, TypeCategory.class);
String typeName = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, String.class);
String typeDescription = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, String.class);
LOG.info("Restoring type {}.{}.{}", typeCategory, typeName, typeDescription);
switch (typeCategory) {
case ENUM:
......@@ -248,19 +248,19 @@ public class GraphBackedTypeStore implements ITypeStore {
case STRUCT:
AttributeDefinition[] attributes = getAttributes(vertex, typeName);
structs.add(new StructTypeDefinition(typeName, typeDescription, typeVersion, attributes));
structs.add(new StructTypeDefinition(typeName, typeDescription, attributes));
break;
case CLASS:
ImmutableSet<String> superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex, typeName);
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, typeVersion, superTypes, attributes));
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, superTypes, attributes));
break;
case TRAIT:
superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex, typeName);
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, typeVersion, superTypes, attributes));
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, superTypes, attributes));
break;
default:
......@@ -270,37 +270,40 @@ public class GraphBackedTypeStore implements ITypeStore {
return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
}
private EnumTypeDefinition getEnumType(Vertex vertex) {
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
String typeVersion = vertex.getProperty(Constants.TYPEVERSION_PROPERTY_KEY);
private EnumTypeDefinition getEnumType(AtlasVertex vertex) throws AtlasException {
String typeName = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, String.class);
String typeDescription = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, String.class);
List<EnumValue> enumValues = new ArrayList<>();
List<String> values = graphHelper.getProperty(vertex, getPropertyKey(typeName));
List<String> values = vertex.getListProperty(getPropertyKey(typeName));
for (String value : values) {
String valueProperty = getPropertyKey(typeName, value);
enumValues.add(new EnumValue(value, (Integer) graphHelper.getProperty(vertex, valueProperty)));
enumValues.add(new EnumValue(value, GraphHelper.getSingleValuedProperty(vertex, valueProperty, Integer.class)));
}
return new EnumTypeDefinition(typeName, typeDescription, typeVersion, enumValues.toArray(new EnumValue[enumValues.size()]));
return new EnumTypeDefinition(typeName, typeDescription, enumValues.toArray(new EnumValue[enumValues.size()]));
}
private ImmutableSet<String> getSuperTypes(Vertex vertex) {
private ImmutableSet<String> getSuperTypes(AtlasVertex vertex) {
Set<String> superTypes = new HashSet<>();
Iterator<Edge> edges = graphHelper.getOutGoingEdgesByLabel(vertex, SUPERTYPE_EDGE_LABEL);
Iterator<AtlasEdge> edges = vertex.getEdges(AtlasEdgeDirection.OUT, SUPERTYPE_EDGE_LABEL).iterator();
while (edges.hasNext()) {
Edge edge = edges.next();
superTypes.add((String) edge.getVertex(Direction.IN).getProperty(Constants.TYPENAME_PROPERTY_KEY));
AtlasEdge edge = edges.next();
superTypes.add(edge.getInVertex().getProperty(Constants.TYPENAME_PROPERTY_KEY, String.class));
}
return ImmutableSet.copyOf(superTypes);
}
private AttributeDefinition[] getAttributes(Vertex vertex, String typeName) throws AtlasException {
private AttributeDefinition[] getAttributes(AtlasVertex vertex, String typeName) throws AtlasException {
List<AttributeDefinition> attributes = new ArrayList<>();
List<String> attrNames = graphHelper.getProperty(vertex, getPropertyKey(typeName));
List<String> attrNames = vertex.getListProperty(getPropertyKey(typeName));
if (attrNames != null) {
for (String attrName : attrNames) {
try {
String propertyKey = getPropertyKey(typeName, attrName);
attributes.add(AttributeInfo.fromJson((String) graphHelper.getProperty(vertex, propertyKey)));
AttributeDefinition attrValue = AttributeInfo.fromJson((String) vertex.getJsonProperty(propertyKey));
if (attrValue != null)
{
attributes.add(attrValue);
}
} catch (JSONException e) {
throw new AtlasException(e);
}
......@@ -315,24 +318,24 @@ public class GraphBackedTypeStore implements ITypeStore {
* @param typeName
* @return vertex
*/
Vertex findVertex(DataTypes.TypeCategory category, String typeName) {
LOG.debug("Finding vertex for {}.{}", category, typeName);
AtlasVertex findVertex(DataTypes.TypeCategory category, String typeName) {
LOG.debug("Finding AtlasVertex for {}.{}", category, typeName);
Iterator results = titanGraph.query().has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
Vertex vertex = null;
Iterator results = graph.query().has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
AtlasVertex vertex = null;
if (results != null && results.hasNext()) {
//There should be just one vertex with the given typeName
vertex = (Vertex) results.next();
//There should be just one AtlasVertex with the given typeName
vertex = (AtlasVertex) results.next();
}
return vertex;
}
private Vertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription, String typeVersion) {
Vertex vertex = findVertex(category, typeName);
private AtlasVertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription) {
AtlasVertex vertex = findVertex(category, typeName);
if (vertex == null) {
LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
vertex = titanGraph.addVertex(null);
setProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark as type vertex
vertex = graph.addVertex();
setProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark as type AtlasVertex
setProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category);
setProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName);
}
......@@ -344,16 +347,6 @@ public class GraphBackedTypeStore implements ITypeStore {
} else {
LOG.debug(" type description is null ");
}
if (typeVersion != null) {
String oldVersion = getPropertyKey(Constants.TYPEVERSION_PROPERTY_KEY);
if (!typeVersion.equals(oldVersion)) {
setProperty(vertex, Constants.TYPEVERSION_PROPERTY_KEY, typeVersion);
LOG.info(" updating type {} to version {}", typeName, typeVersion);
}
} else {
LOG.info(" type version is null ");
}
return vertex;
}
}
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -18,10 +18,18 @@
package org.apache.atlas.services;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -36,6 +44,7 @@ import org.apache.atlas.query.QueryParser;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -60,29 +69,22 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.collection.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
/**
* Simple wrapper over TypeSystem and MetadataRepository services with hooks
......@@ -119,7 +121,8 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
}
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
//for testing only
public DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders,
......@@ -153,8 +156,6 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
restoreTypeSystem();
}
AtlasPatchHandler.handlePatches(this, typeSystem);
maxAuditResults = configuration.getShort(CONFIG_MAX_AUDIT_RESULTS, DEFAULT_MAX_AUDIT_RESULTS);
}
......@@ -243,6 +244,7 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
typeDefinition = ParamChecker.notEmpty(typeDefinition, "type definition");
TypesDef typesDef = validateTypeDefinition(typeDefinition);
try {
final TypeSystem.TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(typesDef, isUpdate);
final Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
......@@ -334,40 +336,13 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
return guids;
}
private ITypedReferenceableInstance[] deserializeClassInstances(String entityInstanceDefinition)
throws AtlasException {
try {
JSONArray referableInstances = new JSONArray(entityInstanceDefinition);
ITypedReferenceableInstance[] instances = new ITypedReferenceableInstance[referableInstances.length()];
for (int index = 0; index < referableInstances.length(); index++) {
Referenceable entityInstance =
InstanceSerialization.fromJsonReferenceable(referableInstances.getString(index), true);
ITypedReferenceableInstance typedInstrance = getTypedReferenceableInstance(entityInstance);
instances[index] = typedInstrance;
}
return instances;
} catch(ValueConversionException | TypeNotFoundException e) {
throw e;
} catch (Exception e) { // exception from deserializer
LOG.error("Unable to deserialize json={}", entityInstanceDefinition, e);
throw new IllegalArgumentException("Unable to deserialize json", e);
}
private ITypedReferenceableInstance[] deserializeClassInstances(String entityInstanceDefinition) throws AtlasException {
return GraphHelper.deserializeClassInstances(typeSystem, entityInstanceDefinition);
}
@Override
public ITypedReferenceableInstance getTypedReferenceableInstance(Referenceable entityInstance) throws AtlasException {
final String entityTypeName = ParamChecker.notEmpty(entityInstance.getTypeName(), "Entity type cannot be null");
ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName);
//Both assigned id and values are required for full update
//classtype.convert() will remove values if id is assigned. So, set temp id, convert and
// then replace with original id
Id origId = entityInstance.getId();
entityInstance.replaceWithNewId(new Id(entityInstance.getTypeName()));
ITypedReferenceableInstance typedInstrance = entityType.convert(entityInstance, Multiplicity.REQUIRED);
((ReferenceableInstance)typedInstrance).replaceWithNewId(origId);
return typedInstrance;
return GraphHelper.getTypedReferenceableInstance(typeSystem, entityInstance);
}
/**
......
repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.util;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.SoftDeleteHandler;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Atlas configuration for repository project
*
*/
public class AtlasRepositoryConfiguration {
private static Logger LOG = LoggerFactory.getLogger(AtlasRepositoryConfiguration.class);
public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
@SuppressWarnings("unchecked")
public static Class<? extends TypeCache> getTypeCache() {
// Get the type cache implementation class from Atlas configuration.
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config, TYPE_CACHE_IMPLEMENTATION_PROPERTY,
DefaultTypeCache.class.getName(), TypeCache.class);
} catch (AtlasException e) {
LOG.error("Error loading typecache ", e);
return DefaultTypeCache.class;
}
}
private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
@SuppressWarnings("unchecked")
public static Class<? extends EntityAuditRepository> getAuditRepositoryImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
@SuppressWarnings("unchecked")
public static Class<? extends DeleteHandler> getDeleteHandlerImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
private static final String GRAPH_DATABASE_IMPLEMENTATION_PROPERTY = "atlas.graphdb.backend";
private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = "org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase";
@SuppressWarnings("unchecked")
public static Class<? extends GraphDatabase> getGraphDatabaseImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
GRAPH_DATABASE_IMPLEMENTATION_PROPERTY, DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS, GraphDatabase.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
}
repository/src/main/scala/org/apache/atlas/query/ClosureQuery.scala 100755 → 100644
......@@ -20,7 +20,7 @@ package org.apache.atlas.query
import java.util
import com.thinkaurelius.titan.core.TitanGraph
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.query.Expressions._
import org.apache.atlas.typesystem.ITypedStruct
import org.apache.atlas.typesystem.json.{InstanceSerialization, Serialization}
......@@ -116,7 +116,7 @@ trait ClosureQuery {
def withPath : Boolean
def persistenceStrategy: GraphPersistenceStrategies
def g: TitanGraph
def g: AtlasGraph[_,_]
def pathExpr : Expressions.Expression = {
closureRelation.tail.foldLeft(closureRelation.head.toExpr)((b,a) => b.field(a.toFieldName))
......@@ -184,8 +184,8 @@ trait ClosureQuery {
* foreach resultRow
* for each Path entry
* add an entry in the edges Map
* add an entry for the Src Vertex to the vertex Map
* add an entry for the Dest Vertex to the vertex Map
* add an entry for the Src AtlasVertex to the vertex Map
* add an entry for the Dest AtlasVertex to the vertex Map
*/
res.rows.map(_.asInstanceOf[StructInstance]).foreach { r =>
......@@ -207,7 +207,7 @@ trait ClosureQuery {
}
currVertex = nextVertex
}
val vertex = r.get(TypeUtils.ResultWithPathStruct.resultAttrName)
val AtlasVertex = r.get(TypeUtils.ResultWithPathStruct.resultAttrName)
vertices.put(id(srcVertex), vertexStruct(srcVertex,
r.get(TypeUtils.ResultWithPathStruct.resultAttrName).asInstanceOf[ITypedStruct],
s"${SRC_PREFIX}_"))
......@@ -242,6 +242,7 @@ trait SingleInstanceClosureQuery[T] extends ClosureQuery {
}
}
import scala.language.existentials;
/**
* A ClosureQuery to compute '''Lineage''' for Hive tables. Assumes the Lineage relation is captured in a ''CTAS''
* type, and the table relations are captured as attributes from a CTAS instance to Table instances.
......@@ -266,7 +267,7 @@ case class InputLineageClosureQuery(tableTypeName : String,
selectAttributes : Option[List[String]],
withPath : Boolean,
persistenceStrategy: GraphPersistenceStrategies,
g: TitanGraph
g: AtlasGraph[_,_]
) extends SingleInstanceClosureQuery[String] {
val closureType : String = tableTypeName
......@@ -306,7 +307,7 @@ case class OutputLineageClosureQuery(tableTypeName : String,
selectAttributes : Option[List[String]],
withPath : Boolean,
persistenceStrategy: GraphPersistenceStrategies,
g: TitanGraph
g: AtlasGraph[_,_]
) extends SingleInstanceClosureQuery[String] {
val closureType : String = tableTypeName
......
repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
......@@ -21,17 +21,20 @@ package org.apache.atlas.query
import java.util
import java.util.Date
import com.thinkaurelius.titan.core.TitanVertex
import com.tinkerpop.blueprints.{Vertex, Direction}
import org.apache.atlas.AtlasException
import scala.collection.JavaConversions.seqAsJavaList
import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException}
import org.apache.atlas.query.TypeUtils.FieldInfo
import org.apache.atlas.repository.graph.{GraphHelper, GraphBackedMetadataRepository}
import org.apache.atlas.repository.RepositoryException
import org.apache.atlas.repository.graphdb._
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.types.DataTypes._
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.types._
import org.apache.atlas.typesystem.{ITypedInstance, ITypedReferenceableInstance}
import scala.collection.JavaConversions._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
......@@ -40,12 +43,23 @@ import scala.collection.mutable.ArrayBuffer
* Represents the Bridge between the QueryProcessor and the Graph Persistence scheme used.
* Some of the behaviors captured are:
* - how is type and id information stored in the Vertex that represents an [[ITypedReferenceableInstance]]
* - how are edges representing trait and attribute relationships labelled.
* - how are edges representing trait and attribute relationships labeled.
* - how are attribute names mapped to Property Keys in Vertices.
*
* This is a work in progress.
*
*/
trait GraphPersistenceStrategies {
@throws(classOf[RepositoryException])
def getGraph() : AtlasGraph[_,_]
def getSupportedGremlinVersion() : GremlinVersion = getGraph().getSupportedGremlinVersion;
def generatePersisentToLogicalConversionExpression(expr: String, t: IDataType[_]) : String = getGraph().generatePersisentToLogicalConversionExpression(expr, t);
def isPropertyValueConversionNeeded(attrType: IDataType[_]) : Boolean = getGraph().isPropertyValueConversionNeeded(attrType);
def initialQueryCondition = if (getGraph().requiresInitialIndexedPredicate()) { s""".${getGraph().getInitialIndexedPredicate}""" } else {""};
/**
* Name of attribute used to store typeName in vertex
*/
......@@ -93,7 +107,7 @@ trait GraphPersistenceStrategies {
* @param v
* @return
*/
def traitNames(v: TitanVertex): java.util.List[String]
def traitNames(v: AtlasVertex[_,_]): java.util.List[String]
def edgeLabel(fInfo: FieldInfo): String = fInfo match {
case FieldInfo(dataType, aInfo, null, null) => edgeLabel(dataType, aInfo)
......@@ -101,7 +115,26 @@ trait GraphPersistenceStrategies {
case FieldInfo(dataType, null, null, traitName) => traitLabel(dataType, traitName)
}
def fieldPrefixInSelect: String
def fieldPrefixInSelect(): String = {
if(getSupportedGremlinVersion() == GremlinVersion.THREE) {
//this logic is needed to remove extra results from
//what is emitted by repeat loops. Technically
//for queries that don't have a loop in them we could just use "it"
//the reason for this is that in repeat loops with an alias,
//although the alias gets set to the right value, for some
//reason the select actually includes all vertices that were traversed
//through in the loop. In these cases, we only want the last vertex
//traversed in the loop to be selected. The logic here handles that
//case by converting the result to a list and just selecting the
//last item from it.
"((it as Vertex[]) as List<Vertex>).last()"
}
else {
"it"
}
}
/**
* extract the Id from a Vertex.
......@@ -109,11 +142,30 @@ trait GraphPersistenceStrategies {
* @param v
* @return
*/
def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id
def getIdFromVertex(dataTypeNm: String, v: AtlasVertex[_,_]): Id
def constructInstance[U](dataType: IDataType[U], v: java.lang.Object): U
def gremlinCompOp(op: ComparisonExpression) = op.symbol match {
def gremlinCompOp(op: ComparisonExpression) = {
if( getSupportedGremlinVersion() == GremlinVersion.TWO) {
gremlin2CompOp(op);
}
else {
gremlin3CompOp(op);
}
}
def gremlinPrimitiveOp(op: ComparisonExpression) = op.symbol match {
case "=" => "=="
case "!=" => "!="
case ">" => ">"
case ">=" => ">="
case "<" => "<"
case "<=" => "<="
case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin")
}
private def gremlin2CompOp(op: ComparisonExpression) = op.symbol match {
case "=" => "T.eq"
case "!=" => "T.neq"
case ">" => "T.gt"
......@@ -123,6 +175,16 @@ trait GraphPersistenceStrategies {
case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin")
}
private def gremlin3CompOp(op: ComparisonExpression) = op.symbol match {
case "=" => "eq"
case "!=" => "neq"
case ">" => "gt"
case ">=" => "gte"
case "<" => "lt"
case "<=" => "lte"
case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin")
}
def loopObjectExpression(dataType: IDataType[_]) = {
_typeTestExpression(dataType.getName, "it.object")
}
......@@ -162,12 +224,27 @@ trait GraphPersistenceStrategies {
Seq(s"""filter${_typeTestExpression(typeName, "it")}""")
}
/**
* type test expression that ends up in the emit clause in
* loop/repeat steps and a few other places
*/
private def _typeTestExpression(typeName: String, itRef: String): String = {
if( getSupportedGremlinVersion() == GremlinVersion.TWO) {
s"""{(${itRef}.'${typeAttributeName}' == '${typeName}') |
|(${itRef}.'${superTypeAttributeName}' ?
|${itRef}.'${superTypeAttributeName}'.contains('${typeName}') : false)}""".
| (${itRef}.'${superTypeAttributeName}' ?
| ${itRef}.'${superTypeAttributeName}'.contains('${typeName}') : false)}""".
stripMargin.replace(System.getProperty("line.separator"), "")
}
else {
//gremlin 3
s"""has('${typeAttributeName}',eq('${typeName}')).or().has('${superTypeAttributeName}',eq('${typeName}'))"""
}
}
private def propertyValueSet(vertexRef : String, attrName: String) : String = {
s"""org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils.set(${vertexRef}.values('${attrName})"""
}
private def typeTestExpressionMultiStep(typeName : String, intSeq : IntSequence) : Seq[String] = {
......@@ -176,11 +253,21 @@ trait GraphPersistenceStrategies {
newSetVar(varName),
fillVarWithTypeInstances(typeName, varName),
fillVarWithSubTypeInstances(typeName, varName),
if(getSupportedGremlinVersion() == GremlinVersion.TWO) {
s"$varName._()"
}
else {
//this bit of groovy magic converts the set of vertices in varName into
//a String containing the ids of all the vertices. This becomes the argument
//to g.V(). This is needed because Gremlin 3 does not support
// _()
//s"g.V(${varName}.collect{it.id()} as String[])"
s"g.V(${varName} as Object[])${initialQueryCondition}"
}
)
}
private def newSetVar(varName : String) = s"$varName = [] as Set"
private def newSetVar(varName : String) = s"def $varName = [] as Set"
private def fillVarWithTypeInstances(typeName : String, fillVar : String) = {
s"""g.V().has("${typeAttributeName}", "${typeName}").fill($fillVar)"""
......@@ -191,27 +278,36 @@ trait GraphPersistenceStrategies {
}
}
object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
import scala.language.existentials;
import org.apache.atlas.repository.RepositoryException
import org.apache.atlas.repository.RepositoryException
import org.apache.atlas.repository.RepositoryException
import org.apache.atlas.repository.RepositoryException
case class GraphPersistenceStrategy1(g: AtlasGraph[_,_]) extends GraphPersistenceStrategies {
val typeAttributeName = "typeName"
val superTypeAttributeName = "superTypeNames"
val idAttributeName = "guid"
val stateAttributeName = "state"
override def getGraph() : AtlasGraph[_,_] = {
return g;
}
def edgeLabel(dataType: IDataType[_], aInfo: AttributeInfo) = s"__${dataType.getName}.${aInfo.name}"
def edgeLabel(propertyName: String) = s"__${propertyName}"
val fieldPrefixInSelect = "it"
def traitLabel(cls: IDataType[_], traitName: String) = s"$traitName"
def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName"
def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphHelper.getQualifiedFieldName(dataType, aInfo.name)
def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id =
def getIdFromVertex(dataTypeNm: String, v: AtlasVertex[_,_]): Id =
new Id(v.getId.toString, 0, dataTypeNm)
def traitNames(v: TitanVertex): java.util.List[String] = {
val s = v.getProperty[String]("traitNames")
def traitNames(v: AtlasVertex[_,_]): java.util.List[String] = {
val s = v.getProperty("traitNames", classOf[String])
if (s != null) {
Seq[String](s.split(","): _*)
} else {
......@@ -228,17 +324,17 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
if dataType.getName == TypeSystem.getInstance().getIdType.getName => {
val sType = dataType.asInstanceOf[StructType]
val sInstance = sType.createInstance()
val tV = v.asInstanceOf[TitanVertex]
val tV = v.asInstanceOf[AtlasVertex[_,_]]
sInstance.set(TypeSystem.getInstance().getIdType.typeNameAttrName,
tV.getProperty[java.lang.String](typeAttributeName))
tV.getProperty(typeAttributeName, classOf[java.lang.String]))
sInstance.set(TypeSystem.getInstance().getIdType.idAttrName,
tV.getProperty[java.lang.String](idAttributeName))
tV.getProperty(idAttributeName, classOf[java.lang.String]))
dataType.convert(sInstance, Multiplicity.OPTIONAL)
}
case DataTypes.TypeCategory.STRUCT => {
val sType = dataType.asInstanceOf[StructType]
val sInstance = sType.createInstance()
loadStructInstance(sType, sInstance, v.asInstanceOf[TitanVertex])
loadStructInstance(sType, sInstance, v.asInstanceOf[AtlasVertex[_,_]])
dataType.convert(sInstance, Multiplicity.OPTIONAL)
}
case DataTypes.TypeCategory.TRAIT => {
......@@ -248,12 +344,12 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
* this is not right, we should load the Instance associated with this trait.
* for now just loading the trait struct.
*/
loadStructInstance(tType, tInstance, v.asInstanceOf[TitanVertex])
loadStructInstance(tType, tInstance, v.asInstanceOf[AtlasVertex[_,_]])
dataType.convert(tInstance, Multiplicity.OPTIONAL)
}
case DataTypes.TypeCategory.CLASS => {
val cType = dataType.asInstanceOf[ClassType]
val cInstance = constructClassInstance(dataType.asInstanceOf[ClassType], v.asInstanceOf[TitanVertex])
val cInstance = constructClassInstance(dataType.asInstanceOf[ClassType], v.asInstanceOf[AtlasVertex[_,_]])
dataType.convert(cInstance, Multiplicity.OPTIONAL)
}
case DataTypes.TypeCategory.ENUM => dataType.convert(v, Multiplicity.OPTIONAL)
......@@ -262,7 +358,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
}
def loadStructInstance(dataType: IConstructableType[_, _ <: ITypedInstance],
typInstance: ITypedInstance, v: TitanVertex): Unit = {
typInstance: ITypedInstance, v: AtlasVertex[_,_]): Unit = {
import scala.collection.JavaConversions._
dataType.fieldMapping().fields.foreach { t =>
val fName = t._1
......@@ -271,15 +367,15 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
}
}
def constructClassInstance(dataType: ClassType, v: TitanVertex): ITypedReferenceableInstance = {
def constructClassInstance(dataType: ClassType, v: AtlasVertex[_,_]): ITypedReferenceableInstance = {
val id = getIdFromVertex(dataType.name, v)
val tNms = traitNames(v)
val cInstance = dataType.createInstance(id, tNms: _*)
// load traits
tNms.foreach { tNm =>
val tLabel = traitLabel(dataType, tNm)
val edges = v.getEdges(Direction.OUT, tLabel)
val tVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex]
val edges = v.getEdges(AtlasEdgeDirection.OUT, tLabel)
val tVertex = edges.iterator().next().getInVertex().asInstanceOf[AtlasVertex[_,_]]
val tType = TypeSystem.getInstance().getDataType[TraitType](classOf[TraitType], tNm)
val tInstance = cInstance.getTrait(tNm).asInstanceOf[ITypedInstance]
loadStructInstance(tType, tInstance, tVertex)
......@@ -288,7 +384,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
cInstance
}
def loadAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex): Unit = {
def loadAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: AtlasVertex[_,_]): Unit = {
aInfo.dataType.getTypeCategory match {
case DataTypes.TypeCategory.PRIMITIVE => loadPrimitiveAttribute(dataType, aInfo, i, v)
case DataTypes.TypeCategory.ENUM => loadEnumAttribute(dataType, aInfo, i, v)
......@@ -303,26 +399,26 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
}
}
private def loadEnumAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex)
private def loadEnumAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: AtlasVertex[_,_])
: Unit = {
val fName = fieldNameInVertex(dataType, aInfo)
i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName))
i.setInt(aInfo.name, v.getProperty(fName, classOf[java.lang.Integer]))
}
private def loadPrimitiveAttribute(dataType: IDataType[_], aInfo: AttributeInfo,
i: ITypedInstance, v: TitanVertex): Unit = {
i: ITypedInstance, v: AtlasVertex[_,_]): Unit = {
val fName = fieldNameInVertex(dataType, aInfo)
aInfo.dataType() match {
case x: BooleanType => i.setBoolean(aInfo.name, v.getProperty[java.lang.Boolean](fName))
case x: ByteType => i.setByte(aInfo.name, v.getProperty[java.lang.Byte](fName))
case x: ShortType => i.setShort(aInfo.name, v.getProperty[java.lang.Short](fName))
case x: IntType => i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName))
case x: LongType => i.setLong(aInfo.name, v.getProperty[java.lang.Long](fName))
case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName))
case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName))
case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName))
case x: BooleanType => i.setBoolean(aInfo.name, v.getProperty(fName, classOf[java.lang.Boolean]))
case x: ByteType => i.setByte(aInfo.name, v.getProperty(fName, classOf[java.lang.Byte]))
case x: ShortType => i.setShort(aInfo.name, v.getProperty(fName, classOf[java.lang.Short]))
case x: IntType => i.setInt(aInfo.name, v.getProperty(fName, classOf[java.lang.Integer]))
case x: LongType => i.setLong(aInfo.name, v.getProperty(fName, classOf[java.lang.Long]))
case x: FloatType => i.setFloat(aInfo.name, v.getProperty(fName, classOf[java.lang.Float]))
case x: DoubleType => i.setDouble(aInfo.name, v.getProperty(fName, classOf[java.lang.Double]))
case x: StringType => i.setString(aInfo.name, v.getProperty(fName, classOf[java.lang.String]))
case x: DateType => {
val dateVal = v.getProperty[java.lang.Long](fName)
val dateVal = v.getProperty(fName, classOf[java.lang.Long])
i.setDate(aInfo.name, new Date(dateVal))
}
case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
......@@ -331,9 +427,9 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
private def loadArrayAttribute[T](dataType: IDataType[_], aInfo: AttributeInfo,
i: ITypedInstance, v: TitanVertex): Unit = {
i: ITypedInstance, v: AtlasVertex[_,_]): Unit = {
import scala.collection.JavaConversions._
val list: java.util.List[_] = v.getProperty(aInfo.name)
val list: java.util.List[_] = v.getListProperty(aInfo.name)
val arrayType: DataTypes.ArrayType = aInfo.dataType.asInstanceOf[ArrayType]
var values = new util.ArrayList[Any]
......@@ -344,13 +440,13 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
}
private def loadStructAttribute(dataType: IDataType[_], aInfo: AttributeInfo,
i: ITypedInstance, v: TitanVertex, edgeLbl: Option[String] = None): Unit = {
i: ITypedInstance, v: AtlasVertex[_,_], edgeLbl: Option[String] = None): Unit = {
val eLabel = edgeLbl match {
case Some(x) => x
case None => edgeLabel(FieldInfo(dataType, aInfo, null))
}
val edges = v.getEdges(Direction.OUT, eLabel)
val sVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex]
val edges = v.getEdges(AtlasEdgeDirection.OUT, eLabel)
val sVertex = edges.iterator().next().getInVertex().asInstanceOf[AtlasVertex[_,_]]
if (aInfo.dataType().getTypeCategory == DataTypes.TypeCategory.STRUCT) {
val sType = aInfo.dataType().asInstanceOf[StructType]
val sInstance = sType.createInstance()
......@@ -364,7 +460,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
private def mapVertexToCollectionEntry(instanceVertex: TitanVertex, attributeInfo: AttributeInfo, elementType: IDataType[_], i: ITypedInstance, value: Any): Any = {
private def mapVertexToCollectionEntry(instanceVertex: AtlasVertex[_,_], attributeInfo: AttributeInfo, elementType: IDataType[_], i: ITypedInstance, value: Any): Any = {
elementType.getTypeCategory match {
case DataTypes.TypeCategory.PRIMITIVE => value
case DataTypes.TypeCategory.ENUM => value
......
repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala
......@@ -18,10 +18,9 @@
package org.apache.atlas.query
import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
import org.apache.atlas.query.Expressions._
import com.thinkaurelius.titan.core.TitanGraph
import com.tinkerpop.pipes.util.structures.Row
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.query.TypeUtils.ResultWithPathStruct
import org.apache.atlas.typesystem.json._
import org.apache.atlas.typesystem.types._
......@@ -40,12 +39,7 @@ case class GremlinQueryResult(query: String,
def toJson = JsonHelper.toJson(this)
}
class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: TitanGraph) {
val manager: ScriptEngineManager = new ScriptEngineManager
val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
val bindings: Bindings = engine.createBindings
bindings.put("g", g)
class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: AtlasGraph[_,_]) {
/**
*
......@@ -53,16 +47,17 @@ class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceS
* @param qryResultObj is the object constructed for the output w/o the Path.
* @return a ResultWithPathStruct
*/
def addPathStruct(gResultObj : AnyRef, qryResultObj : Any) : Any = {
if ( !qry.isPathExpresion) {
def addPathStruct(gResultObj: AnyRef, qryResultObj: Any): Any = {
if (!qry.isPathExpression) {
qryResultObj
} else {
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
val iPaths = gResultObj.asInstanceOf[java.util.List[AnyRef]].init
val oPaths = iPaths.map { p =>
persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, p)
val oPaths = iPaths.map { value =>
persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, value)
}.toList.asJava
val sType = qry.expr.dataType.asInstanceOf[StructType]
val sInstance = sType.createInstance()
......@@ -72,8 +67,8 @@ class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceS
}
}
def instanceObject(v : AnyRef) : AnyRef = {
if ( qry.isPathExpresion ) {
def instanceObject(v: AnyRef): AnyRef = {
if (qry.isPathExpression) {
import scala.collection.JavaConversions._
v.asInstanceOf[java.util.List[AnyRef]].last
} else {
......@@ -83,29 +78,37 @@ class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceS
def evaluate(): GremlinQueryResult = {
import scala.collection.JavaConversions._
val debug:Boolean = false
val rType = qry.expr.dataType
val oType = if (qry.isPathExpresion) qry.expr.children(0).dataType else rType
val rawRes = engine.eval(qry.queryStr, bindings)
val oType = if (qry.isPathExpression) {
qry.expr.children(0).dataType
}
else {
rType
}
val rawRes = g.executeGremlinScript(qry.queryStr, qry.isPathExpression);
if(debug) {
println(" rawRes " +rawRes)
}
if (!qry.hasSelectList) {
val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { v =>
val iV = instanceObject(v)
val o = persistenceStrategy.constructInstance(oType, iV)
val instObj = instanceObject(v)
val o = persistenceStrategy.constructInstance(oType, instObj)
addPathStruct(v, o)
}
GremlinQueryResult(qry.expr.toString, rType, rows.toList)
} else {
val sType = oType.asInstanceOf[StructType]
val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { r =>
val rV = instanceObject(r).asInstanceOf[Row[java.util.List[AnyRef]]]
val rV = instanceObject(r)
val sInstance = sType.createInstance()
val selObj = SelectExpressionHelper.extractSelectExpression(qry.expr)
if (selObj.isDefined)
{
if (selObj.isDefined) {
val selExpr = selObj.get.asInstanceOf[Expressions.SelectExpression]
selExpr.selectListWithAlias.foreach { aE =>
val cName = aE.alias
val (src, idx) = qry.resultMaping(cName)
val v = rV.getColumn(src).get(idx)
val v = getColumnValue(rV, src, idx)
sInstance.set(cName, persistenceStrategy.constructInstance(aE.dataType, v))
}
}
......@@ -115,6 +118,30 @@ class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceS
}
}
private def getColumnValue(rowValue: AnyRef, colName: String, idx: Integer) : AnyRef = {
var rawColumnValue: AnyRef = null;
if(rowValue.isInstanceOf[java.util.Map[_,_]]) {
val columnsMap = rowValue.asInstanceOf[java.util.Map[String,AnyRef]];
rawColumnValue = columnsMap.get(colName);
}
else {
//when there is only one column, result does not come back as a map
rawColumnValue = rowValue;
}
var value : AnyRef = null;
if(rawColumnValue.isInstanceOf[java.util.List[_]] && idx >= 0) {
val arr = rawColumnValue.asInstanceOf[java.util.List[AnyRef]];
value = arr.get(idx);
}
else {
value = rawColumnValue;
}
return value;
}
}
object JsonHelper {
......
repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
......@@ -18,16 +18,30 @@
package org.apache.atlas.query
import org.apache.atlas.query.TypeUtils.FieldInfo
import org.apache.atlas.query.Expressions._
import org.apache.atlas.typesystem.types.{DataTypes, TypeSystem}
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory
import org.joda.time.format.ISODateTimeFormat
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import org.apache.atlas.query.TypeUtils.FieldInfo;
import org.apache.atlas.query.Expressions._
import org.apache.atlas.repository.graphdb.GremlinVersion
import org.apache.atlas.typesystem.types.DataTypes
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory
import org.apache.atlas.typesystem.types.IDataType
import org.apache.commons.lang.StringEscapeUtils
import org.apache.atlas.typesystem.types.TypeSystem
import org.apache.atlas.typesystem.types.AttributeInfo
import org.joda.time.format.ISODateTimeFormat
import org.apache.atlas.typesystem.types.DataTypes.BigDecimalType
import org.apache.atlas.typesystem.types.DataTypes.ByteType
import org.apache.atlas.typesystem.types.DataTypes.BooleanType
import org.apache.atlas.typesystem.types.DataTypes.DateType
import org.apache.atlas.typesystem.types.DataTypes.BigIntegerType
import org.apache.atlas.typesystem.types.DataTypes.IntType
import org.apache.atlas.typesystem.types.DataTypes.StringType
import org.apache.atlas.typesystem.types.DataTypes.LongType
import org.apache.atlas.typesystem.types.DataTypes.DoubleType
import org.apache.atlas.typesystem.types.DataTypes.FloatType
import org.apache.atlas.typesystem.types.DataTypes.ShortType
trait IntSequence {
def next: Int
......@@ -37,10 +51,11 @@ case class GremlinQuery(expr: Expression, queryStr: String, resultMaping: Map[St
def hasSelectList = resultMaping != null
def isPathExpresion = expr.isInstanceOf[PathExpression]
def isPathExpression = expr.isInstanceOf[PathExpression]
}
trait SelectExpressionHandling {
/**
......@@ -222,41 +237,50 @@ class GremlinTranslator(expr: Expression,
case fe@FieldExpression(fieldName, fInfo, child)
if fe.dataType.getTypeCategory == TypeCategory.PRIMITIVE || fe.dataType.getTypeCategory == TypeCategory.ARRAY => {
val fN = "\"" + gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo) + "\""
child match {
case Some(e) => s"${genQuery(e, inSelect)}.$fN"
case None => s"$fN"
}
genPropertyAccessExpr(child, fInfo, fN, inSelect)
}
case fe@FieldExpression(fieldName, fInfo, child)
if fe.dataType.getTypeCategory == TypeCategory.CLASS || fe.dataType.getTypeCategory == TypeCategory.STRUCT => {
val direction = if (fInfo.isReverse) "in" else "out"
val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo)
val step = s"""$direction("$edgeLbl")"""
child match {
case Some(e) => s"${genQuery(e, inSelect)}.$step"
case None => step
}
generateAndPrependExpr(child, inSelect, s"""$step""")
}
case fe@FieldExpression(fieldName, fInfo, child) if fInfo.traitName != null => {
val direction = gPersistenceBehavior.instanceToTraitEdgeDirection
val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo)
val step = s"""$direction("$edgeLbl")"""
child match {
case Some(e) => s"${genQuery(e, inSelect)}.$step"
case None => step
}
generateAndPrependExpr(child, inSelect, s"""$step""")
}
case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => {
val qualifiedPropertyName = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}"
val persistentExprValue = translateValueToPersistentForm(fInfo, l)
return generateAndPrependExpr(ch, inSelect, s"""has("${qualifiedPropertyName}", ${gPersistenceBehavior.gremlinCompOp(c)}, $persistentExprValue)""")
return genHasPredicate(ch, fInfo, fieldName, inSelect, c, l)
}
case fil@FilterExpression(child, condExpr) => {
s"${genQuery(child, inSelect)}.${genQuery(condExpr, inSelect)}"
}
case l@LogicalExpression(symb, children) => {
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.THREE) {
if(children.length == 1) {
//gremlin 3 treats one element expressions as 'false'. Avoid
//creating a boolean expression in this case. Inline the expression
//note: we can't simply omit it, since it will cause us to traverse the edge!
//use 'where' instead
var child : Expression = children.head;
//if child is a back expression, that expression becomes an argument to where
return s"""where(${genQuery(child, inSelect)})""";
}
else {
// Gremlin 3 does not support _() syntax
//
return s"""$symb${children.map( genQuery(_, inSelect)).mkString("(", ",", ")")}"""
}
}
else {
s"""$symb${children.map("_()." + genQuery(_, inSelect)).mkString("(", ",", ")")}"""
}
}
case sel@SelectExpression(child, selList) => {
val m = groupSelectExpressionsBySrc(sel)
var srcNamesList: List[String] = List()
......@@ -269,14 +293,25 @@ class GremlinTranslator(expr: Expression,
genQuery(selExpr, true)
}
}
val srcNamesString = srcNamesList.mkString("[", ",", "]")
val srcExprsStringList = srcExprsList.map {
_.mkString("[", ",", "]")
}
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) {
val srcNamesString = srcNamesList.mkString("[", ",", "]")
val srcExprsString = srcExprsStringList.foldLeft("")(_ + "{" + _ + "}")
s"${genQuery(child, inSelect)}.select($srcNamesString)$srcExprsString"
}
else {
//gremlin 3
val srcNamesString = srcNamesList.mkString("", ",", "")
val srcExprsString = srcExprsStringList.foldLeft("")(_ + ".by({" + _ + "} as Function)")
s"${genQuery(child, inSelect)}.select($srcNamesString)$srcExprsString"
}
}
case loop@LoopExpression(input, loopExpr, t) => {
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) {
val inputQry = genQuery(input, inSelect)
val loopingPathGExpr = genQuery(loopExpr, inSelect)
val loopGExpr = s"""loop("${input.asInstanceOf[AliasExpression].alias}")"""
......@@ -285,8 +320,31 @@ class GremlinTranslator(expr: Expression,
val enablePathExpr = s".enablePath()"
s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}${enablePathExpr}"""
}
case BackReference(alias, _, _) =>
if (inSelect) gPersistenceBehavior.fieldPrefixInSelect else s"""back("$alias")"""
else {
//gremlin 3 - TODO - add support for circular lineage
val inputQry = genQuery(input, inSelect)
val repeatExpr = s"""repeat(__.${genQuery(loopExpr, inSelect)})"""
val optTimesExpr = if (t.isDefined) s".times(${t.get.value})" else ""
val emitExpr = s""".emit(${gPersistenceBehavior.loopObjectExpression(input.dataType)})"""
s"""${inputQry}.${repeatExpr}${optTimesExpr}${emitExpr}"""
}
}
case BackReference(alias, _, _) => {
if (inSelect) {
gPersistenceBehavior.fieldPrefixInSelect()
}
else {
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) {
s"""back("$alias")"""
}
else {
s"""select("$alias")"""
}
}
}
case AliasExpression(child, alias) => s"""${genQuery(child, inSelect)}.as("$alias")"""
case isTraitLeafExpression(traitName, Some(clsExp)) =>
s"""out("${gPersistenceBehavior.traitLabel(clsExp.dataType, traitName)}")"""
......@@ -294,11 +352,14 @@ class GremlinTranslator(expr: Expression,
s"""out("${gPersistenceBehavior.traitLabel(child.dataType, traitName)}")"""
case hasFieldLeafExpression(fieldName, clsExp) => clsExp match {
case None => s"""has("$fieldName")"""
case Some(x) =>
x match {
case c: ClassExpression =>
s"""has("${x.asInstanceOf[ClassExpression].clsName}.$fieldName")"""
case default => s"""has("$fieldName")"""
case Some(x) => {
val fi = TypeUtils.resolveReference(clsExp.get.dataType, fieldName);
if(! fi.isDefined) {
s"""has("$fieldName")"""
}
else {
s"""has("${gPersistenceBehavior.fieldNameInVertex(fi.get.dataType, fi.get.attrInfo)}")"""
}
}
}
case hasFieldUnaryExpression(fieldName, child) =>
......@@ -314,33 +375,125 @@ class GremlinTranslator(expr: Expression,
s"${genQuery(child, inSelect)}"
}
case pe@PathExpression(child) => {
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) {
s"${genQuery(child, inSelect)}.path"
}
else {
s"${genQuery(child, inSelect)}.path()"
}
}
case order@OrderExpression(child, odr, asc) => {
var orderby = ""
var orderExpression = odr
if(odr.isInstanceOf[BackReference]) { orderExpression = odr.asInstanceOf[BackReference].reference }
else if (odr.isInstanceOf[AliasExpression]) { orderExpression = odr.asInstanceOf[AliasExpression].child}
if(odr.isInstanceOf[BackReference]) {
orderExpression = odr.asInstanceOf[BackReference].reference
}
else if (odr.isInstanceOf[AliasExpression]) {
orderExpression = odr.asInstanceOf[AliasExpression].child
}
val orderbyProperty = genQuery(orderExpression, false)
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) {
val bProperty = s"it.b.$orderbyProperty"
val aProperty = s"it.a.$orderbyProperty"
val aCondition = s"($aProperty != null ? $aProperty.toLowerCase(): $aProperty)"
val bCondition = s"($bProperty != null ? $bProperty.toLowerCase(): $bProperty)"
var orderby = ""
asc match {
orderby = asc match {
//builds a closure comparison function based on provided order by clause in DSL. This will be used to sort the results by gremlin order pipe.
//Ordering is case insensitive.
case false=> orderby = s"order{$bCondition <=> $aCondition}"//descending
case _ => orderby = s"order{$aCondition <=> $bCondition}"
case false=> s"order{$bCondition <=> $aCondition}"//descending
case _ => s"order{$aCondition <=> $bCondition}"
}
}
else {
val orderbyProperty = genQuery(orderExpression, true);
val aPropertyExpr = gremlin3ToLowerCase("a");
val bPropertyExpr = gremlin3ToLowerCase("b");
orderby = asc match {
//builds a closure comparison function based on provided order by clause in DSL. This will be used to sort the results by gremlin order pipe.
//Ordering is case insensitive.
case false=> s"""order().by({$orderbyProperty'}, { a,b -> $bPropertyExpr <=> $aPropertyExpr })"""
case _ => s"""order().by({$orderbyProperty},{ a,b -> $aPropertyExpr <=> $bPropertyExpr })"""
}
}
s"""${genQuery(child, inSelect)}.$orderby"""
}
case limitOffset@LimitExpression(child, limit, offset) => {
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) {
val totalResultRows = limit.value + offset.value
s"""${genQuery(child, inSelect)} [$offset..<$totalResultRows]"""
}
else {
val totalResultRows = limit.value + offset.value
s"""${genQuery(child, inSelect)}.range($offset,$totalResultRows)"""
}
}
case x => throw new GremlinTranslationException(x, "expression not yet supported")
}
def gremlin3ToLowerCase(varName : String) : String = {
s"""($varName != null ? $varName.toString().toLowerCase() : null)"""
}
def genPropertyAccessExpr(e: Option[Expression], fInfo : FieldInfo, quotedPropertyName: String, inSelect: Boolean) : String = {
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) {
generateAndPrependExpr(e, inSelect, s"""$quotedPropertyName""")
}
else {
val attrInfo : AttributeInfo = fInfo.attrInfo;
val attrType : IDataType[_] = attrInfo.dataType;
if(inSelect) {
val expr = generateAndPrependExpr(e, inSelect, s"""property($quotedPropertyName).orElse(null)""");
return gPersistenceBehavior.generatePersisentToLogicalConversionExpression(expr, attrType);
}
else {
val unmapped = s"""values($quotedPropertyName)"""
val expr = if(gPersistenceBehavior.isPropertyValueConversionNeeded(attrType)) {
val conversionFunction = gPersistenceBehavior.generatePersisentToLogicalConversionExpression(s"""it.get()""", attrType);
s"""$unmapped.map{ $conversionFunction }"""
}
else {
unmapped
}
generateAndPrependExpr(e, inSelect, expr)
}
}
}
def generateAndPrependExpr(e1: Option[Expression], inSelect: Boolean, e2: String) : String = e1 match {
case Some(x) => s"""${genQuery(x, inSelect)}.$e2"""
case None => e2
}
def genHasPredicate(e: Option[Expression], fInfo : FieldInfo, fieldName: String, inSelect: Boolean, c: ComparisonExpression, expr: Expression) : String = {
val qualifiedPropertyName = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}"
val persistentExprValue = translateValueToPersistentForm(fInfo, expr);
if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) {
return generateAndPrependExpr(e, inSelect, s"""has("${qualifiedPropertyName}", ${gPersistenceBehavior.gremlinCompOp(c)}, $persistentExprValue)""");
}
else {
val attrInfo : AttributeInfo = fInfo.attrInfo;
val attrType : IDataType[_] = attrInfo.dataType;
if(gPersistenceBehavior.isPropertyValueConversionNeeded(attrType)) {
//for some types, the logical value cannot be stored directly in the underlying graph,
//and conversion logic is needed to convert the persistent form of the value
//to the actual value. In cases like this, we generate a conversion expression to
//do this conversion and use the filter step to perform the comparsion in the gremlin query
val vertexExpr = "((Vertex)it.get())";
val conversionExpr = gPersistenceBehavior.generatePersisentToLogicalConversionExpression(s"""$vertexExpr.value("$qualifiedPropertyName")""", attrType);
return generateAndPrependExpr(e, inSelect, s"""filter{$vertexExpr.property("$qualifiedPropertyName").isPresent() && $conversionExpr ${gPersistenceBehavior.gremlinPrimitiveOp(c)} $persistentExprValue}""");
}
else {
return generateAndPrependExpr(e, inSelect, s"""has("${qualifiedPropertyName}", ${gPersistenceBehavior.gremlinCompOp(c)}($persistentExprValue))""");
}
}
}
def translateValueToPersistentForm(fInfo: FieldInfo, l: Expression): Any = {
......@@ -376,40 +529,42 @@ class GremlinTranslator(expr: Expression,
}
else if(dataType == DataTypes.DOUBLE_TYPE) {
return s"""${l}d"""
} else if(dataType == DataTypes.STRING_TYPE) {
}
else if(dataType == DataTypes.STRING_TYPE) {
return string(escape(l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE)));
} else {
l
}
else {
return l
}
def generateAndPrependExpr(e1: Option[Expression], inSelect: Boolean, e2: String) : String = e1 match {
case Some(x) => s"""${genQuery(x, inSelect)}.$e2"""
case None => e2
}
def genFullQuery(expr: Expression): String = {
def genFullQuery(expr: Expression, hasSelect: Boolean): String = {
var q = genQuery(expr, false)
val debug:Boolean = false
if(gPersistenceBehavior.addGraphVertexPrefix(preStatements)) {
q = s"g.V.$q"
q = s"g.V()${gPersistenceBehavior.initialQueryCondition}.$q"
}
q = s"$q.toList()"
q = s"$q.toList()${gPersistenceBehavior.getGraph().getOutputTransformationPredicate(hasSelect, expr.isInstanceOf[PathExpression])}"
if(debug) {
println(" query " + q)
}
q = (preStatements ++ Seq(q) ++ postStatements).mkString("", ";", "")
/*
* the L:{} represents a groovy code block; the label is needed
* to distinguish it from a groovy closure.
*/
s"L:{$q}"
}
}
def translate(): GremlinQuery = {
var e1 = expr.transformUp(wrapAndRule)
e1.traverseUp(validateComparisonForm)
e1 = e1.transformUp(new AddAliasToSelectInput)
e1.traverseUp(validateSelectExprHaveOneSrc)
e1 = e1.transformUp(addAliasToLoopInput())
......@@ -419,19 +574,15 @@ class GremlinTranslator(expr: Expression,
//Following code extracts the select expressions from expression tree.
val se = SelectExpressionHelper.extractSelectExpression(e1)
if (se.isDefined)
{
if (se.isDefined) {
val rMap = buildResultMapping(se.get)
GremlinQuery(e1, genFullQuery(e1), rMap)
}
else
{
GremlinQuery(e1, genFullQuery(e1), null)
}
GremlinQuery(e1, genFullQuery(e1, true), rMap)
} else {
GremlinQuery(e1, genFullQuery(e1, false), null)
}
}
}
object SelectExpressionHelper {
/**
* This method extracts the child select expression from parent expression
......
repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala
......@@ -18,21 +18,27 @@
package org.apache.atlas.query
import com.thinkaurelius.titan.core.TitanGraph
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.query.Expressions._
import org.slf4j.{Logger, LoggerFactory}
object QueryProcessor {
val LOG : Logger = LoggerFactory.getLogger("org.apache.atlas.query.QueryProcessor")
def evaluate(e: Expression, g: TitanGraph, gP : GraphPersistenceStrategies = GraphPersistenceStrategy1):
def evaluate(e: Expression, g: AtlasGraph[_,_], gP : GraphPersistenceStrategies = null):
GremlinQueryResult = {
var strategy = gP;
if(strategy == null) {
strategy = GraphPersistenceStrategy1(g);
}
val e1 = validate(e)
val q = new GremlinTranslator(e1, gP).translate()
val q = new GremlinTranslator(e1, strategy).translate()
LOG.debug("Query: " + e1)
LOG.debug("Expression Tree:\n" + e1.treeString)
LOG.debug("Gremlin Query: " + q.queryStr)
new GremlinEvaluator(q, gP, g).evaluate()
new GremlinEvaluator(q, strategy, g).evaluate()
}
def validate(e: Expression): Expression = {
......
repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
......@@ -17,15 +17,15 @@
*/
package org.apache.atlas;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
......@@ -45,11 +45,9 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.annotations.Guice;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
/**
* Base Class to set up hive types and instances for tests
......@@ -63,29 +61,21 @@ public class BaseRepositoryTest {
@Inject
protected MetadataRepository repository;
@Inject
protected GraphProvider<TitanGraph> graphProvider;
protected void setUp() throws Exception {
//force graph initialization / built in type registration
TestUtils.getGraph();
setUpTypes();
new GraphBackedSearchIndexer(graphProvider);
RequestContext.createContext();
new GraphBackedSearchIndexer();
TestUtils.resetRequestContext();
setupInstances();
TestUtils.dumpGraph(graphProvider.get());
TestUtils.dumpGraph(TestUtils.getGraph());
}
protected void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
private void setUpTypes() throws Exception {
......
repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
......@@ -18,14 +18,11 @@
package org.apache.atlas;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.testng.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
/**
* Unit test for Guice injector service loading
*
......@@ -35,12 +32,8 @@ import javax.inject.Inject;
@Guice(modules = RepositoryMetadataModule.class)
public class RepositoryServiceLoadingTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Test
public void testGetGraphService() throws Exception {
Assert.assertNotNull(graphProvider);
Assert.assertNotNull(graphProvider.get());
Assert.assertNotNull(AtlasGraphProvider.getGraphInstance());
}
}
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -18,12 +18,36 @@
package org.apache.atlas;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
......@@ -40,25 +64,18 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
import java.io.File;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
/**
* Test utility class.
......@@ -73,16 +90,30 @@ public final class TestUtils {
/**
* Dumps the graph in GSON format in the path returned.
*
* @param titanGraph handle to graph
* @param graph handle to graph
* @return path to the dump file
* @throws Exception
*/
public static String dumpGraph(TitanGraph titanGraph) throws Exception {
public static String dumpGraph(AtlasGraph<?,?> graph) throws Exception {
File tempFile = File.createTempFile("graph", ".gson");
System.out.println("tempFile.getPath() = " + tempFile.getPath());
GraphSONWriter.outputGraph(titanGraph, tempFile.getPath());
GraphHelper.dumpToLog(graph);
FileOutputStream os = null;
try {
os = new FileOutputStream(tempFile);
graph.exportToGson(os);
}
finally {
if(os != null) {
try {
os.close();
}
catch(IOException e) {
e.printStackTrace();
}
}
}
GraphHelper.dumpToLog(titanGraph);
return tempFile.getPath();
}
......@@ -469,4 +500,44 @@ public final class TestUtils {
}
return null;
}
public static void resetRequestContext() {
RequestContext.createContext();
}
public static void setupGraphProvider(MetadataRepository repo) throws AtlasException {
TypeCache typeCache = null;
try {
typeCache = AtlasRepositoryConfiguration.getTypeCache().newInstance();
}
catch(Throwable t) {
typeCache = new DefaultTypeCache();
}
final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer();
Provider<TypesChangeListener> indexerProvider = new Provider<TypesChangeListener>() {
@Override
public TypesChangeListener get() {
return indexer;
}
};
Configuration config = ApplicationProperties.get();
ITypeStore typeStore = new GraphBackedTypeStore();
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo,
typeStore,
new ReservedTypesRegistrar(),
Collections.singletonList(indexerProvider),
new ArrayList<Provider<EntityChangeListener>>(), TypeSystem.getInstance(), config, typeCache);
//commit the created types
getGraph().commit();
}
public static AtlasGraph getGraph() {
return AtlasGraphProvider.getGraphInstance();
}
}
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -18,7 +18,23 @@
package org.apache.atlas.discovery;
import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.BaseRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
......@@ -28,8 +44,8 @@ import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id;
......@@ -49,21 +65,7 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import com.google.common.collect.ImmutableSet;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
......@@ -75,6 +77,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
private GraphBackedDiscoveryService discoveryService;
private QueryParams queryParams = new QueryParams(40, 0);
@Override
@BeforeClass
public void setUp() throws Exception {
super.setUp();
......@@ -112,11 +115,11 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
}
}
TitanGraphProvider provider = new TitanGraphProvider();
//We need to commit the transaction before creating the indices to release the locks held by the transaction.
//otherwise, the index commit will fail while waiting for the those locks to be released.
provider.get().commit();
GraphBackedSearchIndexer idx = new GraphBackedSearchIndexer(provider);
AtlasGraphProvider.getGraphInstance().commit();
GraphBackedSearchIndexer idx = new GraphBackedSearchIndexer();
idx.onAdd(newTypes);
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
......@@ -18,20 +18,43 @@
package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Vertex;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtils.COLUMN_TYPE;
import static org.apache.atlas.TestUtils.NAME;
import static org.apache.atlas.TestUtils.PII;
import static org.apache.atlas.TestUtils.PROCESS_TYPE;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.apache.atlas.TestUtils.createColumnEntity;
import static org.apache.atlas.TestUtils.createDBEntity;
import static org.apache.atlas.TestUtils.createTableEntity;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasClient.EntityResult;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -61,28 +84,8 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtils.COLUMN_TYPE;
import static org.apache.atlas.TestUtils.NAME;
import static org.apache.atlas.TestUtils.PII;
import static org.apache.atlas.TestUtils.PROCESS_TYPE;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.apache.atlas.TestUtils.createColumnEntity;
import static org.apache.atlas.TestUtils.createDBEntity;
import static org.apache.atlas.TestUtils.createTableEntity;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
/**
* Test for GraphBackedMetadataRepository.deleteEntities
......@@ -93,10 +96,7 @@ import static org.testng.Assert.fail;
@Guice(modules = RepositoryMetadataModule.class)
public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
@Inject
private GraphProvider<TitanGraph> graphProvider;
protected GraphBackedMetadataRepository repositoryService;
protected MetadataRepository repositoryService;
private TypeSystem typeSystem;
......@@ -106,12 +106,50 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
@BeforeClass
public void setUp() throws Exception {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
new GraphBackedSearchIndexer(graphProvider);
new GraphBackedSearchIndexer();
final GraphBackedMetadataRepository delegate = new GraphBackedMetadataRepository(getDeleteHandler(typeSystem));
repositoryService = new GraphBackedMetadataRepository(graphProvider, getDeleteHandler(typeSystem));
repositoryService = (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
new Class[]{MetadataRepository.class}, new InvocationHandler() {
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
boolean useTransaction = GraphBackedMetadataRepository.class.getMethod(
method.getName(), method.getParameterTypes())
.isAnnotationPresent(GraphTransaction.class);
try {
Object result = method.invoke(delegate, args);
if(useTransaction) {
System.out.println("Committing changes");
TestUtils.getGraph().commit();
System.out.println("Commit succeeded.");
}
return result;
}
catch(InvocationTargetException e) {
e.getCause().printStackTrace();
if(useTransaction) {
System.out.println("Rolling back changes due to exception.");
TestUtils.getGraph().rollback();
}
throw e.getCause();
}
catch(Throwable t) {
t.printStackTrace();
if(useTransaction) {
System.out.println("Rolling back changes due to exception.");
TestUtils.getGraph().rollback();
}
throw t;
}
}
});
TestUtils.defineDeptEmployeeTypes(typeSystem);
TestUtils.createHiveTypes(typeSystem);
......@@ -139,22 +177,13 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
@BeforeMethod
public void setupContext() {
RequestContext.createContext();
TestUtils.resetRequestContext();
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
@Test
......@@ -366,7 +395,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
assertVerticesDeleted(getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance"));
}
protected abstract void assertVerticesDeleted(List<Vertex> vertices);
protected abstract void assertVerticesDeleted(List<AtlasVertex> vertices);
@Test
public void testDeleteEntitiesWithCompositeMapReference() throws Exception {
......@@ -388,8 +417,8 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
String edgeLabel = GraphHelper.getEdgeLabel(compositeMapOwnerType, compositeMapOwnerType.fieldMapping.fields.get("map"));
String mapEntryLabel = edgeLabel + "." + "value1";
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel);
Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey());
AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class);
Assert.assertNotNull(object);
List<String> deletedEntities = deleteEntities(mapOwnerGuid).getDeletedEntities();
......@@ -432,11 +461,11 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
ITypedReferenceableInstance max = repositoryService.getEntityDefinition(nameGuidMap.get("Max"));
String maxGuid = max.getId()._getId();
Vertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long creationTimestamp = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY);
AtlasVertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long creationTimestamp = GraphHelper.getSingleValuedProperty(vertex, Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(creationTimestamp);
Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Long modificationTimestampPreUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPreUpdate);
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition(nameGuidMap.get("Jane"));
......@@ -457,7 +486,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPostUpdate);
Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
......@@ -474,7 +503,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Long modificationTimestampPost2ndUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPost2ndUpdate);
Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
......@@ -764,7 +793,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
// Verify MapOwner.map attribute has expected value.
String mapValueGuid = null;
Vertex mapOwnerVertex = null;
AtlasVertex mapOwnerVertex = null;
mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid);
for (String mapAttrName : Arrays.asList("map", "biMap")) {
Object object = mapOwnerInstance.get(mapAttrName);
......@@ -776,7 +805,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
Assert.assertNotNull(mapValueInstance);
mapValueGuid = mapValueInstance.getId()._getId();
mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey());
object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class);
Assert.assertNotNull(object);
}
......@@ -884,8 +913,8 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
String edgeLabel = GraphHelper.getEdgeLabel(mapOwnerType, mapOwnerType.fieldMapping.fields.get("map"));
String mapEntryLabel = edgeLabel + "." + "value1";
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel);
Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey());
AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class);
Assert.assertNotNull(object);
// Verify deleting the target of required map attribute throws a NullRequiredAttributeException.
......@@ -1076,10 +1105,11 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
repositoryService.createEntities(db, table);
}
protected List<Vertex> getVertices(String propertyName, Object value) {
Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value);
List<Vertex> list = new ArrayList<>();
for (Vertex vertex : vertices) {
protected List<AtlasVertex> getVertices(String propertyName, Object value) {
AtlasGraph graph = TestUtils.getGraph();
Iterable<AtlasVertex> vertices = graph.getVertices(propertyName, value);
List<AtlasVertex> list = new ArrayList<>();
for (AtlasVertex vertex : vertices) {
list.add(vertex);
}
return list;
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -18,15 +18,26 @@
package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.inject.Inject;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
......@@ -35,6 +46,12 @@ import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
......@@ -60,26 +77,11 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import scala.actors.threadpool.Arrays;
/**
* GraphBackedMetadataRepository test
......@@ -91,9 +93,6 @@ import static org.testng.Assert.assertTrue;
public class GraphBackedMetadataRepositoryTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
@Inject
......@@ -108,7 +107,7 @@ public class GraphBackedMetadataRepositoryTest {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
new GraphBackedSearchIndexer(graphProvider);
new GraphBackedSearchIndexer();
TestUtils.defineDeptEmployeeTypes(typeSystem);
TestUtils.createHiveTypes(typeSystem);
......@@ -116,23 +115,13 @@ public class GraphBackedMetadataRepositoryTest {
@BeforeMethod
public void setupContext() {
RequestContext.createContext();
TestUtils.resetRequestContext();
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
@Test
......@@ -187,9 +176,10 @@ public class GraphBackedMetadataRepositoryTest {
}
private boolean assertEdge(String id, String typeName) throws Exception {
TitanGraph graph = graphProvider.get();
Vertex vertex = (Vertex)graph.query().has(Constants.GUID_PROPERTY_KEY, id).vertices().iterator().next();
Iterable<Edge> edges = vertex.getEdges(Direction.OUT, Constants.INTERNAL_PROPERTY_KEY_PREFIX + typeName + ".ref");
AtlasGraph graph = TestUtils.getGraph();
Iterable<AtlasVertex> vertices = graph.query().has(Constants.GUID_PROPERTY_KEY, id).vertices();
AtlasVertex AtlasVertex = vertices.iterator().next();
Iterable<AtlasEdge> edges = AtlasVertex.getEdges(AtlasEdgeDirection.OUT, Constants.INTERNAL_PROPERTY_KEY_PREFIX + typeName + ".ref");
if (!edges.iterator().hasNext()) {
ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(id);
assertNotNull(entity.get("ref"));
......@@ -290,9 +280,9 @@ public class GraphBackedMetadataRepositoryTest {
@GraphTransaction
String getGUID() {
Vertex tableVertex = getTableEntityVertex();
AtlasVertex tableVertex = getTableEntityVertex();
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
String guid = GraphHelper.getSingleValuedProperty(tableVertex, Constants.GUID_PROPERTY_KEY, String.class);
if (guid == null) {
Assert.fail();
}
......@@ -300,12 +290,12 @@ public class GraphBackedMetadataRepositoryTest {
}
@GraphTransaction
Vertex getTableEntityVertex() {
TitanGraph graph = graphProvider.get();
GraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TestUtils.TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
AtlasVertex getTableEntityVertex() {
AtlasGraph graph = TestUtils.getGraph();
AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, ComparisionOperator.EQUAL, TestUtils.TABLE_TYPE);
Iterator<AtlasVertex> results = query.vertices().iterator();
// returning one since guid should be unique
Vertex tableVertex = results.hasNext() ? results.next() : null;
AtlasVertex tableVertex = results.hasNext() ? results.next() : null;
if (tableVertex == null) {
Assert.fail();
}
......@@ -365,8 +355,8 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testGetTraitNames")
public void testAddTrait() throws Exception {
final String aGUID = getGUID();
Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
AtlasVertex AtlasVertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPreUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPreUpdate);
List<String> traitNames = repositoryService.getTraitNames(aGUID);
......@@ -388,7 +378,7 @@ public class GraphBackedMetadataRepositoryTest {
// Verify modification timestamp was updated.
GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPostUpdate);
}
......@@ -406,7 +396,7 @@ public class GraphBackedMetadataRepositoryTest {
repositoryService.addTrait(aGUID, traitInstance);
TestUtils.dumpGraph(graphProvider.get());
TestUtils.dumpGraph(TestUtils.getGraph());
// refresh trait names
List<String> traitNames = repositoryService.getTraitNames(aGUID);
......@@ -437,8 +427,8 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testAddTrait")
public void testDeleteTrait() throws Exception {
final String aGUID = getGUID();
Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
AtlasVertex AtlasVertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPreUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPreUpdate);
List<String> traitNames = repositoryService.getTraitNames(aGUID);
......@@ -457,7 +447,7 @@ public class GraphBackedMetadataRepositoryTest {
// Verify modification timestamp was updated.
GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
Assert.assertNotNull(modificationTimestampPostUpdate);
Assert.assertTrue(modificationTimestampPostUpdate > modificationTimestampPreUpdate);
}
......@@ -477,20 +467,20 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testCreateEntity")
public void testGetIdFromVertex() throws Exception {
Vertex tableVertex = getTableEntityVertex();
AtlasVertex tableVertex = getTableEntityVertex();
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
String guid = GraphHelper.getSingleValuedProperty(tableVertex, Constants.GUID_PROPERTY_KEY, String.class);
if (guid == null) {
Assert.fail();
}
Id expected = new Id(guid, tableVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), TestUtils.TABLE_TYPE);
Id expected = new Id(guid, GraphHelper.getSingleValuedProperty(tableVertex, Constants.VERSION_PROPERTY_KEY, Integer.class), TestUtils.TABLE_TYPE);
Assert.assertEquals(GraphHelper.getIdFromVertex(TestUtils.TABLE_TYPE, tableVertex), expected);
}
@Test(dependsOnMethods = "testCreateEntity")
public void testGetTypeName() throws Exception {
Vertex tableVertex = getTableEntityVertex();
AtlasVertex tableVertex = getTableEntityVertex();
Assert.assertEquals(GraphHelper.getTypeName(tableVertex), TestUtils.TABLE_TYPE);
}
......@@ -559,7 +549,7 @@ public class GraphBackedMetadataRepositoryTest {
String dslQuery = "hive_table as t where name = 'bar' "
+ "database where name = 'foo' and description = 'foo database' select t";
TestUtils.dumpGraph(graphProvider.get());
TestUtils.dumpGraph(TestUtils.getGraph());
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery, queryParams);
......@@ -592,7 +582,7 @@ public class GraphBackedMetadataRepositoryTest {
//but with elasticsearch, doesn't work without sleep. why??
long sleepInterval = 1000;
TestUtils.dumpGraph(graphProvider.get());
TestUtils.dumpGraph(TestUtils.getGraph());
//person in hr department whose name is john
Thread.sleep(sleepInterval);
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -99,7 +99,7 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo
}
@Override
protected void assertVerticesDeleted(List<Vertex> vertices) {
protected void assertVerticesDeleted(List<AtlasVertex> vertices) {
assertEquals(vertices.size(), 0);
}
......@@ -171,10 +171,10 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo
assertNull(mapOwnerInstance.get("map"));
assertNull(mapOwnerInstance.get("biMap"));
Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
Object object = mapOwnerVertex.getProperty("MapOwner.map.value1");
AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
Object object = mapOwnerVertex.getProperty("MapOwner.map.value1", String.class);
assertNull(object);
object = mapOwnerVertex.getProperty("MapOwner.biMap.value1");
object = mapOwnerVertex.getProperty("MapOwner.biMap.value1", String.class);
assertNull(object);
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -116,9 +116,9 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo
}
@Override
protected void assertVerticesDeleted(List<Vertex> vertices) {
for (Vertex vertex : vertices) {
assertEquals(vertex.getProperty(Constants.STATE_PROPERTY_KEY), Id.EntityState.DELETED.name());
protected void assertVerticesDeleted(List<AtlasVertex> vertices) {
for (AtlasVertex vertex : vertices) {
assertEquals(GraphHelper.getSingleValuedProperty(vertex, Constants.STATE_PROPERTY_KEY, String.class), Id.EntityState.DELETED.name());
}
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerMockTest.java
......@@ -18,36 +18,33 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import org.apache.atlas.AtlasException;
import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.commons.configuration.Configuration;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
public class GraphBackedSearchIndexerMockTest {
public class GraphBackedSearchIndexerMockTest implements IAtlasGraphProvider {
@Mock
private Configuration configuration;
@Mock
private GraphProvider<TitanGraph> graphProvider;
@Mock
private TitanGraph titanGraph;
private AtlasGraph graph;
@Mock
private TitanManagement titanManagement;
private AtlasGraphManagement management;
@BeforeMethod
public void setup() {
......@@ -57,25 +54,23 @@ public class GraphBackedSearchIndexerMockTest {
@Test
public void testSearchIndicesAreInitializedOnConstructionWhenHAIsDisabled() throws IndexException, RepositoryException {
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY, false)).thenReturn(false);
when(graphProvider.get()).thenReturn(titanGraph);
when(titanGraph.getManagementSystem()).thenReturn(titanManagement);
when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(this, configuration);
verify(titanManagement).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
verify(management).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
}
@Test
public void testSearchIndicesAreNotInitializedOnConstructionWhenHAIsEnabled() throws IndexException, RepositoryException {
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(graphProvider.get()).thenReturn(titanGraph);
when(titanGraph.getManagementSystem()).thenReturn(titanManagement);
when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
new GraphBackedSearchIndexer(graphProvider, configuration);
verifyZeroInteractions(titanManagement);
new GraphBackedSearchIndexer(this, configuration);
verifyZeroInteractions(management);
}
......@@ -83,13 +78,18 @@ public class GraphBackedSearchIndexerMockTest {
public void testIndicesAreReinitializedWhenServerBecomesActive() throws AtlasException {
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
when(graphProvider.get()).thenReturn(titanGraph);
when(titanGraph.getManagementSystem()).thenReturn(titanManagement);
when(titanManagement.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
when(graph.getManagementSystem()).thenReturn(management);
when(management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)).thenReturn(true);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(graphProvider, configuration);
GraphBackedSearchIndexer graphBackedSearchIndexer = new GraphBackedSearchIndexer(this, configuration);
graphBackedSearchIndexer.instanceIsActive();
verify(titanManagement).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
verify(management).containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY);
}
@Override
public AtlasGraph get() {
return graph;
}
}
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
......@@ -18,16 +18,22 @@
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import static junit.framework.Assert.assertTrue;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import java.util.Arrays;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumType;
......@@ -39,45 +45,45 @@ import org.apache.commons.lang.RandomStringUtils;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.Arrays;
import static junit.framework.Assert.assertTrue;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import com.google.inject.Inject;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedSearchIndexerTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedSearchIndexer graphBackedSearchIndexer;
@Test
public void verifySystemMixedIndexes() {
TitanGraph titanGraph = graphProvider.get();
TitanManagement managementSystem = titanGraph.getManagementSystem();
TitanGraphIndex vertexIndex = managementSystem.getGraphIndex(Constants.VERTEX_INDEX);
AtlasGraph graph = TestUtils.getGraph();
AtlasGraphManagement managementSystem = graph.getManagementSystem();
try {
AtlasGraphIndex vertexIndex = managementSystem.getGraphIndex(Constants.VERTEX_INDEX);
assertNotNull(vertexIndex);
assertTrue(vertexIndex.isMixedIndex());
assertTrue(Vertex.class.isAssignableFrom(vertexIndex.getIndexedElement()));
assertFalse(vertexIndex.isEdgeIndex());
assertTrue(vertexIndex.isVertexIndex());
TitanGraphIndex edgeIndex = managementSystem.getGraphIndex(Constants.EDGE_INDEX);
AtlasGraphIndex edgeIndex = managementSystem.getGraphIndex(Constants.EDGE_INDEX);
assertNotNull(edgeIndex);
assertTrue(edgeIndex.isMixedIndex());
assertTrue(Edge.class.isAssignableFrom(edgeIndex.getIndexedElement()));
assertTrue(edgeIndex.isEdgeIndex());
assertFalse(edgeIndex.isVertexIndex());
verifyVertexIndexContains(managementSystem, Constants.STATE_PROPERTY_KEY);
}
finally {
managementSystem.rollback();
}
}
@Test
public void verifySystemCompositeIndexes() {
TitanGraph titanGraph = graphProvider.get();
TitanManagement managementSystem = titanGraph.getManagementSystem();
AtlasGraph graph = TestUtils.getGraph();
AtlasGraphManagement managementSystem = graph.getManagementSystem();
try {
verifySystemCompositeIndex(managementSystem, Constants.GUID_PROPERTY_KEY, true);
verifyVertexIndexContains(managementSystem, Constants.GUID_PROPERTY_KEY);
......@@ -90,36 +96,49 @@ public class GraphBackedSearchIndexerTest {
verifySystemCompositeIndex(managementSystem, Constants.TRAIT_NAMES_PROPERTY_KEY, false);
verifyVertexIndexContains(managementSystem, Constants.TRAIT_NAMES_PROPERTY_KEY);
}
finally {
managementSystem.rollback();
}
}
@Test
public void verifyFullTextIndex() {
TitanGraph titanGraph = graphProvider.get();
TitanManagement managementSystem = titanGraph.getManagementSystem();
TitanGraphIndex fullTextIndex = managementSystem.getGraphIndex(Constants.FULLTEXT_INDEX);
AtlasGraph graph = TestUtils.getGraph();
AtlasGraphManagement managementSystem = graph.getManagementSystem();
try {
AtlasGraphIndex fullTextIndex = managementSystem.getGraphIndex(Constants.FULLTEXT_INDEX);
assertTrue(fullTextIndex.isMixedIndex());
Arrays.asList(fullTextIndex.getFieldKeys()).contains(
managementSystem.getPropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY));
}
finally {
managementSystem.rollback();
}
}
@Test
public void verifyTypeStoreIndexes() {
TitanGraph titanGraph = graphProvider.get();
TitanManagement managementSystem = titanGraph.getManagementSystem();
AtlasGraph graph = TestUtils.getGraph();
AtlasGraphManagement managementSystem = graph.getManagementSystem();
try {
verifySystemCompositeIndex(managementSystem, Constants.TYPENAME_PROPERTY_KEY, true);
verifyVertexIndexContains(managementSystem, Constants.TYPENAME_PROPERTY_KEY);
verifySystemCompositeIndex(managementSystem, Constants.VERTEX_TYPE_PROPERTY_KEY, false);
verifyVertexIndexContains(managementSystem, Constants.VERTEX_TYPE_PROPERTY_KEY);
}
finally {
managementSystem.rollback();
}
}
@Test
public void verifyUserDefinedTypeIndex() throws AtlasException {
TitanGraph titanGraph = graphProvider.get();
TitanManagement managementSystem = titanGraph.getManagementSystem();
AtlasGraph graph = TestUtils.getGraph();
AtlasGraphManagement managementSystem = graph.getManagementSystem();
try {
TypeSystem typeSystem = TypeSystem.getInstance();
String enumName = "randomEnum" + RandomStringUtils.randomAlphanumeric(10);
......@@ -139,15 +158,20 @@ public class GraphBackedSearchIndexerTest {
verifyVertexIndexContains(managementSystem, "Database.managedType");
}
finally {
//search indexer uses its own titan management transaction
managementSystem.rollback();
}
}
private void verifyVertexIndexContains(TitanManagement managementSystem, String indexName) {
TitanGraphIndex vertexIndex = managementSystem.getGraphIndex(Constants.VERTEX_INDEX);
PropertyKey[] fieldKeys = vertexIndex.getFieldKeys();
private void verifyVertexIndexContains(AtlasGraphManagement managementSystem, String indexName) {
AtlasGraphIndex vertexIndex = managementSystem.getGraphIndex(Constants.VERTEX_INDEX);
Set<AtlasPropertyKey> fieldKeys = vertexIndex.getFieldKeys();
Arrays.asList(fieldKeys).contains(managementSystem.getPropertyKey(indexName));
}
private void verifySystemCompositeIndex(TitanManagement managementSystem, String indexName, boolean isUnique) {
TitanGraphIndex systemIndex = managementSystem.getGraphIndex(indexName);
private void verifySystemCompositeIndex(AtlasGraphManagement managementSystem, String indexName, boolean isUnique) {
AtlasGraphIndex systemIndex = managementSystem.getGraphIndex(indexName);
assertNotNull(systemIndex);
assertTrue(systemIndex.isCompositeIndex());
if (isUnique) {
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperMockTest.java
......@@ -17,51 +17,51 @@
*/
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import java.util.Iterator;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.Iterator;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
public class GraphHelperMockTest {
private GraphHelper graphHelperInstance;
private TitanGraph graph;
private AtlasGraph graph;
@BeforeClass
public void setup() {
MockitoAnnotations.initMocks(this);
graph = mock(TitanGraph.class);
graph = mock(AtlasGraph.class);
graphHelperInstance = GraphHelper.getInstance(graph);
}
@Test(expectedExceptions = RepositoryException.class)
public void testGetOrCreateEdgeLabelWithMaxRetries() throws Exception {
final String edgeLabel = "testLabel";
TitanVertex v1 = mock(TitanVertex.class);
TitanVertex v2 = mock(TitanVertex.class);
AtlasVertex v1 = mock(AtlasVertex.class);
AtlasVertex v2 = mock(AtlasVertex.class);
Iterable noEdgesIterable = new Iterable<Edge>() {
Iterable noEdgesIterable = new Iterable<AtlasEdge>() {
@Override
public Iterator<Edge> iterator() {
return new Iterator<Edge>() {
public Iterator<AtlasEdge> iterator() {
return new Iterator<AtlasEdge>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public Edge next() {
public AtlasEdge next() {
return null;
}
......@@ -71,33 +71,33 @@ public class GraphHelperMockTest {
};
}
};
when(v2.getEdges(Direction.IN)).thenReturn(noEdgesIterable);
when(v1.getEdges(Direction.OUT)).thenReturn(noEdgesIterable);
when(v2.getEdges(AtlasEdgeDirection.IN)).thenReturn(noEdgesIterable);
when(v1.getEdges(AtlasEdgeDirection.OUT)).thenReturn(noEdgesIterable);
when(v1.getId()).thenReturn(new String("1234"));
when(v2.getId()).thenReturn(new String("5678"));
when(graph.addEdge(null, v1, v2, edgeLabel)).thenThrow(new RuntimeException("Unique property constraint violated"));
when(graph.addEdge(v1, v2, edgeLabel)).thenThrow(new RuntimeException("Unique property constraint violated"));
graphHelperInstance.getOrCreateEdge(v1, v2, edgeLabel);
}
@Test
public void testGetOrCreateEdgeLabelWithRetries() throws Exception {
final String edgeLabel = "testLabel";
TitanVertex v1 = mock(TitanVertex.class);
TitanVertex v2 = mock(TitanVertex.class);
Edge edge = mock(Edge.class);
AtlasVertex v1 = mock(AtlasVertex.class);
AtlasVertex v2 = mock(AtlasVertex.class);
AtlasEdge edge = mock(AtlasEdge.class);
Iterable noEdgesIterable = new Iterable<Edge>() {
Iterable noEdgesIterable = new Iterable<AtlasEdge>() {
@Override
public Iterator<Edge> iterator() {
return new Iterator<Edge>() {
public Iterator<AtlasEdge> iterator() {
return new Iterator<AtlasEdge>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public Edge next() {
public AtlasEdge next() {
return null;
}
......@@ -107,15 +107,15 @@ public class GraphHelperMockTest {
};
}
};
when(v2.getEdges(Direction.IN)).thenReturn(noEdgesIterable);
when(v1.getEdges(Direction.OUT)).thenReturn(noEdgesIterable);
when(v2.getEdges(AtlasEdgeDirection.IN)).thenReturn(noEdgesIterable);
when(v1.getEdges(AtlasEdgeDirection.OUT)).thenReturn(noEdgesIterable);
when(v1.getId()).thenReturn(new String("v1"));
when(v2.getId()).thenReturn(new String("v2"));
when(edge.getId()).thenReturn(new String("edge"));
when(graph.addEdge(null, v1, v2, edgeLabel))
when(graph.addEdge(v1, v2, edgeLabel))
.thenThrow(new RuntimeException("Unique property constraint violated")).thenReturn(edge);
Edge redge = graphHelperInstance.getOrCreateEdge(v1, v2, edgeLabel);
AtlasEdge redge = graphHelperInstance.getOrCreateEdge(v1, v2, edgeLabel);
assertEquals(edge, redge);
}
}
repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java
......@@ -25,8 +25,8 @@ import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import java.util.HashMap;
import java.util.Iterator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
......@@ -35,6 +35,9 @@ import javax.inject.Inject;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphHelper.VertexInfo;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
......@@ -44,16 +47,9 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphHelperTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@DataProvider(name = "encodeDecodeTestData")
private Object[][] createTestData() {
......@@ -81,25 +77,14 @@ public class GraphHelperTest {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
new GraphBackedSearchIndexer(graphProvider);
new GraphBackedSearchIndexer();
TestUtils.defineDeptEmployeeTypes(typeSystem);
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
public void tearDown() {
AtlasGraphProvider.cleanup();
}
@Test
......@@ -116,7 +101,7 @@ public class GraphHelperTest {
deptGuid = guid;
}
}
Vertex deptVertex = GraphHelper.getInstance().getVertexForGUID(deptGuid);
AtlasVertex deptVertex = GraphHelper.getInstance().getVertexForGUID(deptGuid);
Set<VertexInfo> compositeVertices = GraphHelper.getInstance().getCompositeVertices(deptVertex);
HashMap<String, VertexInfo> verticesByGuid = new HashMap<>();
for (VertexInfo vertexInfo: compositeVertices) {
......@@ -141,14 +126,13 @@ public class GraphHelperTest {
@Test
public void testGetOutgoingEdgesByLabel() throws Exception {
TitanGraph graph = graphProvider.get();
TitanVertex v1 = graph.addVertex();
TitanVertex v2 = graph.addVertex();
v1.addEdge("l1", v2);
v1.addEdge("l2", v2);
AtlasGraph graph = TestUtils.getGraph();
AtlasVertex v1 = graph.addVertex();
AtlasVertex v2 = graph.addVertex();
graph.addEdge(v1, v2, "l1");
graph.addEdge(v1, v2, "l2");
Iterator<Edge> iterator = GraphHelper.getInstance().getOutGoingEdgesByLabel(v1, "l1");
Iterator<AtlasEdge> iterator = GraphHelper.getInstance().getOutGoingEdgesByLabel(v1, "l1");
assertTrue(iterator.hasNext());
assertTrue(iterator.hasNext());
assertNotNull(iterator.next());
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -18,18 +18,23 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Predicate;
import com.tinkerpop.blueprints.Vertex;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import javax.inject.Inject;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
......@@ -45,11 +50,6 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
@Test
@Guice(modules = RepositoryMetadataModule.class)
public class GraphRepoMapperScaleTest {
......@@ -58,9 +58,6 @@ public class GraphRepoMapperScaleTest {
private static final String TABLE_NAME = "bar";
@Inject
GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
@Inject
......@@ -73,6 +70,9 @@ public class GraphRepoMapperScaleTest {
@BeforeClass
@GraphTransaction
public void setUp() throws Exception {
//force up front graph initialization
TestUtils.getGraph();
searchIndexer = new GraphBackedSearchIndexer(new AtlasGraphProvider(), ApplicationProperties.get());
//Make sure we can cleanup the index directory
Collection<IDataType> typesAdded = TestUtils.createHiveTypes(typeSystem);
searchIndexer.onAdd(typesAdded);
......@@ -80,23 +80,13 @@ public class GraphRepoMapperScaleTest {
@BeforeMethod
public void setupContext() {
RequestContext.createContext();
TestUtils.resetRequestContext();
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
@Test
......@@ -130,7 +120,7 @@ public class GraphRepoMapperScaleTest {
searchWithOutIndex("hive_table.name", "bar-999");
searchWithIndex("hive_table.name", "bar-999");
searchWithIndex("hive_table.created", Compare.GREATER_THAN_EQUAL, TestUtils.TEST_DATE_IN_LONG, 1000);
searchWithIndex("hive_table.created", ComparisionOperator.GREATER_THAN_EQUAL, TestUtils.TEST_DATE_IN_LONG, 1000);
for (int index = 500; index < 600; index++) {
searchWithIndex("hive_table.name", "bar-" + index);
......@@ -140,12 +130,13 @@ public class GraphRepoMapperScaleTest {
}
private void searchWithOutIndex(String key, String value) {
TitanGraph graph = graphProvider.get();
AtlasGraph graph = TestUtils.getGraph();
long start = System.currentTimeMillis();
int count = 0;
try {
GraphQuery query = graph.query().has(key, Compare.EQUAL, value);
for (Vertex ignored : query.vertices()) {
AtlasGraphQuery query = graph.query().has(key, ComparisionOperator.EQUAL, value);
Iterable<AtlasVertex> result = query.vertices();
for (AtlasVertex ignored : result) {
count++;
}
} finally {
......@@ -154,14 +145,17 @@ public class GraphRepoMapperScaleTest {
}
}
private void searchWithIndex(String key, String value) {
TitanGraph graph = graphProvider.get();
AtlasGraph graph = TestUtils.getGraph();
long start = System.currentTimeMillis();
int count = 0;
try {
String queryString = "v.\"" + key + "\":(" + value + ")";
TitanIndexQuery query = graph.indexQuery(Constants.VERTEX_INDEX, queryString);
for (TitanIndexQuery.Result<Vertex> ignored : query.vertices()) {
AtlasIndexQuery query = graph.indexQuery(Constants.VERTEX_INDEX, queryString);
Iterator<AtlasIndexQuery.Result> result = query.vertices();
while(result.hasNext()) {
result.next();
count++;
}
} finally {
......@@ -170,13 +164,14 @@ public class GraphRepoMapperScaleTest {
}
}
private void searchWithIndex(String key, Predicate searchPredicate, Object value, int expectedResults) {
TitanGraph graph = graphProvider.get();
private void searchWithIndex(String key, ComparisionOperator op, Object value, int expectedResults) {
AtlasGraph graph = TestUtils.getGraph();
long start = System.currentTimeMillis();
int count = 0;
try {
GraphQuery query = graph.query().has(key, searchPredicate, value);
for (Vertex ignored : query.vertices()) {
AtlasGraphQuery query = graph.query().has(key, op, value);
Iterable<AtlasVertex> itrble = query.vertices();
for (AtlasVertex ignored : itrble) {
count++;
}
} finally {
......
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -18,19 +18,26 @@
package org.apache.atlas.repository.typestore;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
......@@ -53,23 +60,13 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedTypeStoreTest {
private static final String DESCRIPTION = "_description";
@Inject
private GraphProvider<TitanGraph> graphProvider;
private static final String DESCRIPTION = "_description";
@Inject
private ITypeStore typeStore;
......@@ -86,18 +83,10 @@ public class GraphBackedTypeStoreTest {
@AfterClass
public void tearDown() throws Exception {
ts.reset();
try {
graphProvider.get().shutdown();
} catch (Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
@Test
public void testStore() throws AtlasException {
ImmutableList<String> typeNames = ts.getTypeNames();
......@@ -112,10 +101,10 @@ public class GraphBackedTypeStoreTest {
}
private void dumpGraph() {
TitanGraph graph = graphProvider.get();
for (Vertex v : graph.getVertices()) {
AtlasGraph<?, ?> graph = TestUtils.getGraph();
for (AtlasVertex<?,?> v : graph.getVertices()) {
System.out.println("****v = " + GraphHelper.vertexString(v));
for (Edge e : v.getEdges(Direction.OUT)) {
for (AtlasEdge<?,?> e : v.getEdges(AtlasEdgeDirection.OUT)) {
System.out.println("****e = " + GraphHelper.edgeString(e));
}
}
......@@ -215,17 +204,17 @@ public class GraphBackedTypeStoreTest {
// ATLAS-474: verify that type update did not write duplicate edges to the type store.
if (typeStore instanceof GraphBackedTypeStore) {
GraphBackedTypeStore gbTypeStore = (GraphBackedTypeStore) typeStore;
Vertex typeVertex = gbTypeStore.findVertex(TypeCategory.CLASS, "Department");
AtlasVertex typeVertex = gbTypeStore.findVertex(TypeCategory.CLASS, "Department");
int edgeCount = countOutgoingEdges(typeVertex, gbTypeStore.getEdgeLabel("Department", "employees"));
Assert.assertEquals(edgeCount, 1, "Should only be 1 edge for employees attribute on Department type vertex");
Assert.assertEquals(edgeCount, 1, "Should only be 1 edge for employees attribute on Department type AtlasVertex");
}
}
private int countOutgoingEdges(Vertex typeVertex, String edgeLabel) {
private int countOutgoingEdges(AtlasVertex typeVertex, String edgeLabel) {
Iterator<Edge> outGoingEdgesByLabel = GraphHelper.getInstance().getOutGoingEdgesByLabel(typeVertex, edgeLabel);
Iterator<AtlasEdge> outGoingEdgesByLabel = GraphHelper.getInstance().getOutGoingEdgesByLabel(typeVertex, edgeLabel);
int edgeCount = 0;
for (Iterator<Edge> iterator = outGoingEdgesByLabel; iterator.hasNext();) {
for (Iterator<AtlasEdge> iterator = outGoingEdgesByLabel; iterator.hasNext();) {
iterator.next();
edgeCount++;
}
......
repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java
......@@ -17,14 +17,15 @@
*/
package org.apache.atlas.repository.typestore;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import java.util.HashMap;
import java.util.Map;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
......@@ -40,9 +41,8 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.HashMap;
import java.util.Map;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
/**
......@@ -52,9 +52,6 @@ import java.util.Map;
public class StoreBackedTypeCacheTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private ITypeStore typeStore;
@Inject
......@@ -64,8 +61,15 @@ public class StoreBackedTypeCacheTest {
private Map<String, ClassType> classTypesToTest = new HashMap<>();
@Inject
public StoreBackedTypeCacheTest() {
}
@BeforeClass
public void setUp() throws Exception {
//force graph to be initialized up front
TestUtils.getGraph();
ts = TypeSystem.getInstance();
ts.reset();
ts.setTypeCache(typeCache);
......@@ -85,24 +89,12 @@ public class StoreBackedTypeCacheTest {
@AfterClass
public void tearDown() throws Exception {
ts.reset();
try {
graphProvider.get().shutdown();
}
catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
}
catch(Exception e) {
e.printStackTrace();
}
AtlasGraphProvider.cleanup();
}
@BeforeMethod
public void setupTestMethod() throws Exception {
ts.reset();
typeCache.clear();
}
@Test
......
repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.repository.typestore;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.configuration.Configuration;
......@@ -32,7 +33,7 @@ public class StoreBackedTypeCacheTestModule extends RepositoryMetadataModule {
protected Configuration getConfiguration() {
try {
Configuration configuration = ApplicationProperties.get();
configuration.setProperty(RepositoryMetadataModule.TYPE_CACHE_IMPLEMENTATION_PROPERTY,
configuration.setProperty(AtlasRepositoryConfiguration.TYPE_CACHE_IMPLEMENTATION_PROPERTY,
StoreBackedTypeCache.class.getName());
return configuration;
} catch (AtlasException e) {
......
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
......@@ -18,11 +18,30 @@
package org.apache.atlas.service;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtils.COLUMN_TYPE;
import static org.apache.atlas.TestUtils.PII;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.apache.atlas.TestUtils.createColumnEntity;
import static org.apache.atlas.TestUtils.createDBEntity;
import static org.apache.atlas.TestUtils.createInstance;
import static org.apache.atlas.TestUtils.createTableEntity;
import static org.apache.atlas.TestUtils.randomString;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
......@@ -35,11 +54,7 @@ import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.audit.HBaseTestUtils;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.AtlasTypeAttributePatch;
import org.apache.atlas.services.AtlasTypePatch;
import org.apache.atlas.services.AtlasTypePatch.PatchData;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct;
......@@ -72,29 +87,9 @@ import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
import static org.apache.atlas.TestUtils.COLUMN_TYPE;
import static org.apache.atlas.TestUtils.PII;
import static org.apache.atlas.TestUtils.TABLE_TYPE;
import static org.apache.atlas.TestUtils.createColumnEntity;
import static org.apache.atlas.TestUtils.createDBEntity;
import static org.apache.atlas.TestUtils.createInstance;
import static org.apache.atlas.TestUtils.createTableEntity;
import static org.apache.atlas.TestUtils.randomString;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
@Guice(modules = RepositoryMetadataModule.class)
public class DefaultMetadataServiceTest {
......@@ -102,9 +97,6 @@ public class DefaultMetadataServiceTest {
private MetadataService metadataService;
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private EntityAuditRepository auditRepository;
@Inject
......@@ -115,6 +107,7 @@ public class DefaultMetadataServiceTest {
private Referenceable table;
private Id tableId;
private final String NAME = "name";
......@@ -124,7 +117,7 @@ public class DefaultMetadataServiceTest {
HBaseTestUtils.startCluster();
((HBaseBasedAuditRepository) auditRepository).start();
}
RequestContext.createContext();
TestUtils.resetRequestContext();
RequestContext.get().setUser("testuser");
TypesDef typesDef = TestUtils.defineHiveTypes();
......@@ -145,25 +138,18 @@ public class DefaultMetadataServiceTest {
@AfterTest
public void shutdown() throws Exception {
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
TypeSystem.getInstance().reset();
if (auditRepository instanceof HBaseBasedAuditRepository) {
((HBaseBasedAuditRepository) auditRepository).stop();
HBaseTestUtils.stopCluster();
}
}
finally {
AtlasGraphProvider.cleanup();
}
}
private AtlasClient.EntityResult updateInstance(Referenceable entity) throws Exception {
RequestContext.createContext();
ParamChecker.notNull(entity, "Entity");
......@@ -773,7 +759,7 @@ public class DefaultMetadataServiceTest {
@Test
public void testArrayOfStructs() throws Exception {
//Add array of structs
TestUtils.dumpGraph(graphProvider.get());
TestUtils.dumpGraph(TestUtils.getGraph());
final Struct partition1 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
partition1.set(NAME, "part1");
......@@ -1095,43 +1081,6 @@ public class DefaultMetadataServiceTest {
}
@Test
public void testPatchFrameworkForTypeUpdate() throws AtlasException, JSONException {
String typeName = "test_type_" + RandomStringUtils.randomAlphanumeric(10);
HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef(typeName, ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("type_attr1", DataTypes.STRING_TYPE));
TypesDef typesDef = new TypesDef(typeDef, false);
metadataService.createType(TypesSerialization.toJson(typesDef));
AtlasTypeAttributePatch patch = new AtlasTypeAttributePatch((DefaultMetadataService) metadataService, TypeSystem.getInstance());
AttributeDefinition[] attrDefs = new AttributeDefinition[]{
new AttributeDefinition("type_attr2", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("type_attr3", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)};
// Testing add attribute patch
AtlasTypePatch.PatchData addAttributePatch = new PatchData("ADD_ATTRIBUTE", typeName, "1.0", "2.0", null, attrDefs);
TypesDef newAttrTypesDef = patch.updateTypesDef(typesDef, addAttributePatch);
metadataService.updateType(TypesSerialization.toJson(newAttrTypesDef));
TypesDef addedTypesDef = TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
// test added attributes and update version to 2.0
assertEquals(addedTypesDef.classTypes().head().attributeDefinitions.length, 3);
assertEquals(addedTypesDef.classTypes().head().typeVersion, "2.0");
// Testing update attribute patch
AttributeDefinition[] updateAttrDef = new AttributeDefinition[]{
new AttributeDefinition("type_attr1", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)};
AtlasTypePatch.PatchData updateAttributePatch = new PatchData("UPDATE_ATTRIBUTE", typeName, "2.0", "3.0", null, updateAttrDef);
TypesDef updateAttrTypesDef = patch.updateTypesDef(addedTypesDef, updateAttributePatch);
metadataService.updateType(TypesSerialization.toJson(updateAttrTypesDef));
TypesDef updatedTypesDef = TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
// test update attribute to optional and update version to 3.0
assertEquals(updatedTypesDef.classTypes().head().attributeDefinitions[0].multiplicity, Multiplicity.OPTIONAL);
assertEquals(updatedTypesDef.classTypes().head().typeVersion, "3.0");
}
@Test
public void testAuditEventsInvalidParams() throws Exception {
//entity id can't be null
try {
......
repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.service;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule;
......@@ -44,8 +44,6 @@ import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
/**
......@@ -67,9 +65,6 @@ public class StoreBackedTypeCacheMetadataServiceTest
private StoreBackedTypeCache storeBackedTypeCache;
@Inject
private GraphProvider<TitanGraph> graphProvider;
private TypeSystem ts;
@BeforeClass
......@@ -94,20 +89,8 @@ public class StoreBackedTypeCacheMetadataServiceTest
@AfterClass
public void tearDown() throws Exception {
ts.reset();
try {
graphProvider.get().shutdown();
}
catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
}
catch(Exception e) {
e.printStackTrace();
}
TypeSystem.getInstance().reset();
AtlasGraphProvider.cleanup();
}
@Test
......
repository/src/test/java/org/apache/atlas/utils/HiveModel.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.utils;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.atlas.TestUtils;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
/**
* Allows easy creation of entities for classes in the hive test model.
*
*/
public class HiveModel {
public static class StructInstance {
public String getTypeName() {
return getClass().getSimpleName();
}
public Struct toStruct() throws Exception {
Struct entity = new Struct(getTypeName());
addDeclaredFields(getClass(), entity);
return entity;
}
protected void addDeclaredFields(Class clazz, Struct r) throws Exception {
for (Field f : clazz.getDeclaredFields()) {
if (Modifier.isTransient(f.getModifiers())) {
continue;
}
String fieldName = f.getName();
f.setAccessible(true);
Object value = f.get(this);
if (value instanceof List) {
List listValue = (List) value;
List toSet = new ArrayList(listValue.size());
for (Object listItem : listValue) {
Object toAdd = null;
toAdd = convertValue(listItem);
toSet.add(toAdd);
}
r.set(fieldName, toSet);
} else {
Object converted = convertValue(value);
r.set(fieldName, converted);
}
}
if (clazz != StructInstance.class) {
addDeclaredFields(clazz.getSuperclass(), r);
}
}
private Object convertValue(Object toConvert) throws Exception {
if (toConvert instanceof ClassInstance) {
return ((ClassInstance) toConvert).toReferenceable();
}
if (toConvert instanceof StructInstance) {
return ((StructInstance) toConvert).toStruct();
} else {
return toConvert;
}
}
}
public static class ClassInstance<T> extends StructInstance {
private transient final Id guid;
private transient List<String> traits = new ArrayList();
public T withTrait(String name) {
traits.add(name);
return getInstance();
}
public T withTraits(List<String> names) {
traits.addAll(names);
return getInstance();
}
public T getInstance() {
return (T) this;
}
public ClassInstance() {
guid = new Id(getTypeName());
}
public Referenceable toReferenceable() throws Exception {
String[] traitArray = new String[traits.size()];
traitArray = traits.toArray(traitArray);
Referenceable entity = new Referenceable(getTypeName(), traitArray);
entity.replaceWithNewId(guid);
addDeclaredFields(getClass(), entity);
return entity;
}
public List<ITypedReferenceableInstance> getTypedReferencebles() throws Exception {
List<ITypedReferenceableInstance> result = new ArrayList();
for (ClassInstance containedInstance : getAllInstances()) {
Referenceable entity = containedInstance.toReferenceable();
ClassType type = TypeSystem.getInstance().getDataType(ClassType.class, entity.getTypeName());
ITypedReferenceableInstance converted = type.convert(entity, Multiplicity.REQUIRED);
result.add(converted);
}
return result;
}
protected List<ClassInstance> getAllInstances() {
return (List) Collections.singletonList(this);
}
public Id getId() {
return guid;
}
}
public static class NamedInstance<T> extends ClassInstance<T> {
private final String name;
public NamedInstance(String name) {
super();
this.name = name;
}
}
public static class HiveOrder extends StructInstance {
private String col;
private int order;
public HiveOrder(String col, int order) {
super();
this.col = col;
this.order = order;
}
}
public static class DB extends NamedInstance<DB> {
private String owner;
private int createTime;
private String clusterName;
public DB(String name, String owner, int createTime, String clusterName) {
super(name);
this.owner = owner;
this.createTime = createTime;
this.clusterName = clusterName;
}
}
public static class StorageDescriptor extends ClassInstance<StorageDescriptor> {
private String inputFormat;
private String outputFormat;
private List<HiveOrder> sortCols;
public StorageDescriptor(String inputFormat, String outputFormat, List<HiveOrder> sortCols) {
super();
this.inputFormat = inputFormat;
this.outputFormat = outputFormat;
this.sortCols = sortCols;
}
}
public static class Column extends NamedInstance<Column> {
private String type;
private StorageDescriptor sd;
public Column(String name, String type) {
super(name);
this.type = type;
}
public void setStorageDescriptor(StorageDescriptor sd) {
this.sd = sd;
}
}
public static class Table extends NamedInstance<Table> {
private DB db;
private Date created;
private StorageDescriptor sd;
private transient List<Column> colDefs;
public Table(String name, DB db, StorageDescriptor sd, List<Column> colDefs) {
this(name, db, sd, new Date(TestUtils.TEST_DATE_IN_LONG), colDefs);
}
public Table(String name, DB db, StorageDescriptor sd, Date created, List<Column> colDefs) {
super(name);
this.colDefs = colDefs;
this.db = db;
this.sd = sd;
this.created = created;
for (Column col : colDefs) {
col.setStorageDescriptor(sd);
}
}
public List<Column> getColumns() {
return colDefs;
}
@Override
protected List<ClassInstance> getAllInstances() {
List<ClassInstance> result = new ArrayList(colDefs.size() + 2);
result.add(sd);
result.addAll(colDefs);
result.add(this);
return result;
}
}
public static class Partition extends ClassInstance<Partition> {
private List<String> values;
private Table table;
public Partition(List<String> values, Table table) {
super();
this.values = values;
this.table = table;
}
}
public static class LoadProcess extends NamedInstance<LoadProcess> {
private List<Table> inputTables;
private Table outputTable;
public LoadProcess(String name, List<Table> inputTables, Table outputTable) {
super(name);
this.inputTables = inputTables;
this.outputTable = outputTable;
}
}
public static class View extends NamedInstance<View> {
private DB db;
private List<Table> inputTables;
public View(String name, DB db, List<Table> inputTables) {
super(name);
this.db = db;
this.inputTables = inputTables;
}
}
}
repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
......@@ -18,45 +18,50 @@
package org.apache.atlas.query
import com.thinkaurelius.titan.core.TitanGraph
import com.thinkaurelius.titan.core.util.TitanCleanup
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy
import org.apache.atlas.query.Expressions._
import org.apache.atlas.repository.graph.{TitanGraphProvider, GraphBackedMetadataRepository}
import org.apache.atlas.repository.graph.{AtlasGraphProvider, GraphBackedMetadataRepository}
import org.apache.atlas.typesystem.types.TypeSystem
import org.testng.annotations.{Test,BeforeClass,AfterClass}
import org.apache.atlas.repository.graph.AtlasGraphProvider
import org.testng.annotations.BeforeMethod
import org.apache.atlas.TestUtils
class GremlinTest extends BaseGremlinTest {
var g: TitanGraph = null
var g: AtlasGraph[_,_] = null
var gp: GraphPersistenceStrategies = null;
var gProvider: TitanGraphProvider = null;
@BeforeMethod
def resetRequestContext() {
TestUtils.resetRequestContext()
}
@BeforeClass
def beforeAll() {
TypeSystem.getInstance().reset()
QueryTestsUtils.setupTypes
gProvider = new TitanGraphProvider()
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider, null))
g = QueryTestsUtils.setupTestGraph(gProvider)
var repo = new GraphBackedMetadataRepository(null);
TestUtils.setupGraphProvider(repo);
//force graph to be initialized first
AtlasGraphProvider.getGraphInstance();
//create types and indices up front. Without this, some of the property keys (particularly __traitNames and __superTypes)
//get ended up created implicitly with some graph backends with the wrong multiplicity. This also makes the queries
//we execute perform better :-)
QueryTestsUtils.setupTypesAndIndices()
gp = new DefaultGraphPersistenceStrategy(repo)
g = QueryTestsUtils.setupTestGraph(repo)
g
}
@AfterClass
def afterAll() {
try {
g.shutdown()
} catch {
case ex: Exception =>
print("Could not shutdown the graph ", ex);
}
try {
TitanCleanup.clear(g);
} catch {
case ex: Exception =>
print("Could not clear the graph ", ex);
}
AtlasGraphProvider.cleanup();
}
@Test def testClass {
val r = QueryProcessor.evaluate(_class("DB"), g, gp)
validateJson(r, """{
......
repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala
......@@ -18,38 +18,42 @@
package org.apache.atlas.query
import com.thinkaurelius.titan.core.TitanGraph
import com.thinkaurelius.titan.core.util.TitanCleanup
import org.apache.atlas.TestUtils
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy
import org.apache.atlas.query.Expressions._
import org.apache.atlas.repository.graph.{TitanGraphProvider, GraphBackedMetadataRepository}
import org.apache.atlas.query.Expressions._class
import org.apache.atlas.query.Expressions._trait
import org.apache.atlas.query.Expressions.id
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.typesystem.types.TypeSystem
import org.testng.annotations.{Test,BeforeClass,AfterClass}
import org.testng.annotations.AfterClass
import org.testng.annotations.BeforeClass
import org.testng.annotations.BeforeMethod
import org.testng.annotations.Test
import org.apache.atlas.repository.graph.AtlasGraphProvider
class GremlinTest2 extends BaseGremlinTest {
var g: TitanGraph = null
var gProvider:TitanGraphProvider = null;
var g: AtlasGraph[_,_] = null
var gp:GraphPersistenceStrategies = null;
@BeforeMethod
def resetRequestContext() {
TestUtils.resetRequestContext();
}
@BeforeClass
def beforeAll() {
TypeSystem.getInstance().reset()
QueryTestsUtils.setupTypes
gProvider = new TitanGraphProvider();
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider, null))
g = QueryTestsUtils.setupTestGraph(gProvider)
var repo = new GraphBackedMetadataRepository(null);
gp = new DefaultGraphPersistenceStrategy(repo)
g = QueryTestsUtils.setupTestGraph(repo)
}
@AfterClass
def afterAll() {
g.shutdown()
try {
TitanCleanup.clear(g);
} catch {
case ex: Exception =>
print("Could not clear the graph ", ex);
}
AtlasGraphProvider.cleanup();
}
@Test def testTraitSelect {
......@@ -111,7 +115,7 @@ class GremlinTest2 extends BaseGremlinTest {
"LoadProcess",
"inputTables",
"outputTable",
None, Some(List("name")), true, GraphPersistenceStrategy1, g).evaluate()
None, Some(List("name")), true, getPersistenceStrategy(g), g).evaluate()
validateJson(r)
}
......@@ -120,7 +124,7 @@ class GremlinTest2 extends BaseGremlinTest {
"LoadProcess",
"inputTables",
"outputTable",
None, Some(List("name")), true, GraphPersistenceStrategy1, g).graph
None, Some(List("name")), true, getPersistenceStrategy(g), g).graph
println(r.toInstanceJson)
//validateJson(r)
......@@ -131,7 +135,7 @@ class GremlinTest2 extends BaseGremlinTest {
"LoadProcess",
"inputTables",
"outputTable",
None, Some(List("name")), true, GraphPersistenceStrategy1, g).evaluate()
None, Some(List("name")), true, getPersistenceStrategy(g), g).evaluate()
validateJson(r)
}
......@@ -140,8 +144,12 @@ class GremlinTest2 extends BaseGremlinTest {
"LoadProcess",
"inputTables",
"outputTable",
None, Some(List("name")), true, GraphPersistenceStrategy1, g).graph
None, Some(List("name")), true, getPersistenceStrategy(g), g).graph
println(r.toInstanceJson)
}
private def getPersistenceStrategy(g: AtlasGraph[_,_]) : GraphPersistenceStrategies = {
return GraphPersistenceStrategy1(g);
}
}
\ No newline at end of file
repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala 100755 → 100644
......@@ -18,306 +18,155 @@
package org.apache.atlas.query
import java.io.File
import java.util.concurrent.atomic.AtomicInteger
import java.util.{Date, UUID}
import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
import scala.collection.JavaConversions._
import com.thinkaurelius.titan.core.TitanGraph
import org.apache.atlas.repository.Constants
import org.apache.atlas.repository.graph.TitanGraphProvider
import org.apache.atlas.TestUtils
import org.apache.commons.io.FileUtils
import scala.collection.mutable.ArrayBuffer
import org.apache.atlas.typesystem.ITypedReferenceableInstance
import org.apache.atlas.typesystem.json.TypedReferenceableInstanceSerializer
import org.apache.atlas.utils.HiveModel.Column
import org.apache.atlas.utils.HiveModel.DB
import org.apache.atlas.utils.HiveModel.HiveOrder
import org.apache.atlas.utils.HiveModel.LoadProcess
import org.apache.atlas.utils.HiveModel.Partition
import org.apache.atlas.utils.HiveModel.StorageDescriptor
import org.apache.atlas.utils.HiveModel.Table
import org.apache.atlas.utils.HiveModel.View
import scala.collection.mutable.Buffer
object HiveTitanSample {
private var nextVertexId: AtomicInteger = new AtomicInteger(0)
private var nextEdgeId: AtomicInteger = new AtomicInteger(1000)
trait Vertex {
val _id: String
def id = _id
val __version = 0
val __guid = s"""${UUID.randomUUID()}""".stripMargin
def addEdge(to: Vertex, label: String, edges: ArrayBuffer[String]): Int = {
val edgeId = nextEdgeId.incrementAndGet();
edges +=
s"""{"_id" : "${edgeId}", "_type" : "edge", "_inV" : "${to.id}", "_outV" : "$id", "_label" : "$label"}"""
edgeId
}
def toGSon(vertices: ArrayBuffer[String],
edges: ArrayBuffer[String]): Unit = {
val sb = new StringBuilder
sb.append( s"""{"${Constants.ENTITY_TYPE_PROPERTY_KEY}" : "${this.getClass.getSimpleName}", "_type" : "vertex"""")
this.getClass.getDeclaredFields filter (_.getName != "traits") foreach { f =>
f.setAccessible(true)
val fV = f.get(this)
val convertedVal = fV match {
case _: String => s""""$fV""""
case ls: List[_] if isPrimitiveType(ls) =>
s"""["${ls.mkString(",")}"]"""
case d: Date => d.getTime
case _ => fV
}
convertedVal match {
case x: Vertex => addEdge(x, s"__${this.getClass.getSimpleName}.${f.getName}", edges)
case l: List[_] => val edgeList = l.map(x =>
s""""${addEdge(x.asInstanceOf[Vertex], s"__${this.getClass.getSimpleName}.${f.getName}", edges)}""""
)
if(l.head.isInstanceOf[Struct]) {
sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : ${edgeList.mkString("[", ",", "]")}""")
}
case _ => sb.append( s""", "${f.getName}" : $convertedVal""")
sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $convertedVal""")
}
}
this.getClass.getDeclaredFields filter (_.getName == "traits") foreach { f =>
f.setAccessible(true)
var traits = f.get(this).asInstanceOf[Option[List[Trait]]]
if (traits.isDefined) {
val fV = traits.get.map(_.getClass.getSimpleName).mkString(",")
sb.append( s""", "${Constants.TRAIT_NAMES_PROPERTY_KEY}" : "$fV"""")
}
}
sb.append("}")
vertices += sb.toString()
}
def isPrimitiveType(ls: List[_]) : Boolean = {
ls.head match {
case _: String => true
case _: Byte => true
case _: Short => true
case _: Int => true
case _: Long => true
case _: Float => true
case _: Double => true
case _: BigDecimal => true
case _: BigInt => true
case _: Boolean => true
case default => false
}
}
}
trait Trait extends Vertex
trait Struct extends Vertex
trait Instance extends Vertex {
val traits: Option[List[Trait]]
override def toGSon(vertices: ArrayBuffer[String],
edges: ArrayBuffer[String]): Unit = {
super.toGSon(vertices, edges)
if (traits.isDefined) {
traits.get foreach { t =>
t.toGSon(vertices, edges)
addEdge(t, s"${t.getClass.getSimpleName}", edges)
}
}
}
}
object HiveTitanSample {
case class JdbcAccess(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
val MetricTrait = "Metric"
val DimensionTrait = "Dimension"
val ETLTrait = "ETL"
val JdbcAccessTrait = "JdbcAccess"
case class PII(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
val salesDB = new DB("Sales", "John ETL", 1000, "test")
case class Dimension(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
case class Metric(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
case class ETL(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
val salesFact = new Table("sales_fact",
salesDB,
new StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(new HiveOrder("customer_id", 0))),
List(
new Column("time_id", "int"),
new Column("product_id", "int"),
new Column("customer_id", "int"),
new Column("created", "date"),
new Column("sales", "double").withTrait(MetricTrait)
)
);
case class DB(name: String, owner: String, createTime: Int, clusterName: String, traits: Option[List[Trait]] = None,
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance
val productDim = new Table("product_dim",
salesDB,
new StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(new HiveOrder("product_id", 0))),
List(
new Column("product_id", "int"),
new Column("product_name", "string"),
new Column("brand_name", "string")
)
).withTrait(DimensionTrait)
case class HiveOrder(col: String, order: Int,
_id: String = "" + nextVertexId.incrementAndGet()) extends Struct
val timeDim = new Table("time_dim",
salesDB,
new StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(new HiveOrder("time_id", 0))),
List(
new Column("time_id", "int"),
new Column("dayOfYear", "int"),
new Column("weekDay", "string")
)
).withTrait(DimensionTrait)
case class StorageDescriptor(inputFormat: String, outputFormat: String,
sortCols: List[Struct], _id: String = "" + nextVertexId.incrementAndGet()) extends Struct {
val customerDim = new Table("customer_dim",
salesDB,
new StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(new HiveOrder("customer_id", 0))),
List(
new Column("customer_id", "int"),
new Column("name", "int"),
new Column("address", "string").withTrait("PII")
)
).withTrait(DimensionTrait)
override def toGSon(vertices: ArrayBuffer[String],
edges: ArrayBuffer[String]): Unit = {
sortCols.foreach(_.toGSon(vertices, edges))
super.toGSon(vertices, edges)
}
}
case class Column(name: String, dataType: String, sd: StorageDescriptor,
traits: Option[List[Trait]] = None,
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance
case class Table(name: String, db: DB, sd: StorageDescriptor,
created: Date,
traits: Option[List[Trait]] = None,
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance
case class TableDef(name: String, db: DB, sd: StorageDescriptor,
columns: List[(String, String, Option[List[Trait]])],
traits: Option[List[Trait]] = None,
created: Option[Date] = None) {
val createdDate : Date = created match {
case Some(x) => x
case None => new Date(TestUtils.TEST_DATE_IN_LONG)
}
val reportingDB = new DB("Reporting", "Jane BI", 1500, "test")
val salesFactDaily = new Table("sales_fact_daily_mv",
reportingDB,
new StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(new HiveOrder("customer_id", 0))),
List(
new Column("time_id", "int"),
new Column("product_id", "int"),
new Column("customer_id", "int"),
new Column("sales", "double").withTrait(MetricTrait)
)
)
val colDefs = columns map { c =>
Column(c._1, c._2, sd, c._3)
}
val tablDef = Table(name, db, sd, createdDate, traits)
val loadSalesFactDaily = new LoadProcess(
"loadSalesDaily",
List(salesFact, timeDim),
salesFactDaily
).withTrait(ETLTrait)
def toGSon(vertices: ArrayBuffer[String],
edges: ArrayBuffer[String]): Unit = {
sd.toGSon(vertices, edges)
colDefs foreach {
_.toGSon(vertices, edges)
}
tablDef.toGSon(vertices, edges)
}
}
case class Partition(values: List[String], table: Table, traits: Option[List[Trait]] = None,
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance
case class LoadProcess(name: String, inputTables: List[Vertex],
outputTable: Vertex,
traits: Option[List[Trait]] = None,
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance
val productDimView = new View(
"product_dim_view",
reportingDB,
List(productDim)
).withTraits(List(DimensionTrait, JdbcAccessTrait))
case class View(name: String, db: DB, inputTables: List[Vertex],
traits: Option[List[Trait]] = None,
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance
val customerDimView = new View(
"customer_dim_view",
reportingDB,
List(customerDim)
val salesDB = DB("Sales", "John ETL", 1000, "test")
val salesFact = TableDef("sales_fact",
salesDB,
StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(HiveOrder("customer_id", 0))),
List(
("time_id", "int", None),
("product_id", "int", None),
("customer_id", "int", None),
("created", "date", None),
("sales", "double", Some(List(Metric())))
))
val productDim = TableDef("product_dim",
salesDB,
StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(HiveOrder("product_id", 0))),
List(
("product_id", "int", None),
("product_name", "string", None),
("brand_name", "string", None)
),
Some(List(Dimension())))
val timeDim = TableDef("time_dim",
salesDB,
StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(HiveOrder("time_id", 0))),
List(
("time_id", "int", None),
("dayOfYear", "int", None),
("weekDay", "string", None)
),
Some(List(Dimension())))
val customerDim = TableDef("customer_dim",
salesDB,
StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(HiveOrder("customer_id", 0))),
List(
("customer_id", "int", None),
("name", "int", None),
("address", "string", Some(List(PII())))
),
Some(List(Dimension())))
).withTraits(List(DimensionTrait, JdbcAccessTrait))
val reportingDB = DB("Reporting", "Jane BI", 1500, "test")
val salesFactDaily = TableDef("sales_fact_daily_mv",
val salesFactMonthly = new Table("sales_fact_monthly_mv",
reportingDB,
StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(HiveOrder("customer_id", 0))),
new StorageDescriptor(
"TextInputFormat",
"TextOutputFormat",
List(new HiveOrder("customer_id", 0))
),
List(
("time_id", "int", None),
("product_id", "int", None),
("customer_id", "int", None),
("sales", "double", Some(List(Metric())))
))
val loadSalesFactDaily = LoadProcess("loadSalesDaily",
List(salesFact.tablDef, timeDim.tablDef), salesFactDaily.tablDef,
Some(List(ETL())))
new Column("time_id", "int"),
new Column("product_id", "int"),
new Column("customer_id", "int"),
new Column("sales", "double").withTrait(MetricTrait)
)
)
val loadSalesFactMonthly = new LoadProcess("loadSalesMonthly",
List(salesFactDaily), salesFactMonthly).withTraits(List(ETLTrait))
val salesDailyPartition = new Partition(List("2015-01-01"), salesFactDaily)
val productDimView = View("product_dim_view", reportingDB,
List(productDim.tablDef),
Some(List(Dimension(), JdbcAccess())))
import scala.collection.JavaConversions._
val customerDimView = View("customer_dim_view", reportingDB,
List(customerDim.tablDef),
Some(List(Dimension(), JdbcAccess())))
def getEntitiesToCreate() : Buffer[ITypedReferenceableInstance] = {
var list = salesDB.getTypedReferencebles() ++
salesFact.getTypedReferencebles() ++
productDim.getTypedReferencebles() ++
timeDim.getTypedReferencebles() ++
customerDim.getTypedReferencebles() ++
reportingDB.getTypedReferencebles() ++
salesFactDaily.getTypedReferencebles() ++
loadSalesFactDaily.getTypedReferencebles() ++
productDimView.getTypedReferencebles() ++
customerDimView.getTypedReferencebles() ++
salesFactMonthly.getTypedReferencebles() ++
loadSalesFactMonthly.getTypedReferencebles() ++
salesDailyPartition.getTypedReferencebles();
return list;
val salesFactMonthly = TableDef("sales_fact_monthly_mv",
reportingDB,
StorageDescriptor("TextInputFormat",
"TextOutputFormat", List(HiveOrder("customer_id", 0))),
List(
("time_id", "int", None),
("product_id", "int", None),
("customer_id", "int", None),
("sales", "double", Some(List(Metric())))
))
val loadSalesFactMonthly = LoadProcess("loadSalesMonthly",
List(salesFactDaily.tablDef), salesFactMonthly.tablDef,
Some(List(ETL())))
val salesDailyPartition = Partition(List("2015-01-01"),salesFactDaily.tablDef)
val vertices: ArrayBuffer[String] = new ArrayBuffer[String]()
val edges: ArrayBuffer[String] = new ArrayBuffer[String]()
salesDB.toGSon(vertices, edges)
salesFact.toGSon(vertices, edges)
productDim.toGSon(vertices, edges)
timeDim.toGSon(vertices, edges)
customerDim.toGSon(vertices, edges)
reportingDB.toGSon(vertices, edges)
salesFactDaily.toGSon(vertices, edges)
loadSalesFactDaily.toGSon(vertices, edges)
productDimView.toGSon(vertices, edges)
customerDimView.toGSon(vertices, edges)
salesFactMonthly.toGSon(vertices, edges)
loadSalesFactMonthly.toGSon(vertices, edges)
salesDailyPartition.toGSon(vertices, edges)
def toGSon(): String = {
s"""{
"mode":"NORMAL",
"vertices": ${vertices.mkString("[\n\t", ",\n\t", "\n]")},
"edges": ${edges.mkString("[\n\t", ",\n\t", "\n]")}
}
""".stripMargin
}
def writeGson(fileName: String): Unit = {
FileUtils.writeStringToFile(new File(fileName), toGSon())
}
val GremlinQueries = List(
......@@ -367,28 +216,28 @@ object HiveTitanSample {
)
}
object TestApp extends App with GraphUtils {
val g: TitanGraph = TitanGraphProvider.getGraphInstance
val manager: ScriptEngineManager = new ScriptEngineManager
val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
val bindings: Bindings = engine.createBindings
bindings.put("g", g)
val hiveGraphFile = FileUtils.getTempDirectory().getPath + File.separator + System.nanoTime() + ".gson"
HiveTitanSample.writeGson(hiveGraphFile)
bindings.put("hiveGraphFile", hiveGraphFile)
try {
engine.eval("g.loadGraphSON(hiveGraphFile)", bindings)
println(engine.eval("g.V.typeName.toList()", bindings))
HiveTitanSample.GremlinQueries.foreach { q =>
println(q)
println("Result: " + engine.eval(q + ".toList()", bindings))
}
} finally {
g.shutdown()
}
}
\ No newline at end of file
//object TestApp extends App with GraphUtils {
//
// val g: TitanGraph = TitanGraphProvider.getGraphInstance
// val manager: ScriptEngineManager = new ScriptEngineManager
// val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
// val bindings: Bindings = engine.createBindings
// bindings.put("g", g)
//
// val hiveGraphFile = FileUtils.getTempDirectory().getPath + File.separator + System.nanoTime() + ".gson"
// HiveTitanSample.writeGson(hiveGraphFile)
// bindings.put("hiveGraphFile", hiveGraphFile)
//
// try {
// engine.eval("g.loadGraphSON(hiveGraphFile)", bindings)
//
// println(engine.eval("g.V.typeName.toList()", bindings))
//
// HiveTitanSample.GremlinQueries.foreach { q =>
// println(q)
// println("Result: " + engine.eval(q + ".toList()", bindings))
// }
// } finally {
// g.shutdown()
// }
//}
\ No newline at end of file
repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
......@@ -18,43 +18,51 @@
package org.apache.atlas.query
import com.thinkaurelius.titan.core.TitanGraph
import com.thinkaurelius.titan.core.util.TitanCleanup
import org.apache.atlas.TestUtils
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy
import org.apache.atlas.query.Expressions._
import org.apache.atlas.repository.graph.{GraphBackedMetadataRepository, TitanGraphProvider}
import org.apache.atlas.query.Expressions._class
import org.apache.atlas.query.Expressions.id
import org.apache.atlas.query.Expressions.int
import org.apache.atlas.repository.graph.AtlasGraphProvider
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.typesystem.types.TypeSystem
import org.testng.annotations.{Test,BeforeClass,AfterClass}
import org.testng.annotations.AfterClass
import org.testng.annotations.BeforeClass
import org.testng.annotations.BeforeMethod
import org.testng.annotations.Test
class LineageQueryTest extends BaseGremlinTest {
var g: TitanGraph = null
var gProvider:TitanGraphProvider = null;
var g: AtlasGraph[_,_] = null
var gp:GraphPersistenceStrategies = null;
@BeforeMethod
def resetRequestContext() {
TestUtils.resetRequestContext()
}
@BeforeClass
def beforeAll() {
TypeSystem.getInstance().reset()
QueryTestsUtils.setupTypes
gProvider = new TitanGraphProvider();
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider, null))
g = QueryTestsUtils.setupTestGraph(gProvider)
var repo = new GraphBackedMetadataRepository(null);
TestUtils.setupGraphProvider(repo);
//force graph to be initialized first
AtlasGraphProvider.getGraphInstance();
//create types and indices up front. Without this, some of the property keys (particularly __traitNames and __superTypes)
//get ended up created implicitly with some graph backends with the wrong multiplicity. This also makes the queries
//we execute perform better :-)
QueryTestsUtils.setupTypesAndIndices()
gp = new DefaultGraphPersistenceStrategy(repo);
g = QueryTestsUtils.setupTestGraph(repo)
}
@AfterClass
def afterAll() {
try {
g.shutdown()
} catch {
case ex: Exception =>
print("Could not shutdown the graph ", ex);
}
try {
TitanCleanup.clear(g);
} catch {
case ex: Exception =>
print("Could not clear the graph ", ex);
}
AtlasGraphProvider.cleanup()
}
val PREFIX_SPACES_REGEX = ("\\n\\s*").r
......
repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
......@@ -22,10 +22,8 @@ import java.io.File
import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
import com.google.common.collect.ImmutableList
import com.thinkaurelius.titan.core.{TitanFactory, TitanGraph}
import com.tinkerpop.blueprints.Vertex
import org.apache.atlas.repository.graphdb.AtlasVertex
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.atlas.repository.graph.TitanGraphProvider
import org.apache.atlas.typesystem.types._
import org.apache.commons.configuration.{Configuration, ConfigurationException, MapConfiguration}
import org.apache.commons.io.FileUtils
......@@ -34,6 +32,13 @@ import org.json.JSONObject
import org.skyscreamer.jsonassert.JSONAssert
import scala.util.Random
import org.apache.atlas.repository.MetadataRepository
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.repository.graph.AtlasGraphProvider
import java.net.URL
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer
import org.apache.atlas.typesystem.TypesDef
import org.apache.atlas.typesystem.ITypedReferenceableInstance
trait GraphUtils {
......@@ -52,12 +57,12 @@ trait GraphUtils {
}
def titanGraph(conf: Configuration) = {
def graph(conf: Configuration) = {
try {
val g = TitanFactory.open(conf)
val g = AtlasGraphProvider.getGraphInstance
val mgmt = g.getManagementSystem
val typname = mgmt.makePropertyKey("typeName").dataType(classOf[String]).make()
mgmt.buildIndex("byTypeName", classOf[Vertex]).addKey(typname).buildCompositeIndex()
val typname = mgmt.makePropertyKey("typeName", classOf[String], null);
mgmt.createExactMatchIndex("byTypeName", false, List(typname));
mgmt.commit()
g
} catch {
......@@ -68,7 +73,21 @@ trait GraphUtils {
object QueryTestsUtils extends GraphUtils {
def setupTypesAndIndices() : Unit = {
val indexer = new GraphBackedSearchIndexer();
val typesDef : TypesDef = defineTypes;
val newTypes = TypeSystem.getInstance.defineTypes(typesDef);
indexer.onAdd(newTypes.values());
}
def setupTypes: Unit = {
val types : TypesDef = defineTypes;
TypeSystem.getInstance.defineTypes(types);
}
def defineTypes: TypesDef = {
def attrDef(name: String, dT: IDataType[_],
m: Multiplicity = Multiplicity.OPTIONAL,
isComposite: Boolean = false,
......@@ -144,36 +163,29 @@ object QueryTestsUtils extends GraphUtils {
def jdbcTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "JdbcAccess", null, null,
Array[AttributeDefinition]())
TypeSystem.getInstance().defineTypes(ImmutableList.of[EnumTypeDefinition],
ImmutableList.of[StructTypeDefinition](hiveOrderDef),
ImmutableList.of[HierarchicalTypeDefinition[TraitType]](dimTraitDef, piiTraitDef,
TypesDef(Seq[EnumTypeDefinition](),
Seq[StructTypeDefinition](hiveOrderDef),
Seq[HierarchicalTypeDefinition[TraitType]](dimTraitDef, piiTraitDef,
metricTraitDef, etlTraitDef, jdbcTraitDef),
ImmutableList.of[HierarchicalTypeDefinition[ClassType]](dbClsDef, storageDescClsDef, columnClsDef, tblClsDef,
Seq[HierarchicalTypeDefinition[ClassType]](dbClsDef, storageDescClsDef, columnClsDef, tblClsDef,
partitionClsDef, loadProcessClsDef, viewClsDef))
()
}
def setupTestGraph(gp: TitanGraphProvider): TitanGraph = {
var conf = TitanGraphProvider.getConfiguration
conf.setProperty("storage.directory",
conf.getString("storage.directory") + "/../graph-data/" + RandomStringUtils.randomAlphanumeric(10))
val g = TitanFactory.open(conf)
val manager: ScriptEngineManager = new ScriptEngineManager
val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
val bindings: Bindings = engine.createBindings
bindings.put("g", g)
val hiveGraphFile = FileUtils.getTempDirectory().getPath + File.separator + System.nanoTime() + ".gson"
HiveTitanSample.writeGson(hiveGraphFile)
bindings.put("hiveGraphFile", hiveGraphFile)
engine.eval("g.loadGraphSON(hiveGraphFile)", bindings)
def setupTestGraph(repo : MetadataRepository): AtlasGraph[_,_] = {
val g = AtlasGraphProvider.getGraphInstance();
val entities = HiveTitanSample.getEntitiesToCreate();
repo.createEntities(entities:_*)
g.commit();
g
}
}
trait BaseGremlinTest {
val STRUCT_NAME_REGEX = (TypeUtils.TEMP_STRUCT_NAME_PREFIX + "\\d+").r
def validateJson(r: GremlinQueryResult, expected: String = null): Unit = {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
......@@ -41,6 +41,11 @@ public final class Multiplicity {
this.isUnique = isUnique;
}
public boolean isMany() {
return upper > 1;
}
public boolean nullAllowed() {
return lower == 0;
}
......
typesystem/src/test/resources/atlas-application.properties
......@@ -19,6 +19,8 @@
#system property
atlas.data=${sys:user.dir}/target/data
#re-use existing property
atlas.graph.data=${atlas.data}/graph
......@@ -30,10 +32,17 @@ atlas.db=${atlasdb}
atlas.TypeSystem.impl=org.apache.atlas.typesystem.types.TypeSystem
######### Atlas Server Configs #########
atlas.rest.address=http://localhost:31000
######### Graph Database Configs #########
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=${graphdb.backend.impl}
# Graph Storage
atlas.graph.storage.backend=${titan.storage.backend}
......
webapp/pom.xml
......@@ -35,6 +35,7 @@
<projectBaseDir>${project.basedir}/..</projectBaseDir>
<debug.jetty.daemon>true</debug.jetty.daemon>
<packages.to.exclude />
<log4j.configuration.url>file://${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</log4j.configuration.url>
</properties>
<profiles>
......@@ -85,6 +86,18 @@
<packages.to.exclude>WEB-INF/lib/je-*.jar</packages.to.exclude>
</properties>
</profile>
<profile>
<id>Windows</id>
<activation>
<os>
<family>windows</family>
</os>
</activation>
<properties>
<log4j.configuration.url>file:/${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</log4j.configuration.url>
</properties>
</profile>
</profiles>
<dependencies>
......@@ -105,11 +118,6 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-titan0</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId>
</dependency>
......@@ -148,6 +156,12 @@
<artifactId>atlas-catalog</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-impls</artifactId>
<type>pom</type>
</dependency>
<!-- supports simple auth handler -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
......@@ -455,7 +469,7 @@
<systemProperties>
<systemProperty>
<name>log4j.configuration</name>
<value>file://${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</value>
<value>${log4j.configuration.url}</value>
</systemProperty>
<systemProperty>
<name>atlas.log.file</name>
......
webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java
......@@ -18,26 +18,19 @@
package org.apache.atlas.web.listeners;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
import com.google.inject.Provider;
import com.google.inject.Stage;
import com.google.inject.TypeLiteral;
import com.google.inject.servlet.GuiceServletContextListener;
import com.sun.jersey.api.core.PackagesResourceConfig;
import com.sun.jersey.guice.JerseyServletModule;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletContextEvent;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.notification.NotificationModule;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.service.Services;
import org.apache.atlas.web.filters.ActiveServerFilter;
import org.apache.atlas.web.filters.AuditFilter;
......@@ -48,9 +41,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.bridge.SLF4JBridgeHandler;
import javax.servlet.ServletContextEvent;
import java.util.HashMap;
import java.util.Map;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.Stage;
import com.google.inject.servlet.GuiceServletContextListener;
import com.sun.jersey.api.core.PackagesResourceConfig;
import com.sun.jersey.guice.JerseyServletModule;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
public class GuiceServletConfig extends GuiceServletContextListener {
......@@ -160,9 +158,7 @@ public class GuiceServletConfig extends GuiceServletContextListener {
//stop services
stopServices();
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
final Graph graph = graphProvider.get().get();
final AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
try {
graph.shutdown();
......
webapp/src/main/java/org/apache/atlas/web/resources/BaseService.java
......@@ -18,22 +18,31 @@
package org.apache.atlas.web.resources;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import org.apache.atlas.catalog.*;
import org.apache.atlas.catalog.exception.*;
import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.UriInfo;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.Map;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.UriInfo;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.atlas.catalog.JsonSerializer;
import org.apache.atlas.catalog.Request;
import org.apache.atlas.catalog.ResourceProvider;
import org.apache.atlas.catalog.Result;
import org.apache.atlas.catalog.exception.CatalogException;
import org.apache.atlas.catalog.exception.CatalogRuntimeException;
import org.apache.atlas.catalog.exception.InvalidPayloadException;
import org.apache.atlas.catalog.exception.InvalidQueryException;
import org.apache.atlas.catalog.exception.ResourceNotFoundException;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
/**
* Base class for all v1 API services.
*/
......@@ -135,7 +144,7 @@ public abstract class BaseService {
//todo: abstract via AtlasTypeSystem
// ensure that the thread wasn't re-pooled with an existing transaction
protected void initializeGraphTransaction() {
TitanGraphProvider.getGraphInstance().rollback();
AtlasGraphProvider.getGraphInstance().rollback();
}
private RuntimeException wrapRuntimeException(RuntimeException e) {
......
webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java
......@@ -56,6 +56,7 @@ public class NotificationHookConsumerKafkaTest {
@Test
public void testConsumerConsumesNewMessageWithAutoCommitDisabled() throws AtlasException, InterruptedException {
try {
produceMessage(new HookNotification.EntityCreateRequest("test_user1", createEntity()));
NotificationConsumer<HookNotification.HookNotificationMessage> consumer =
......@@ -73,12 +74,15 @@ public class NotificationHookConsumerKafkaTest {
produceMessage(new HookNotification.EntityCreateRequest("test_user2", createEntity()));
consumeOneMessage(consumer, hookConsumer);
verify(localAtlasClient).setUser("test_user2");
}
finally {
kafkaNotification.close();
}
}
@Test(dependsOnMethods = "testConsumerConsumesNewMessageWithAutoCommitDisabled")
public void testConsumerRemainsAtSameMessageWithAutoCommitEnabled() throws Exception {
try {
produceMessage(new HookNotification.EntityCreateRequest("test_user3", createEntity()));
NotificationConsumer<HookNotification.HookNotificationMessage> consumer =
......@@ -97,9 +101,11 @@ public class NotificationHookConsumerKafkaTest {
consumeOneMessage(consumer, hookConsumer);
verify(localAtlasClient).setUser("test_user3");
}
finally {
kafkaNotification.close();
}
}
NotificationConsumer<HookNotification.HookNotificationMessage> createNewConsumer(
KafkaNotification kafkaNotification, boolean autoCommitEnabled) {
......@@ -126,6 +132,7 @@ public class NotificationHookConsumerKafkaTest {
final Referenceable entity = new Referenceable(AtlasClient.DATA_SET_SUPER_TYPE);
entity.set("name", "db" + randomString());
entity.set("description", randomString());
entity.set("qualifiedName", randomString());
return entity;
}
......
webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java
......@@ -16,20 +16,18 @@
*/
package org.apache.atlas.web.listeners;
import com.google.inject.Key;
import com.google.inject.Module;
import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import javax.servlet.ServletContextEvent;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContextEvent;
import com.google.inject.Module;
import com.thinkaurelius.titan.core.util.TitanCleanup;
public class TestGuiceServletConfig extends GuiceServletConfig {
......@@ -47,13 +45,11 @@ public class TestGuiceServletConfig extends GuiceServletConfig {
super.contextDestroyed(servletContextEvent);
if(injector != null) {
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
TitanGraph graph = graphProvider.get().get();
AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
LOG.info("Clearing graph store");
try {
TitanCleanup.clear(graph);
AtlasGraphProvider.cleanup();
} catch (Exception e) {
LOG.warn("Clearing graph store failed ", e);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment