ATLAS-2420: Create indexes for relationship attributes

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -29,7 +29,9 @@ public final class Constants {
      */
 
     public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__";
+    public static final String RELATIONSHIP_PROPERTY_KEY_PREFIX = "_r";
     public static final String GUID_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "guid";
+    public static final String RELATIONSHIP_GUID_PROPERTY_KEY = RELATIONSHIP_PROPERTY_KEY_PREFIX + GUID_PROPERTY_KEY;
 
     /**
      * Entity type name property key.

graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java

@@ -35,15 +35,6 @@ public interface AtlasGraphManagement {
     boolean containsPropertyKey(String key);
 
     /**
-     * Creates a full text index for the given property.
-     *
-     * @param  indexName the name of the index to create
-     * @param propertyKey full text property to index
-     * @param backingIndex the name of the backing index to use
-     */
-    void createFullTextIndex(String indexName, AtlasPropertyKey propertyKey, String backingIndex);
-
-    /**
      * Rolls back the changes that have been made to the management system.
      */
     void rollback();
@@ -87,13 +78,22 @@ public interface AtlasGraphManagement {
     AtlasEdgeLabel getEdgeLabel(String label);
 
     /**
-     * Creates a composite index for the graph.
+     * Creates a composite vertex index for the graph.
+     *
+     * @param propertyName
+     * @param isUnique
+     * @param propertyKeys
+     */
+    void createVertexCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys);
+
+    /**
+     * Creates a composite edge index for the graph.
      *
      * @param propertyName
      * @param isUnique
      * @param propertyKeys
      */
-    void createExactMatchIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys);
+    void createEdgeCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys);
 
     /**
      * Looks up the index with the specified name in the graph.  Returns null if
@@ -109,23 +109,33 @@ public interface AtlasGraphManagement {
      *
      * @param name the name of the index to create
      * @param backingIndex the name of the backing index to use
+     * @param propertyKeys list of propertyKeys to be added to the index
      */
-    void createVertexIndex(String name, String backingIndex, List<AtlasPropertyKey> propertyKeys);
+    void createVertexMixedIndex(String name, String backingIndex, List<AtlasPropertyKey> propertyKeys);
 
     /**
-     * Adds a property key to the given index in the graph.
+     * Creates a mixed Edge index for the graph.
      *
-     * @param vertexIndex
-     * @param propertyKey
+     * @param index the name of the index to create
+     * @param backingIndex the name of the backing index to use
+     * @param propertyKeys list of propertyKeys to be added to the index
      */
-    void addVertexIndexKey(String vertexIndex, AtlasPropertyKey propertyKey);
+    void createEdgeMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys);
 
     /**
-     * Creates a mixed Edge index for the graph.
+     * Creates a full text index for the given property.
      *
      * @param index the name of the index to create
      * @param backingIndex the name of the backing index to use
+     * @param propertyKeys list of propertyKeys to be added to the index
      */
-    void createEdgeIndex(String index, String backingIndex);
+    void createFullTextMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys);
 
+    /**
+     * Adds a property key to the given index in the graph.
+     *
+     * @param vertexIndex
+     * @param propertyKey
+     */
+    void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
 }

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java

@@ -25,6 +25,7 @@ import org.janusgraph.core.schema.Mapping;
 import org.janusgraph.core.schema.PropertyKeyMaker;
 import org.janusgraph.core.schema.JanusGraphIndex;
 import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.JanusGraphManagement.IndexBuilder;
 import org.janusgraph.graphdb.internal.Token;
 import org.apache.atlas.repository.graphdb.AtlasCardinality;
 import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
@@ -33,7 +34,6 @@ import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
 import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
 import org.apache.commons.lang.StringUtils;
 import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Element;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,50 +46,52 @@ import java.util.Set;
  * Janus implementation of AtlasGraphManagement.
  */
 public class AtlasJanusGraphManagement implements AtlasGraphManagement {
-
-    private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphManagement.class);
-
+    private static final Logger LOG            = LoggerFactory.getLogger(AtlasJanusGraphManagement.class);
     private static final char[] RESERVED_CHARS = { '{', '}', '"', '$', Token.SEPARATOR_CHAR };
 
-    private AtlasJanusGraph graph;
+    private AtlasJanusGraph      graph;
     private JanusGraphManagement management;
-
-    private Set<String> newMultProperties = new HashSet<>();
+    private Set<String>          newMultProperties = new HashSet<>();
 
     public AtlasJanusGraphManagement(AtlasJanusGraph graph, JanusGraphManagement managementSystem) {
         this.management = managementSystem;
-        this.graph = graph;
+        this.graph      = graph;
     }
 
     @Override
-    public void createVertexIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
+    public void createVertexMixedIndex(String indexName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
+        IndexBuilder indexBuilder = management.buildIndex(indexName, Vertex.class);
 
-        JanusGraphManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
         for (AtlasPropertyKey key : propertyKeys) {
             PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
             indexBuilder.addKey(janusKey);
         }
+
         indexBuilder.buildMixedIndex(backingIndex);
     }
 
     @Override
-    public void createEdgeIndex(String index, String backingIndex) {
-        buildMixedIndex(index, Edge.class, backingIndex);
-    }
+    public void createEdgeMixedIndex(String indexName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
+        IndexBuilder indexBuilder = management.buildIndex(indexName, Edge.class);
 
-    private void buildMixedIndex(String index, Class<? extends Element> janusClass, String backingIndex) {
+        for (AtlasPropertyKey key : propertyKeys) {
+            PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
+            indexBuilder.addKey(janusKey);
+        }
 
-        management.buildIndex(index, janusClass).buildMixedIndex(backingIndex);
+        indexBuilder.buildMixedIndex(backingIndex);
     }
 
     @Override
-    public void createFullTextIndex(String indexName, AtlasPropertyKey propertyKey, String backingIndex) {
+    public void createFullTextMixedIndex(String indexName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
+        IndexBuilder indexBuilder = management.buildIndex(indexName, Vertex.class);
 
-        PropertyKey fullText = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
+        for (AtlasPropertyKey key : propertyKeys) {
+            PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
+            indexBuilder.addKey(janusKey, org.janusgraph.core.schema.Parameter.of("mapping", Mapping.TEXT));
+        }
 
-        management.buildIndex(indexName, Vertex.class)
-                .addKey(fullText, org.janusgraph.core.schema.Parameter.of("mapping", Mapping.TEXT))
-                .buildMixedIndex(backingIndex);
+        indexBuilder.buildMixedIndex(backingIndex);
     }
 
     @Override
@@ -114,25 +116,28 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
         //for some reason, name checking was removed from StandardPropertyKeyMaker.make()
         //in Janus.  For consistency, do the check here.
         Preconditions.checkArgument(StringUtils.isNotBlank(name), "Need to specify name");
+
         for (char c : RESERVED_CHARS) {
-            Preconditions.checkArgument(name.indexOf(c) < 0, "Name can not contains reserved character %s: %s", c,
-                    name);
+            Preconditions.checkArgument(name.indexOf(c) < 0, "Name can not contains reserved character %s: %s", c, name);
         }
 
     }
 
     @Override
     public AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality) {
-
         if (cardinality.isMany()) {
             newMultProperties.add(propertyName);
         }
+
         PropertyKeyMaker propertyKeyBuilder = management.makePropertyKey(propertyName).dataType(propertyClass);
+
         if (cardinality != null) {
             Cardinality janusCardinality = AtlasJanusObjectFactory.createCardinality(cardinality);
             propertyKeyBuilder.cardinality(janusCardinality);
         }
+
         PropertyKey propertyKey = propertyKeyBuilder.make();
+
         return GraphDbObjectFactory.createPropertyKey(propertyKey);
     }
 
@@ -151,6 +156,7 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
 
         for (int i = 0;; i++) {
             String deletedKeyName = janusPropertyKey + "_deleted_" + i;
+
             if (null == management.getPropertyKey(deletedKeyName)) {
                 management.changeName(janusPropertyKey, deletedKeyName);
                 break;
@@ -161,6 +167,7 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
     @Override
     public AtlasPropertyKey getPropertyKey(String propertyName) {
         checkName(propertyName);
+
         return GraphDbObjectFactory.createPropertyKey(management.getPropertyKey(propertyName));
     }
 
@@ -169,37 +176,50 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
         return GraphDbObjectFactory.createEdgeLabel(management.getEdgeLabel(label));
     }
 
-    public void createExactMatchVertexIndex(String propertyName, boolean enforceUniqueness,
-                                      List<AtlasPropertyKey> propertyKeys) {
-
-        JanusGraphManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
-        for (AtlasPropertyKey key : propertyKeys) {
-            PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
-            indexBuilder.addKey(janusKey);
-        }
-        if (enforceUniqueness) {
-            indexBuilder.unique();
-        }
-        indexBuilder.buildCompositeIndex();
-    }
-
     @Override
-    public void addVertexIndexKey(String indexName, AtlasPropertyKey propertyKey) {
-        PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
+    public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
+        PropertyKey     janusKey    = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
         JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
+
         management.addIndexKey(vertexIndex, janusKey);
     }
 
     @Override
     public AtlasGraphIndex getGraphIndex(String indexName) {
         JanusGraphIndex index = management.getGraphIndex(indexName);
+
         return GraphDbObjectFactory.createGraphIndex(index);
     }
 
     @Override
-    public void createExactMatchIndex(String propertyName, boolean isUnique,
-            List<AtlasPropertyKey> propertyKeys) {
-        createExactMatchVertexIndex(propertyName, isUnique, propertyKeys);
+    public void createVertexCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
+        IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
+
+        for (AtlasPropertyKey key : propertyKeys) {
+            PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
+            indexBuilder.addKey(janusKey);
+        }
+
+        if (isUnique) {
+            indexBuilder.unique();
+        }
+
+        indexBuilder.buildCompositeIndex();
     }
 
-}
+    @Override
+    public void createEdgeCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
+        IndexBuilder indexBuilder = management.buildIndex(propertyName, Edge.class);
+
+        for (AtlasPropertyKey key : propertyKeys) {
+            PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
+            indexBuilder.addKey(janusKey);
+        }
+
+        if (isUnique) {
+            indexBuilder.unique();
+        }
+
+        indexBuilder.buildCompositeIndex();
+    }
+}
\ No newline at end of file

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java

@@ -56,8 +56,7 @@ public abstract class AbstractGraphDatabaseTest {
         AtlasGraphManagement mgmt = db.getGraph().getManagementSystem();
 
         if (mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
-            mgmt.createVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX,
-                    Collections.<AtlasPropertyKey>emptyList());
+            mgmt.createVertexMixedIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX, Collections.emptyList());
         }
         mgmt.makePropertyKey("age13", Integer.class, AtlasCardinality.SINGLE);
 
@@ -100,14 +99,14 @@ public abstract class AbstractGraphDatabaseTest {
         AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
         try {
             if (propertyClass != Integer.class) {
-                management.addVertexIndexKey(BACKING_INDEX_NAME, key);
+                management.addMixedIndex(BACKING_INDEX_NAME, key);
             }
         } catch(Throwable t) {
             //ok
             t.printStackTrace();
         }
         try {
-            management.createExactMatchIndex(propertyName, isUnique, Collections.singletonList(key));
+            management.createVertexCompositeIndex(propertyName, isUnique, Collections.singletonList(key));
 
         } catch(Throwable t) {
             //ok

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusDatabaseTest.java

@@ -30,10 +30,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
 import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
 import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.runner.LocalSolrRunner;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import java.math.BigDecimal;
@@ -43,7 +41,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
-import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
 import static org.testng.Assert.*;
 
 /**
@@ -67,7 +64,7 @@ public class AtlasJanusDatabaseTest {
                 AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName);
                 if (propertyKey == null) {
                     propertyKey = mgmt.makePropertyKey(propertyName, String.class, AtlasCardinality.SET);
-                    mgmt.createExactMatchIndex(propertyName, false, Collections.singletonList(propertyKey));
+                    mgmt.createVertexCompositeIndex(propertyName, false, Collections.singletonList(propertyKey));
                 }
             }
             mgmt.commit();

graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java

@@ -58,23 +58,16 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
     }
 
     @Override
-    public void createEdgeIndex(String index, String backingIndex) {
-        buildMixedIndex(index, Edge.class, backingIndex);
-    }
-
-    private void buildMixedIndex(String index, Class<? extends Element> titanClass, String backingIndex) {
-
-        management.buildIndex(index, titanClass).buildMixedIndex(backingIndex);
+    public void createEdgeMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
     }
 
     @Override
-    public void createFullTextIndex(String indexName, AtlasPropertyKey propertyKey, String backingIndex) {
+    public void createFullTextMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
+    }
 
-        PropertyKey fullText = TitanObjectFactory.createPropertyKey(propertyKey);
+    private void buildMixedIndex(String index, Class<? extends Element> titanClass, String backingIndex) {
 
-        management.buildIndex(indexName, Vertex.class)
-                .addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
-                .buildMixedIndex(backingIndex);
+        management.buildIndex(index, titanClass).buildMixedIndex(backingIndex);
     }
 
     @Override
@@ -146,8 +139,8 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
     }
 
     @Override
-    public void createExactMatchIndex(String propertyName, boolean enforceUniqueness,
-            List<AtlasPropertyKey> propertyKeys) {
+    public void createVertexCompositeIndex(String propertyName, boolean enforceUniqueness,
+                                           List<AtlasPropertyKey> propertyKeys) {
 
         TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
         for(AtlasPropertyKey key : propertyKeys) {
@@ -161,7 +154,23 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
     }
 
     @Override
-    public void createVertexIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
+    public void createEdgeCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
+        TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Edge.class);
+
+        for(AtlasPropertyKey key : propertyKeys) {
+            PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
+            indexBuilder.addKey(titanKey);
+        }
+
+        if (isUnique) {
+            indexBuilder.unique();
+        }
+
+        indexBuilder.buildCompositeIndex();
+    }
+
+    @Override
+    public void createVertexMixedIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
 
         TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
         for(AtlasPropertyKey key : propertyKeys) {
@@ -173,7 +182,7 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
 
 
     @Override
-    public void addVertexIndexKey(String indexName, AtlasPropertyKey propertyKey) {
+    public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
         PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
         TitanGraphIndex vertexIndex = management.getGraphIndex(indexName);
         management.addIndexKey(vertexIndex, titanKey);

graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java

@@ -55,8 +55,7 @@ public abstract class AbstractGraphDatabaseTest {
         AtlasGraphManagement mgmt = db.getGraph().getManagementSystem();
 
         if (mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
-            mgmt.createVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX,
-                    Collections.<AtlasPropertyKey>emptyList());
+            mgmt.createVertexMixedIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX, Collections.emptyList());
         }
         mgmt.makePropertyKey("age13", Integer.class, AtlasCardinality.SINGLE);
 
@@ -100,14 +99,14 @@ public abstract class AbstractGraphDatabaseTest {
         AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
         try {
             if (propertyClass != Integer.class) {
-                management.addVertexIndexKey(BACKING_INDEX_NAME, key);
+                management.addMixedIndex(BACKING_INDEX_NAME, key);
             }
         } catch(Throwable t) {
             //ok
             t.printStackTrace();
         }
         try {
-            management.createExactMatchIndex(propertyName, isUnique, Collections.singletonList(key));
+            management.createVertexCompositeIndex(propertyName, isUnique, Collections.singletonList(key));
 
         } catch(Throwable t) {
             //ok

graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java

@@ -65,7 +65,7 @@ public class Titan0DatabaseTest {
                 AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName);
                 if (propertyKey == null) {
                     propertyKey = mgmt.makePropertyKey(propertyName, String.class, AtlasCardinality.SET);
-                    mgmt.createExactMatchIndex(propertyName, false, Collections.singletonList(propertyKey));
+                    mgmt.createVertexCompositeIndex(propertyName, false, Collections.singletonList(propertyKey));
                 }
             }
             mgmt.commit();

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -591,7 +591,7 @@ public final class GraphHelper {
         AtlasEdge ret;
 
         try {
-            ret = findEdge(Constants.GUID_PROPERTY_KEY, guid);
+            ret = findEdge(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, guid);
         } catch (EntityNotFoundException e) {
             throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_GUID_NOT_FOUND, guid);
         }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java

@@ -423,7 +423,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
             final String guid = UUID.randomUUID().toString();
 
             AtlasGraphUtilsV1.setProperty(ret, Constants.ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
-            AtlasGraphUtilsV1.setProperty(ret, Constants.GUID_PROPERTY_KEY, guid);
+            AtlasGraphUtilsV1.setProperty(ret, Constants.RELATIONSHIP_GUID_PROPERTY_KEY, guid);
             AtlasGraphUtilsV1.setProperty(ret, Constants.VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
             AtlasGraphUtilsV1.setProperty(ret, Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
         }