ATLAS-1060 Add composite indexes for exact match performance improvements for…

addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java

@@ -97,7 +97,7 @@ public class FalconDataModelGenerator {
     private void createClusterEntityClass() throws AtlasException {
         AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
                 new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        null),
+                        false, true, null),
                 // map of tags
                 new AttributeDefinition(TAGS,
                         DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
@@ -113,7 +113,7 @@ public class FalconDataModelGenerator {
     private void createFeedCreationEntityClass() throws AtlasException {
         AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
                 new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
-                        false, null)
+                        false, false, true, null)
         };
 
         HierarchicalTypeDefinition<ClassType> definition =
@@ -127,7 +127,7 @@ public class FalconDataModelGenerator {
         AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
                 TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
                 new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
-                        false, null),
+                        false, false, true, null),
                 new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
                 // map of tags
                 new AttributeDefinition(TAGS,
@@ -156,12 +156,12 @@ public class FalconDataModelGenerator {
         AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
                 TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
                 new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
-                        false, null),
+                        false, false, true, null),
                 // map of tags
                 new AttributeDefinition(TAGS,
                         DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
                         Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
+                new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
                 // wf properties
                 new AttributeDefinition(WFPROPERTIES,
                         DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),

addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala

@@ -45,9 +45,9 @@ object FSDataModel extends App {
             //Is a regular file or a directory. If true, it is a file else a directory
             "isFile" ~ (boolean, optional, indexed)
             //Is a symlink or not
-            "isSymlink" ~ (boolean, optional, indexed)
+            "isSymlink" ~ (boolean, optional)
             //Optional and may not be set for a directory
-            "fileSize" ~ (long, optional, indexed)
+            "fileSize" ~ (long, optional)
             "group" ~ (string, optional, indexed)
             "posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed)
         }
@@ -66,7 +66,7 @@ object FSDataModel extends App {
             //Making cluster optional since path is already unique containing the namenode URI
             AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed)
             "numberOfReplicas" ~ (int, optional, indexed)
-            "extendedAttributes" ~ (map(string, string), optional, indexed)
+            "extendedAttributes" ~ (map(string, string), optional)
         }
         //TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists
     }

addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java

@@ -204,7 +204,7 @@ public class HiveDataModelGenerator {
     private void createDBClass() throws AtlasException {
         AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
                 new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        null),
+                        false, true, null),
                 new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
                         null),
                 new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
@@ -220,7 +220,7 @@ public class HiveDataModelGenerator {
 
     private void createColumnClass() throws AtlasException {
         AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+                new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
                 new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
                 //Making this optional since this is an incompatible change
                 //Reverse attribute to 'columns' in Table
@@ -258,7 +258,7 @@ public class HiveDataModelGenerator {
                 new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
                         null),
                 new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),};
+                        false, true, null),};
         HierarchicalTypeDefinition<ClassType> definition =
                 new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null,
                     ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
@@ -271,9 +271,9 @@ public class HiveDataModelGenerator {
                 new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
                 new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
                 new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        null),
+                        false, true, null),
                 new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        null),
+                        false, true, null),
                 new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
                         null),
                 new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,

addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java

@@ -54,7 +54,7 @@ public class HiveMetastoreBridgeIT extends HiveITBase {
 
         int tableCount = atlasClient.listEntities(HiveDataTypes.HIVE_TABLE.getName()).size();
 
-        //Now import using import tool - should be no-op
+        //Now import using import tool - should be no-op. This also tests update since table exists
         hiveMetaStoreBridge.importTable(atlasClient.getEntity(dbId), DEFAULT_DB, tableName, true);
         String tableId2 = assertTableIsRegistered(DEFAULT_DB, tableName);
         assertEquals(tableId2, tableId);

addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java

@@ -1502,7 +1502,7 @@ public class HiveHookIT extends HiveITBase {
         final String fmtQuery = "alter database %s set OWNER %s %s";
         String query = String.format(fmtQuery, dbName, "USER", owner);
 
-        runCommand(query);
+        runCommandWithDelay(query, 1000);
 
         assertDatabaseIsRegistered(dbName, new AssertPredicate() {
             @Override

addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java

@@ -119,13 +119,13 @@ public class SqoopDataModelGenerator {
     private void createSqoopDbStoreClass() throws AtlasException {
         AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
                 new AttributeDefinition(DB_STORE_TYPE,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+                        DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
                 new AttributeDefinition(DB_STORE_USAGE,
                         DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
                 new AttributeDefinition(STORE_URI,
                         DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
                 new AttributeDefinition(SOURCE,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)
+                        DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)
         };
 
         HierarchicalTypeDefinition<ClassType> definition =
@@ -139,12 +139,12 @@ public class SqoopDataModelGenerator {
     private void createSqoopProcessClass() throws AtlasException {
         AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
                 new AttributeDefinition(OPERATION,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+                        DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
                 new AttributeDefinition(CMD_LINE_OPTS, STRING_MAP_TYPE.getName(), Multiplicity.REQUIRED, false, null),
                 new AttributeDefinition(START_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
                 new AttributeDefinition(END_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
                 new AttributeDefinition(USER,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
+                        DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
         };
 
         HierarchicalTypeDefinition<ClassType> definition =

release-log.txt

@@ -3,7 +3,7 @@ Apache Atlas Release Notes
 
 --trunk - unreleased
 INCOMPATIBLE CHANGES:
-
+ATLAS-1060 Add composite indexes for exact match performance improvements for all attributes (sumasai via shwethags)
 
 ALL CHANGES:
 ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags)

repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java

@@ -185,15 +185,14 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
     private void createSuperTypes() throws AtlasException {
         HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
                 .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
-                        TypesUtil.createUniqueRequiredAttrDef(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
-                                DataTypes.STRING_TYPE));
+                 new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
         createType(referenceableType);
 
         HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
                 .createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
-                        TypesUtil.createRequiredAttrDef(AtlasClient.NAME, DataTypes.STRING_TYPE),
+                        new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
                         TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
-                        TypesUtil.createOptionalAttrDef(AtlasClient.OWNER, DataTypes.STRING_TYPE));
+                        new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
         createType(assetType);
 
         HierarchicalTypeDefinition<ClassType> infraType = TypesUtil

repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java

@@ -385,6 +385,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
                 {"hive_table where name='sales_fact', db where name='Reporting'", 0},
                 {"hive_partition as p where values = ['2015-01-01']", 1},
 //              {"StorageDesc select cols", 6} //Not working since loading of lists needs to be fixed yet
+
+                //check supertypeNames
+                {"DataSet where name='sales_fact'", 1},
+                {"Asset where name='sales_fact'", 1}
         };
     }
 

repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java

@@ -135,6 +135,7 @@ public class GraphBackedSearchIndexerTest {
 
         verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY, false);
         verifyVertexIndexContains(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY);
+        verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.SUPER_TYPES_PROPERTY_KEY, false);
 
         verifyVertexIndexContains(managementSystem, "Database.managedType");
     }
@@ -146,13 +147,13 @@ public class GraphBackedSearchIndexerTest {
     }
 
     private void verifySystemCompositeIndex(TitanManagement managementSystem, String indexName, boolean isUnique) {
-        TitanGraphIndex guidIndex = managementSystem.getGraphIndex(indexName);
-        assertNotNull(guidIndex);
-        assertTrue(guidIndex.isCompositeIndex());
+        TitanGraphIndex systemIndex = managementSystem.getGraphIndex(indexName);
+        assertNotNull(systemIndex);
+        assertTrue(systemIndex.isCompositeIndex());
         if (isUnique) {
-            assertTrue(guidIndex.isUnique());
+            assertTrue(systemIndex.isUnique());
         } else {
-            assertFalse(guidIndex.isUnique());
+            assertFalse(systemIndex.isUnique());
         }
     }
 }

repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala

@@ -81,7 +81,7 @@ class LineageQueryTest extends BaseGremlinTest {
                          |        },
                          |        "isComposite":false,
                          |        "isUnique":false,
-                         |        "isIndexable":true,
+                         |        "isIndexable":false,
                          |        "reverseAttributeName":null
                          |      },
                          |      {
@@ -94,7 +94,7 @@ class LineageQueryTest extends BaseGremlinTest {
                          |        },
                          |        "isComposite":false,
                          |        "isUnique":false,
-                         |        "isIndexable":true,
+                         |        "isIndexable":false,
                          |        "reverseAttributeName":null
                          |      },
                          |      {
@@ -107,7 +107,7 @@ class LineageQueryTest extends BaseGremlinTest {
                          |        },
                          |        "isComposite":false,
                          |        "isUnique":false,
-                         |        "isIndexable":true,
+                         |        "isIndexable":false,
                          |        "reverseAttributeName":null
                          |      },
                          |      {
@@ -120,7 +120,7 @@ class LineageQueryTest extends BaseGremlinTest {
                          |        },
                          |        "isComposite":false,
                          |        "isUnique":false,
-                         |        "isIndexable":true,
+                         |        "isIndexable":false,
                          |        "reverseAttributeName":null
                          |      }
                          |    ]
@@ -213,7 +213,7 @@ class LineageQueryTest extends BaseGremlinTest {
                           |        },
                           |        "isComposite":false,
                           |        "isUnique":false,
-                          |        "isIndexable":true,
+                          |        "isIndexable":false,
                           |        "reverseAttributeName":null
                           |      },
                           |      {
@@ -226,7 +226,7 @@ class LineageQueryTest extends BaseGremlinTest {
                           |        },
                           |        "isComposite":false,
                           |        "isUnique":false,
-                          |        "isIndexable":true,
+                          |        "isIndexable":false,
                           |        "reverseAttributeName":null
                           |      },
                           |      {
@@ -239,7 +239,7 @@ class LineageQueryTest extends BaseGremlinTest {
                           |        },
                           |        "isComposite":false,
                           |        "isUnique":false,
-                          |        "isIndexable":true,
+                          |        "isIndexable":false,
                           |        "reverseAttributeName":null
                           |      },
                           |      {
@@ -252,7 +252,7 @@ class LineageQueryTest extends BaseGremlinTest {
                           |        },
                           |        "isComposite":false,
                           |        "isUnique":false,
-                          |        "isIndexable":true,
+                          |        "isIndexable":false,
                           |        "reverseAttributeName":null
                           |      }
                           |    ]
@@ -365,7 +365,7 @@ class LineageQueryTest extends BaseGremlinTest {
         },
         "isComposite":false,
         "isUnique":false,
-        "isIndexable":true,
+        "isIndexable":false,
         "reverseAttributeName":null
       },
       {
@@ -378,7 +378,7 @@ class LineageQueryTest extends BaseGremlinTest {
         },
         "isComposite":false,
         "isUnique":false,
-        "isIndexable":true,
+        "isIndexable":false,
         "reverseAttributeName":null
       }
     ]
@@ -434,7 +434,7 @@ class LineageQueryTest extends BaseGremlinTest {
                           |        },
                           |        "isComposite":false,
                           |        "isUnique":false,
-                          |        "isIndexable":true,
+                          |        "isIndexable":false,
                           |        "reverseAttributeName":null
                           |      },
                           |      {
@@ -447,7 +447,7 @@ class LineageQueryTest extends BaseGremlinTest {
                           |        },
                           |        "isComposite":false,
                           |        "isUnique":false,
-                          |        "isIndexable":true,
+                          |        "isIndexable":false,
                           |        "reverseAttributeName":null
                           |      },
                           |      {
@@ -460,7 +460,7 @@ class LineageQueryTest extends BaseGremlinTest {
                           |        },
                           |        "isComposite":false,
                           |        "isUnique":false,
-                          |        "isIndexable":true,
+                          |        "isIndexable":false,
                           |        "reverseAttributeName":null
                           |      },
                           |      {
@@ -473,7 +473,7 @@ class LineageQueryTest extends BaseGremlinTest {
                           |        },
                           |        "isComposite":false,
                           |        "isUnique":false,
-                          |        "isIndexable":true,
+                          |        "isIndexable":false,
                           |        "reverseAttributeName":null
                           |      }
                           |    ]

typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java

@@ -38,7 +38,7 @@ public final class AttributeDefinition {
 
     public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
             String reverseAttributeName) {
-        this(name, dataTypeName, multiplicity, isComposite, false, true, reverseAttributeName);
+        this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName);
 
     }