Skip to content

A
atlas
Commit 91072c10 by Shwetha GS
Options

ATLAS-1060 Add composite indexes for exact match performance improvements for…

ATLAS-1060 Add composite indexes for exact match performance improvements for all attributes (sumasai via shwethags)
parent 89f25d41
Showing with 48 additions and 44 deletions
addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
......@@ -97,7 +97,7 @@ public class FalconDataModelGenerator {
private void createClusterEntityClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
false, true, null),
// map of tags
new AttributeDefinition(TAGS,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
......@@ -113,7 +113,7 @@ public class FalconDataModelGenerator {
private void createFeedCreationEntityClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
false, null)
false, false, true, null)
};
HierarchicalTypeDefinition<ClassType> definition =
......@@ -127,7 +127,7 @@ public class FalconDataModelGenerator {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
false, null),
false, false, true, null),
new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
// map of tags
new AttributeDefinition(TAGS,
......@@ -156,12 +156,12 @@ public class FalconDataModelGenerator {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
false, null),
false, false, true, null),
// map of tags
new AttributeDefinition(TAGS,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
// wf properties
new AttributeDefinition(WFPROPERTIES,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
......
addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
......@@ -45,9 +45,9 @@ object FSDataModel extends App {
//Is a regular file or a directory. If true, it is a file else a directory
"isFile" ~ (boolean, optional, indexed)
//Is a symlink or not
"isSymlink" ~ (boolean, optional, indexed)
"isSymlink" ~ (boolean, optional)
//Optional and may not be set for a directory
"fileSize" ~ (long, optional, indexed)
"fileSize" ~ (long, optional)
"group" ~ (string, optional, indexed)
"posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed)
}
......@@ -66,7 +66,7 @@ object FSDataModel extends App {
//Making cluster optional since path is already unique containing the namenode URI
AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed)
"numberOfReplicas" ~ (int, optional, indexed)
"extendedAttributes" ~ (map(string, string), optional, indexed)
"extendedAttributes" ~ (map(string, string), optional)
}
//TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists
}
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
......@@ -204,7 +204,7 @@ public class HiveDataModelGenerator {
private void createDBClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
false, true, null),
new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
......@@ -220,7 +220,7 @@ public class HiveDataModelGenerator {
private void createColumnClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
//Making this optional since this is an incompatible change
//Reverse attribute to 'columns' in Table
......@@ -258,7 +258,7 @@ public class HiveDataModelGenerator {
new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),};
false, true, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null,
ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
......@@ -271,9 +271,9 @@ public class HiveDataModelGenerator {
new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
false, true, null),
new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
false, true, null),
new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java
......@@ -54,7 +54,7 @@ public class HiveMetastoreBridgeIT extends HiveITBase {
int tableCount = atlasClient.listEntities(HiveDataTypes.HIVE_TABLE.getName()).size();
//Now import using import tool - should be no-op
//Now import using import tool - should be no-op. This also tests update since table exists
hiveMetaStoreBridge.importTable(atlasClient.getEntity(dbId), DEFAULT_DB, tableName, true);
String tableId2 = assertTableIsRegistered(DEFAULT_DB, tableName);
assertEquals(tableId2, tableId);
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
......@@ -1502,7 +1502,7 @@ public class HiveHookIT extends HiveITBase {
final String fmtQuery = "alter database %s set OWNER %s %s";
String query = String.format(fmtQuery, dbName, "USER", owner);
runCommand(query);
runCommandWithDelay(query, 1000);
assertDatabaseIsRegistered(dbName, new AssertPredicate() {
@Override
......
addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
......@@ -119,13 +119,13 @@ public class SqoopDataModelGenerator {
private void createSqoopDbStoreClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(DB_STORE_TYPE,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(DB_STORE_USAGE,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(STORE_URI,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(SOURCE,
DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)
DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)
};
HierarchicalTypeDefinition<ClassType> definition =
......@@ -139,12 +139,12 @@ public class SqoopDataModelGenerator {
private void createSqoopProcessClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(OPERATION,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(CMD_LINE_OPTS, STRING_MAP_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(START_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(END_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(USER,
DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
};
HierarchicalTypeDefinition<ClassType> definition =
......
release-log.txt
......@@ -3,7 +3,7 @@ Apache Atlas Release Notes
--trunk - unreleased
INCOMPATIBLE CHANGES:
ATLAS-1060 Add composite indexes for exact match performance improvements for all attributes (sumasai via shwethags)
ALL CHANGES:
ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags)
......
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -185,15 +185,14 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private void createSuperTypes() throws AtlasException {
HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
.createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
DataTypes.STRING_TYPE));
new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
createType(referenceableType);
HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
.createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
TypesUtil.createRequiredAttrDef(AtlasClient.NAME, DataTypes.STRING_TYPE),
new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef(AtlasClient.OWNER, DataTypes.STRING_TYPE));
new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
createType(assetType);
HierarchicalTypeDefinition<ClassType> infraType = TypesUtil
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -385,6 +385,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_table where name='sales_fact', db where name='Reporting'", 0},
{"hive_partition as p where values = ['2015-01-01']", 1},
// {"StorageDesc select cols", 6} //Not working since loading of lists needs to be fixed yet
//check supertypeNames
{"DataSet where name='sales_fact'", 1},
{"Asset where name='sales_fact'", 1}
};
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
......@@ -135,6 +135,7 @@ public class GraphBackedSearchIndexerTest {
verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY, false);
verifyVertexIndexContains(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY);
verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.SUPER_TYPES_PROPERTY_KEY, false);
verifyVertexIndexContains(managementSystem, "Database.managedType");
}
......@@ -146,13 +147,13 @@ public class GraphBackedSearchIndexerTest {
}
private void verifySystemCompositeIndex(TitanManagement managementSystem, String indexName, boolean isUnique) {
TitanGraphIndex guidIndex = managementSystem.getGraphIndex(indexName);
assertNotNull(guidIndex);
assertTrue(guidIndex.isCompositeIndex());
TitanGraphIndex systemIndex = managementSystem.getGraphIndex(indexName);
assertNotNull(systemIndex);
assertTrue(systemIndex.isCompositeIndex());
if (isUnique) {
assertTrue(guidIndex.isUnique());
assertTrue(systemIndex.isUnique());
} else {
assertFalse(guidIndex.isUnique());
assertFalse(systemIndex.isUnique());
}
}
}
repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
......@@ -81,7 +81,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -94,7 +94,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -107,7 +107,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -120,7 +120,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
......@@ -213,7 +213,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -226,7 +226,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -239,7 +239,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -252,7 +252,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
......@@ -365,7 +365,7 @@ class LineageQueryTest extends BaseGremlinTest {
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"isIndexable":false,
"reverseAttributeName":null
},
{
......@@ -378,7 +378,7 @@ class LineageQueryTest extends BaseGremlinTest {
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"isIndexable":false,
"reverseAttributeName":null
}
]
......@@ -434,7 +434,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -447,7 +447,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -460,7 +460,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| },
| {
......@@ -473,7 +473,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
......@@ -38,7 +38,7 @@ public final class AttributeDefinition {
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
String reverseAttributeName) {
this(name, dataTypeName, multiplicity, isComposite, false, true, reverseAttributeName);
this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment