ATLAS-17 Parameterize schema API query per typeName (shwethags)

release-log.txt

@@ -4,15 +4,15 @@ Apache Atlas Release Notes
 --trunk - unreleased
 
 INCOMPATIBLE CHANGES:
+ATLAS-17 Parameterize schema API query per typeName (shwethags)
 
 ALL CHANGES:
-
 ATLAS-10 Update trunk version to 0.6-incubating-SNAPSHOT (shwethags)
 
+
 --Release 0.5-incubating
 
 ALL CHANGES:
-
 ATLAS-13 Add project website (Venkatesh Seetharam)
 ATLAS-12 Update the copyright in Notice and License files (Venkatesh Seetharam)
 ATLAS-9 Create branch-0.5 and update version to 0.5-incubating (shwethags)

repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java

@@ -32,6 +32,7 @@ import org.apache.atlas.query.HiveWhereUsedQuery;
 import org.apache.atlas.repository.EntityNotFoundException;
 import org.apache.atlas.repository.MetadataRepository;
 import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -53,26 +54,27 @@ public class HiveLineageService implements LineageService {
     private static final Option<List<String>> SELECT_ATTRIBUTES =
             Some.<List<String>>apply(List.<String>fromArray(new String[]{"name"}));
 
+    public static final String HIVE_TABLE_SCHEMA_QUERY_PREFIX = "atlas.lineage.hive.table.schema.query.";
+
     private static final String HIVE_TABLE_TYPE_NAME;
     private static final String HIVE_PROCESS_TYPE_NAME;
     private static final String HIVE_PROCESS_INPUT_ATTRIBUTE_NAME;
     private static final String HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME;
 
-    private static final String HIVE_TABLE_SCHEMA_QUERY;
     private static final String HIVE_TABLE_EXISTS_QUERY;
 
+    private static final PropertiesConfiguration propertiesConf;
+
     static {
         // todo - externalize this using type system - dog food
         try {
-            PropertiesConfiguration conf = PropertiesUtil.getApplicationProperties();
-            HIVE_TABLE_TYPE_NAME = conf.getString("atlas.lineage.hive.table.type.name", "DataSet");
-            HIVE_PROCESS_TYPE_NAME = conf.getString("atlas.lineage.hive.process.type.name", "Process");
-            HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = conf.getString("atlas.lineage.hive.process.inputs.name", "inputs");
-            HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = conf.getString("atlas.lineage.hive.process.outputs.name", "outputs");
-
-            HIVE_TABLE_SCHEMA_QUERY =
-                    conf.getString("atlas.lineage.hive.table.schema.query", "hive_table where name=\"%s\", columns");
-            HIVE_TABLE_EXISTS_QUERY = conf.getString("atlas.lineage.hive.table.exists.query",
+            propertiesConf = PropertiesUtil.getApplicationProperties();
+            HIVE_TABLE_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.table.type.name", "DataSet");
+            HIVE_PROCESS_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.process.type.name", "Process");
+            HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.inputs.name", "inputs");
+            HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.outputs.name", "outputs");
+
+            HIVE_TABLE_EXISTS_QUERY = propertiesConf.getString("atlas.lineage.hive.table.exists.query",
                     "from " + HIVE_TABLE_TYPE_NAME + " where name=\"%s\"");
         } catch (AtlasException e) {
             throw new RuntimeException(e);
@@ -195,9 +197,10 @@ public class HiveLineageService implements LineageService {
     public String getSchema(String tableName) throws AtlasException {
         LOG.info("Fetching schema for tableName={}", tableName);
         ParamChecker.notEmpty(tableName, "table name cannot be null");
-        validateTableExists(tableName);
+        String typeName = validateTableExists(tableName);
 
-        final String schemaQuery = String.format(HIVE_TABLE_SCHEMA_QUERY, tableName);
+        final String schemaQuery =
+                String.format(propertiesConf.getString(HIVE_TABLE_SCHEMA_QUERY_PREFIX + typeName), tableName);
         return discoveryService.searchByDSL(schemaQuery);
     }
 
@@ -206,11 +209,14 @@ public class HiveLineageService implements LineageService {
      *
      * @param tableName table name
      */
-    private void validateTableExists(String tableName) throws AtlasException {
+    private String validateTableExists(String tableName) throws AtlasException {
         final String tableExistsQuery = String.format(HIVE_TABLE_EXISTS_QUERY, tableName);
         GremlinQueryResult queryResult = discoveryService.evaluate(tableExistsQuery);
         if (!(queryResult.rows().length() > 0)) {
             throw new EntityNotFoundException(tableName + " does not exist");
         }
+
+        ReferenceableInstance referenceable = (ReferenceableInstance)queryResult.rows().apply(0);
+        return referenceable.getTypeName();
     }
 }

repository/src/test/resources/application.properties

@@ -36,7 +36,7 @@ atlas.graph.index.search.elasticsearch.local-mode=true
 #atlas.lineage.hive.process.outputs.name=outputs
 
 ## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
 
 
 #########  Security Properties  #########

src/conf/application.properties

@@ -36,7 +36,8 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
 #atlas.lineage.hive.process.outputs.name=outputs
 
 ## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
+atlas.lineage.hive.table.schema.query.Table=Table where name='%s'\, columns
 
 
 #########  Security Properties  #########

webapp/src/main/resources/application.properties

@@ -36,7 +36,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
 #atlas.lineage.hive.process.outputs.name=outputs
 
 ## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
 
 
 #########  Security Properties  #########