Skip to content

A
atlas
Commit fad5c8e6 by gss2002
Options

Commit fixes to HiveLineageInfo HiveHook and hivetypes.pom.xml to remove

bad dependency on calcite-0.9.2-incubating-snapshot which is no longer useable and Hive POM needs to be updated at Central for Hive 0.14 and Hive 1.0 to make it calcite-0.9.2-incubating. Removed where/groupby parsing for now as value is limited and doesnt parse nested queries well. Handle createtable/createview and prepare to handle alters and inserts
parent 2181803a
Showing with 213 additions and 48 deletions
hivetypes/pom.xml
...@@ -56,26 +56,86 @@ ...@@ -56,26 +56,86 @@
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId> <artifactId>hive-metastore</artifactId>
<version>0.14.0</version> <version>0.14.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId> <artifactId>hive-exec</artifactId>
<version>0.14.0</version> <version>0.14.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId> <artifactId>hive-common</artifactId>
<version>0.14.0</version> <version>0.14.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-cli</artifactId> <artifactId>hive-cli</artifactId>
<version>0.14.0</version> <version>0.14.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId> <artifactId>hive-jdbc</artifactId>
<version>0.14.0</version> <version>0.14.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
<version>0.9.2-incubating</version>
</dependency>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
<version>0.9.2-incubating</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.code.gson</groupId> <groupId>com.google.code.gson</groupId>
......
metadata-bridge-parent/metadata-bridge-hive/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/HiveLineageInfo.java
...@@ -96,20 +96,36 @@ public class HiveLineageInfo implements NodeProcessor { ...@@ -96,20 +96,36 @@ public class HiveLineageInfo implements NodeProcessor {
switch (pt.getToken().getType()) { switch (pt.getToken().getType()) {
case HiveParser.TOK_TABREF: case HiveParser.TOK_FROM:
sourceTables = new ArrayList<SourceTables>();
LOG.debug("From Table Dump: "+pt.dump()); LOG.debug("From Table Dump: "+pt.dump());
fromTableDump(pt); fromTableDump(pt);
this.hlb.setSourceTables(sourceTables);
break; break;
case HiveParser.TOK_SELECT: case HiveParser.TOK_SELECT:
queryColumns = new ArrayList<QueryColumns>();
LOG.debug("Column Dump: "+pt.dump()); LOG.debug("Column Dump: "+pt.dump());
columnTableDump(pt); columnTableDump(pt);
this.hlb.setQueryColumns(queryColumns);
break; break;
case HiveParser.TOK_SELECTDI:
LOG.debug("Column Dump: "+pt.dump());
columnTableDump(pt);
break;
case HiveParser.TOK_CREATETABLE:
createColumns = new ArrayList<CreateColumns>();
LOG.debug("CREATABLE DUMP: "+pt.dump());
createTableDump(pt);
break;
case HiveParser.TOK_CREATEVIEW:
createColumns = new ArrayList<CreateColumns>();
LOG.debug("CREATEVIEW DUMP: "+pt.dump());
createTableDump(pt);
break;
/*
* Currently disabling processing of WHERE and GROUPBY NO VALUE RIGHT NOW
*
case HiveParser.TOK_WHERE: case HiveParser.TOK_WHERE:
whereClause = new ArrayList<WhereClause>(); whereClause = new ArrayList<WhereClause>();
LOG.debug("WHERE CLAUSE DUMP: "+pt.dump()); LOG.debug("WHERE CLAUSE DUMP: "+pt.dump());
...@@ -124,11 +140,7 @@ public class HiveLineageInfo implements NodeProcessor { ...@@ -124,11 +140,7 @@ public class HiveLineageInfo implements NodeProcessor {
this.hlb.setGroupBy(groupBy); this.hlb.setGroupBy(groupBy);
break; break;
case HiveParser.TOK_CREATETABLE: */
createColumns = new ArrayList<CreateColumns>();
LOG.debug("CREATABLE DUMP: "+pt.dump());
createTableDump(pt);
break;
} }
return null; return null;
} }
...@@ -151,30 +163,30 @@ public class HiveLineageInfo implements NodeProcessor { ...@@ -151,30 +163,30 @@ public class HiveLineageInfo implements NodeProcessor {
if (nodeIn.getParent().getText().equalsIgnoreCase(".")) { if (nodeIn.getParent().getText().equalsIgnoreCase(".")) {
ASTNode checkOrAnd = (ASTNode) nodeIn.getParent().getParent().getChild(1).getParent().getParent(); ASTNode checkOrAnd = (ASTNode) nodeIn.getParent().getParent().getChild(1).getParent().getParent();
if (checkOrAnd.getType() == HiveParser.KW_AND || checkOrAnd.getType() == HiveParser.KW_OR) { if (checkOrAnd.getType() == HiveParser.KW_AND || checkOrAnd.getType() == HiveParser.KW_OR) {
LOG.info("WHERE:: "+checkOrAnd.getText()); LOG.debug("WHERE:: "+checkOrAnd.getText());
whreClse.setColumnOperator(checkOrAnd.getText()); whreClse.setColumnOperator(checkOrAnd.getText());
} }
LOG.info("Table Alias:: "+nodeIn.getChild(0).getText()); LOG.debug("Table Alias:: "+nodeIn.getChild(0).getText());
whreClse.setTbAliasOrName(nodeIn.getChild(0).getText()); whreClse.setTbAliasOrName(nodeIn.getChild(0).getText());
LOG.info("Delimiter:: "+nodeIn.getParent().getText()); LOG.debug("Delimiter:: "+nodeIn.getParent().getText());
LOG.info("Column:: "+nodeIn.getParent().getChild(1).getText()); LOG.debug("Column:: "+nodeIn.getParent().getChild(1).getText());
whreClse.setColumnName(nodeIn.getParent().getChild(1).getText()); whreClse.setColumnName(nodeIn.getParent().getChild(1).getText());
LOG.info("Column Qualifer:: "+nodeIn.getParent().getParent().getChild(1).getParent().getText()); LOG.debug("Column Qualifer:: "+nodeIn.getParent().getParent().getChild(1).getParent().getText());
whreClse.setColumnOperator(nodeIn.getParent().getParent().getChild(1).getParent().getText()); whreClse.setColumnOperator(nodeIn.getParent().getParent().getChild(1).getParent().getText());
LOG.info("Column Value:: "+nodeIn.getParent().getParent().getChild(1).getText()); LOG.debug("Column Value:: "+nodeIn.getParent().getParent().getChild(1).getText());
whreClse.setColumnValue(nodeIn.getParent().getParent().getChild(1).getText()); whreClse.setColumnValue(nodeIn.getParent().getParent().getChild(1).getText());
} else { } else {
ASTNode checkOrAnd = (ASTNode) nodeIn.getParent().getParent().getChild(1).getParent(); ASTNode checkOrAnd = (ASTNode) nodeIn.getParent().getParent().getChild(1).getParent();
if (checkOrAnd.getType() == HiveParser.KW_AND || checkOrAnd.getType() == HiveParser.KW_OR) { if (checkOrAnd.getType() == HiveParser.KW_AND || checkOrAnd.getType() == HiveParser.KW_OR) {
LOG.info("WHERE:: "+checkOrAnd.getText()); LOG.debug("WHERE:: "+checkOrAnd.getText());
whreClse.setColumnOperator(checkOrAnd.getText()); whreClse.setColumnOperator(checkOrAnd.getText());
} }
LOG.info("Column:: = "+nodeIn.getChild(0).getText()); LOG.debug("Column:: = "+nodeIn.getChild(0).getText());
whreClse.setColumnName(nodeIn.getChild(0).getText()); whreClse.setColumnName(nodeIn.getChild(0).getText());
//LOG.info("Delimiter "+nodeIn.getParent().getText()); //LOG.info("Delimiter "+nodeIn.getParent().getText());
LOG.info("Column Qualifer:: "+nodeIn.getParent().getChild(1).getParent().getText()); LOG.debug("Column Qualifer:: "+nodeIn.getParent().getChild(1).getParent().getText());
whreClse.setColumnOperator(nodeIn.getParent().getChild(1).getParent().getText()); whreClse.setColumnOperator(nodeIn.getParent().getChild(1).getParent().getText());
LOG.info("Column Value:: "+nodeIn.getParent().getChild(1).getText()); LOG.debug("Column Value:: "+nodeIn.getParent().getChild(1).getText());
whreClse.setColumnValue(nodeIn.getParent().getChild(1).getText()); whreClse.setColumnValue(nodeIn.getParent().getChild(1).getText());
} }
whereClause.add(whreClse); whereClause.add(whreClse);
...@@ -206,12 +218,12 @@ public class HiveLineageInfo implements NodeProcessor { ...@@ -206,12 +218,12 @@ public class HiveLineageInfo implements NodeProcessor {
GroupBy grpBy = hlb.new GroupBy(); GroupBy grpBy = hlb.new GroupBy();
ASTNode parentNode = (ASTNode) nodeIn.getParent(); ASTNode parentNode = (ASTNode) nodeIn.getParent();
if (parentNode.getText().equalsIgnoreCase(".")) { if (parentNode.getText().equalsIgnoreCase(".")) {
LOG.info("GroupBy TableAlias: "+nodeIn.getChild(0).getText()); LOG.debug("GroupBy TableAlias: "+nodeIn.getChild(0).getText());
grpBy.setTbAliasOrName(nodeIn.getChild(0).getText()); grpBy.setTbAliasOrName(nodeIn.getChild(0).getText());
LOG.info("GroupBy Column:: "+parentNode.getChild(1).getText()); LOG.debug("GroupBy Column:: "+parentNode.getChild(1).getText());
grpBy.setColumnName(parentNode.getChild(1).getText()); grpBy.setColumnName(parentNode.getChild(1).getText());
} else { } else {
LOG.info("GroupBy Column: "+nodeIn.getChild(0).getText()); LOG.debug("GroupBy Column: "+nodeIn.getChild(0).getText());
grpBy.setColumnName(nodeIn.getChild(0).getText()); grpBy.setColumnName(nodeIn.getChild(0).getText());
} }
groupBy.add(grpBy); groupBy.add(grpBy);
...@@ -232,15 +244,23 @@ public class HiveLineageInfo implements NodeProcessor { ...@@ -232,15 +244,23 @@ public class HiveLineageInfo implements NodeProcessor {
public void createTableDump(ASTNode nodeIn) { public void createTableDump(ASTNode nodeIn) {
counter = 0; counter = 0;
if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME) != null) { if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME) != null && nodeIn.getAncestor(HiveParser.TOK_WHERE) == null) {
LOG.info("Create TableName:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getText()); LOG.info("Create TableName:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getText());
if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChildCount() == 2) {
LOG.debug("To DataBaseName:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(0).getText());
hlb.setDatabaseName(nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(0).getText());
LOG.debug("To TableName:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(1).getText());
hlb.setTableName(nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(1).getText());
} else {
LOG.debug("To TableName:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(0).getText());
hlb.setTableName(nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(0).getText()); hlb.setTableName(nodeIn.getFirstChildWithType(HiveParser.TOK_TABNAME).getChild(0).getText());
} }
if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION) != null) { }
LOG.info("Create Table Location:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION).getText()); if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION) != null && nodeIn.getAncestor(HiveParser.TOK_WHERE) == null) {
LOG.debug("Create Table Location:: "+nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION).getText());
hlb.setTableLocation(nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION).getChild(0).getText()); hlb.setTableLocation(nodeIn.getFirstChildWithType(HiveParser.TOK_TABLELOCATION).getChild(0).getText());
} }
if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABCOLLIST) != null ) { if (nodeIn.getFirstChildWithType(HiveParser.TOK_TABCOLLIST) != null && nodeIn.getAncestor(HiveParser.TOK_WHERE) == null) {
ctdump((ASTNode)nodeIn.getFirstChildWithType(HiveParser.TOK_TABCOLLIST).getParent()); ctdump((ASTNode)nodeIn.getFirstChildWithType(HiveParser.TOK_TABCOLLIST).getParent());
hlb.setCreateColumns(createColumns); hlb.setCreateColumns(createColumns);
} }
...@@ -253,9 +273,9 @@ public class HiveLineageInfo implements NodeProcessor { ...@@ -253,9 +273,9 @@ public class HiveLineageInfo implements NodeProcessor {
boolean parseChild = true; boolean parseChild = true;
if (nodeIn.getType() == HiveParser.TOK_TABCOL) { if (nodeIn.getType() == HiveParser.TOK_TABCOL) {
CreateColumns crtClmns = hlb.new CreateColumns(); CreateColumns crtClmns = hlb.new CreateColumns();
LOG.info("Create Column Name:: "+nodeIn.getChild(0).getText()); LOG.debug("Create Column Name:: "+nodeIn.getChild(0).getText());
crtClmns.setColumnName(nodeIn.getChild(0).getText()); crtClmns.setColumnName(nodeIn.getChild(0).getText());
LOG.info("Create Column Type:: "+nodeIn.getChild(1).getText()); LOG.debug("Create Column Type:: "+nodeIn.getChild(1).getText());
crtClmns.setColumnType(nodeIn.getChild(1).getText()); crtClmns.setColumnType(nodeIn.getChild(1).getText());
createColumns.add(crtClmns); createColumns.add(crtClmns);
} }
...@@ -276,27 +296,27 @@ public class HiveLineageInfo implements NodeProcessor { ...@@ -276,27 +296,27 @@ public class HiveLineageInfo implements NodeProcessor {
public void fromTableDump(ASTNode nodeIn) { public void fromTableDump(ASTNode nodeIn) {
counter = 0; counter = 0;
ftdump(nodeIn); ftdump(nodeIn);
} }
/** /**
* Walks the fromTable Tree called by fromTableDump * Walks the fromTable Tree called by fromTableDump
*/ */
private void ftdump(ASTNode nodeIn) { private void ftdump(ASTNode nodeIn) {
boolean parseChild = true; boolean parseChild = true;
if (nodeIn.getType() == HiveParser.TOK_TABNAME && nodeIn.getParent().getType() == HiveParser.TOK_TABREF) { if (nodeIn.getType() == HiveParser.TOK_TABNAME && nodeIn.getParent().getType() == HiveParser.TOK_TABREF && nodeIn.getAncestor(HiveParser.TOK_WHERE) == null) {
SourceTables hlbSbls = hlb.new SourceTables(); SourceTables hlbSbls = hlb.new SourceTables();
if (nodeIn.getChildCount() == 2) { if (nodeIn.getChildCount() == 2) {
LOG.info("From DBName:: "+nodeIn.getChild(0).getText()); LOG.debug("From DBName:: "+nodeIn.getChild(0).getText());
hlbSbls.setDatabaseName(nodeIn.getChild(0).getText()); hlbSbls.setDatabaseName(nodeIn.getChild(0).getText());
LOG.info("From TableName:: "+nodeIn.getChild(1).getText()); LOG.debug("From TableName:: "+nodeIn.getChild(1).getText());
hlbSbls.setTableName(nodeIn.getChild(1).getText()); hlbSbls.setTableName(nodeIn.getChild(1).getText());
} else { } else {
LOG.info("From TableName:: "+nodeIn.getChild(0).getText()); LOG.debug("From TableName:: "+nodeIn.getChild(0).getText());
hlbSbls.setTableName(nodeIn.getChild(0).getText()); hlbSbls.setTableName(nodeIn.getChild(0).getText());
} }
if (nodeIn.getType() == HiveParser.TOK_TABNAME && nodeIn.getParent().getChild(1) != null) { if (nodeIn.getType() == HiveParser.TOK_TABNAME && nodeIn.getParent().getChild(1) != null) {
LOG.info("From DB/Table Alias:: "+nodeIn.getParent().getChild(1).getText()); LOG.debug("From DB/Table Alias:: "+nodeIn.getParent().getChild(1).getText());
hlbSbls.setTableAlias(nodeIn.getParent().getChild(1).getText()); hlbSbls.setTableAlias(nodeIn.getParent().getChild(1).getText());
} }
sourceTables.add(hlbSbls); sourceTables.add(hlbSbls);
...@@ -309,7 +329,7 @@ private void ftdump(ASTNode nodeIn) { ...@@ -309,7 +329,7 @@ private void ftdump(ASTNode nodeIn) {
} }
} }
} }
} }
/** /**
* Walks the column Tree called by processWalker * Walks the column Tree called by processWalker
...@@ -318,32 +338,37 @@ private void ftdump(ASTNode nodeIn) { ...@@ -318,32 +338,37 @@ private void ftdump(ASTNode nodeIn) {
public void columnTableDump(ASTNode nodeIn) { public void columnTableDump(ASTNode nodeIn) {
counter = 0; counter = 0;
clmnTdump(nodeIn); clmnTdump(nodeIn);
} }
/** /**
* Walks the columnDump Tree called by columnTableDump * Walks the columnDump Tree called by columnTableDump
*/ */
private void clmnTdump(ASTNode nodeIn) { private void clmnTdump(ASTNode nodeIn) {
boolean parseChild = true; boolean parseChild = true;
if (nodeIn.getType() == HiveParser.TOK_TABLE_OR_COL && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null ) { if (nodeIn.getType() == HiveParser.TOK_TABLE_OR_COL && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null && !(nodeIn.hasAncestor(HiveParser.TOK_WHERE))) {
QueryColumns qclmns = hlb.new QueryColumns(); QueryColumns qclmns = hlb.new QueryColumns();
if (nodeIn.getAncestor(HiveParser.TOK_FUNCTION) != null && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null) { if (nodeIn.getAncestor(HiveParser.TOK_FUNCTION) != null && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null) {
LOG.info("Function Query:: "+nodeIn.getAncestor(HiveParser.TOK_FUNCTION).getChild(0).getText()); LOG.debug("Function Query:: "+nodeIn.getAncestor(HiveParser.TOK_FUNCTION).getChild(0).getText());
qclmns.setColumnFunction(nodeIn.getAncestor(HiveParser.TOK_FUNCTION).getChild(0).getText()); qclmns.setColumnFunction(nodeIn.getAncestor(HiveParser.TOK_FUNCTION).getChild(0).getText());
} }
if (nodeIn.getAncestor(HiveParser.TOK_FUNCTIONDI) != null && nodeIn.getAncestor(HiveParser.TOK_SELEXPR) != null) {
LOG.debug("Function Distinct Query:: "+nodeIn.getAncestor(HiveParser.TOK_FUNCTIONDI).getChild(0).getText());
qclmns.setColumnDistinctFunction(nodeIn.getAncestor(HiveParser.TOK_FUNCTIONDI).getChild(0).getText());
}
if (nodeIn.getParent().getText().equalsIgnoreCase(".")) { if (nodeIn.getParent().getText().equalsIgnoreCase(".")) {
LOG.info("Table Name/Alias:: "+nodeIn.getChild(0).getText()); LOG.debug("Table Name/Alias:: "+nodeIn.getChild(0).getText());
qclmns.setTbAliasOrName(nodeIn.getChild(0).getText()); qclmns.setTbAliasOrName(nodeIn.getChild(0).getText());
LOG.info("Column:: "+nodeIn.getParent().getChild(1).getText()); LOG.debug("Column:: "+nodeIn.getParent().getChild(1).getText());
qclmns.setColumnName(nodeIn.getParent().getChild(1).getText()); qclmns.setColumnName(nodeIn.getParent().getChild(1).getText());
if (nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1) != null) { if (nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1) != null) {
LOG.info("Column Alias:: "+nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1).getText()); LOG.debug("Column Alias:: "+nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1).getText());
qclmns.setColumnAlias(nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1).getText()); qclmns.setColumnAlias(nodeIn.getAncestor(HiveParser.TOK_SELEXPR).getChild(1).getText());
} }
} else { } else {
LOG.info("Column:: "+nodeIn.getChild(0).getText()); LOG.debug("Column:: "+nodeIn.getChild(0).getText());
qclmns.setColumnName(nodeIn.getChild(0).getText()); qclmns.setColumnName(nodeIn.getChild(0).getText());
if (nodeIn.getParent().getChild(1) != null) { if ((nodeIn.getParent().getChild(1) != null && nodeIn.getParent().getChild(1).getType() != HiveParser.TOK_TABLE_OR_COL)) {
LOG.info("Column Alias:: "+nodeIn.getParent().getChild(1).getText()); LOG.debug("Column Alias:: "+nodeIn.getParent().getChild(1).getText());
qclmns.setColumnAlias(nodeIn.getParent().getChild(1).getText()); qclmns.setColumnAlias(nodeIn.getParent().getChild(1).getText());
} }
} }
...@@ -376,9 +401,55 @@ private void ftdump(ASTNode nodeIn) { ...@@ -376,9 +401,55 @@ private void ftdump(ASTNode nodeIn) {
ParseDriver pd = new ParseDriver(); ParseDriver pd = new ParseDriver();
ASTNode tree = pd.parse(query); ASTNode tree = pd.parse(query);
LOG.info("DUMP TREE: "+tree.dump()); LOG.info("DUMP TREE: "+tree.dump());
if (tree.getChild(0).getType() == HiveParser.TOK_DROPDATABASE ) {
hlb.setAction("drop_database");
}
if (tree.getChild(0).getType() == HiveParser.TOK_CREATEDATABASE) {
hlb.setAction("create_database");
}
if (tree.getChild(0).getType() == HiveParser.TOK_CREATETABLE) {
hlb.setAction("create_table");
}
if (tree.getChild(0).getType() == HiveParser.TOK_CREATEVIEW ) {
hlb.setAction("create_view");
}
if (tree.getChild(0).getType() == HiveParser.TOK_DROPTABLE) {
hlb.setAction("drop_table");
}
if (tree.getChild(0).getType() == HiveParser.TOK_INSERT) {
hlb.setAction("insert");
}
if (tree.getChild(0).getType() == HiveParser.TOK_INSERT_INTO) {
hlb.setAction("insert_into");
}
if (tree.getChild(0).getType() == HiveParser.TOK_DROPVIEW) {
hlb.setAction("drop_view");
}
if (tree.getChild(0).getType() == HiveParser.TOK_SHOWDATABASES) {
hlb.setAction("show_databases");
}
if (tree.getChild(0).getType() == HiveParser.TOK_SHOWTABLES) {
hlb.setAction("show_tables");
}
if (tree.getChild(0).getType() == HiveParser.TOK_ALTERVIEW_RENAME) {
hlb.setAction("alter_view_rename");
}
if (tree.getChild(0).getType() == HiveParser.TOK_ALTERTABLE_RENAME) {
hlb.setAction("alter_table_rename");
}
if (tree.getChild(0).getType() == HiveParser.TOK_ANALYZE) {
hlb.setAction("analyze");
}
if (tree.getChild(0).getType() == HiveParser. TOK_QUERY) {
hlb.setAction("select");
}
while ((tree.getToken() == null) && (tree.getChildCount() > 0)) { while ((tree.getToken() == null) && (tree.getChildCount() > 0)) {
tree = (ASTNode) tree.getChild(0); tree = (ASTNode) tree.getChild(0);
} }
sourceTables = new ArrayList<SourceTables>();
queryColumns = new ArrayList<QueryColumns>();
/* /*
* initialize Event Processor and dispatcher. * initialize Event Processor and dispatcher.
...@@ -396,8 +467,15 @@ private void ftdump(ASTNode nodeIn) { ...@@ -396,8 +467,15 @@ private void ftdump(ASTNode nodeIn) {
ArrayList<Node> topNodes = new ArrayList<Node>(); ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.add(tree); topNodes.add(tree);
ogw.startWalking(topNodes, null); ogw.startWalking(topNodes, null);
if (!(sourceTables.isEmpty())) {
this.hlb.setSourceTables(sourceTables);
} }
if (!(queryColumns.isEmpty())) {
this.hlb.setQueryColumns(queryColumns);
}
}
//Main method to run tests and return json/gson feed from a query //Main method to run tests and return json/gson feed from a query
public static void main(String[] args) throws IOException, ParseException, public static void main(String[] args) throws IOException, ParseException,
......
metadata-bridge-parent/metadata-bridge-hive/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/Hook.java
...@@ -87,25 +87,29 @@ public class Hook implements ExecuteWithHookContext { ...@@ -87,25 +87,29 @@ public class Hook implements ExecuteWithHookContext {
executionEngine="mr"; executionEngine="mr";
} }
hiveId = sess.getSessionId(); hiveId = sess.getSessionId();
String defaultdb = null;
switch(hookContext.getHookType()) { switch(hookContext.getHookType()) {
case PRE_EXEC_HOOK: case PRE_EXEC_HOOK:
Set<ReadEntity> db = hookContext.getInputs(); Set<ReadEntity> db_pre = hookContext.getInputs();
for (Object o : db) { for (Object o : db_pre) {
LOG.debug("DB:Table="+o.toString()); LOG.debug("DB:Table="+o.toString());
defaultdb = o.toString().split("@")[0];
} }
currentTime = System.currentTimeMillis(); currentTime = System.currentTimeMillis();
HiveLineageInfo lep_pre = new HiveLineageInfo(); HiveLineageInfo lep_pre = new HiveLineageInfo();
lep_pre.getLineageInfo(query); lep_pre.getLineageInfo(query);
hlb=lep_pre.getHLBean(); hlb=lep_pre.getHLBean();
hlb.setDatabaseName(defaultdb);
hlb.setQueryEndTime(Long.toString(currentTime)); hlb.setQueryEndTime(Long.toString(currentTime));
hlb.setQueryId(queryId); hlb.setQueryId(queryId);
hlb.setQuery(query); hlb.setQuery(query);
hlb.setUser(user); hlb.setUser(user);
hlb.setHiveId(hiveId); hlb.setHiveId(hiveId);
hlb.setSuccess(false); hlb.setSuccess(false);
if (executionEngine != null ) {
if (executionEngine.equalsIgnoreCase("mr")) { if (executionEngine.equalsIgnoreCase("mr")) {
hlb.setExecutionEngine("mapreduce"); hlb.setExecutionEngine("mapreduce");
} }
...@@ -115,15 +119,24 @@ public class Hook implements ExecuteWithHookContext { ...@@ -115,15 +119,24 @@ public class Hook implements ExecuteWithHookContext {
if (executionEngine.equalsIgnoreCase("spark")) { if (executionEngine.equalsIgnoreCase("spark")) {
hlb.setExecutionEngine("spark"); hlb.setExecutionEngine("spark");
} }
} else {
hlb.setExecutionEngine("local");
}
hlb.setQueryStartTime(queryStartTime); hlb.setQueryStartTime(queryStartTime);
fireAndForget(hookContext.getConf(), hlb, queryId); fireAndForget(hookContext.getConf(), hlb, queryId);
break; break;
case POST_EXEC_HOOK: case POST_EXEC_HOOK:
Set<ReadEntity> db_post = hookContext.getInputs();
for (Object o : db_post) {
LOG.debug("DB:Table="+o.toString());
defaultdb = o.toString().split("@")[0];
}
currentTime = System.currentTimeMillis(); currentTime = System.currentTimeMillis();
HiveLineageInfo lep_post = new HiveLineageInfo(); HiveLineageInfo lep_post = new HiveLineageInfo();
lep_post.getLineageInfo(query); lep_post.getLineageInfo(query);
hlb=lep_post.getHLBean(); hlb=lep_post.getHLBean();
hlb.setDatabaseName(defaultdb);
hlb.setQueryEndTime(Long.toString(currentTime)); hlb.setQueryEndTime(Long.toString(currentTime));
hlb.setQueryId(queryId); hlb.setQueryId(queryId);
hlb.setQuery(query); hlb.setQuery(query);
...@@ -131,6 +144,7 @@ public class Hook implements ExecuteWithHookContext { ...@@ -131,6 +144,7 @@ public class Hook implements ExecuteWithHookContext {
hlb.setQueryStartTime(queryStartTime); hlb.setQueryStartTime(queryStartTime);
hlb.setSuccess(true); hlb.setSuccess(true);
hlb.setHiveId(hiveId); hlb.setHiveId(hiveId);
if (executionEngine != null ) {
if (executionEngine.equalsIgnoreCase("mr")) { if (executionEngine.equalsIgnoreCase("mr")) {
hlb.setExecutionEngine("mapreduce"); hlb.setExecutionEngine("mapreduce");
} }
...@@ -140,13 +154,22 @@ public class Hook implements ExecuteWithHookContext { ...@@ -140,13 +154,22 @@ public class Hook implements ExecuteWithHookContext {
if (executionEngine.equalsIgnoreCase("spark")) { if (executionEngine.equalsIgnoreCase("spark")) {
hlb.setExecutionEngine("spark"); hlb.setExecutionEngine("spark");
} }
} else {
hlb.setExecutionEngine("local");
}
fireAndForget(hookContext.getConf(), hlb, queryId); fireAndForget(hookContext.getConf(), hlb, queryId);
break; break;
case ON_FAILURE_HOOK: case ON_FAILURE_HOOK:
Set<ReadEntity> db_fail = hookContext.getInputs();
for (Object o : db_fail) {
LOG.debug("DB:Table="+o.toString());
defaultdb = o.toString().split("@")[0];
}
HiveLineageInfo lep_failed = new HiveLineageInfo(); HiveLineageInfo lep_failed = new HiveLineageInfo();
lep_failed.getLineageInfo(query); lep_failed.getLineageInfo(query);
hlb=lep_failed.getHLBean(); hlb=lep_failed.getHLBean();
hlb.setDatabaseName(defaultdb);
hlb.setQueryEndTime(Long.toString(currentTime)); hlb.setQueryEndTime(Long.toString(currentTime));
hlb.setQueryId(queryId); hlb.setQueryId(queryId);
hlb.setQuery(query); hlb.setQuery(query);
...@@ -155,6 +178,7 @@ public class Hook implements ExecuteWithHookContext { ...@@ -155,6 +178,7 @@ public class Hook implements ExecuteWithHookContext {
hlb.setSuccess(false); hlb.setSuccess(false);
hlb.setFailed(true); hlb.setFailed(true);
hlb.setHiveId(hiveId); hlb.setHiveId(hiveId);
if (executionEngine != null ) {
if (executionEngine.equalsIgnoreCase("mr")) { if (executionEngine.equalsIgnoreCase("mr")) {
hlb.setExecutionEngine("mapreduce"); hlb.setExecutionEngine("mapreduce");
} }
...@@ -164,6 +188,9 @@ public class Hook implements ExecuteWithHookContext { ...@@ -164,6 +188,9 @@ public class Hook implements ExecuteWithHookContext {
if (executionEngine.equalsIgnoreCase("spark")) { if (executionEngine.equalsIgnoreCase("spark")) {
hlb.setExecutionEngine("spark"); hlb.setExecutionEngine("spark");
} }
} else {
hlb.setExecutionEngine("local");
}
fireAndForget(hookContext.getConf(), hlb, queryId); fireAndForget(hookContext.getConf(), hlb, queryId);
break; break;
default: default:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment