Added some integration between bridge and hook

metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/HiveLineageBridge.java

 package org.apache.hadoop.metadata.bridge.hivelineage;
 
+import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.bridge.Bridge;
+import org.apache.hadoop.metadata.types.AttributeDefinition;
+import org.apache.hadoop.metadata.types.ClassType;
+import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
+import org.apache.hadoop.metadata.types.Multiplicity;
 import org.apache.hadoop.metadata.types.TypeSystem;
 
 public class HiveLineageBridge implements Bridge {
 
-	@Override
-	public boolean defineBridgeTypes(TypeSystem ts) {
-		// TODO Auto-generated method stub
-		return false;
-	}
+  static final String LINEAGE_CLASS_TYPE = "HiveLineage";
 
+  @Override
+  public boolean defineBridgeTypes(TypeSystem ts) {
+    try {
+      HierarchicalTypeDefinition<ClassType> lineageClassTypeDef =
+          new HierarchicalTypeDefinition<ClassType>(
+              "ClassType",
+              LINEAGE_CLASS_TYPE,
+              null,
+              new AttributeDefinition[] {
+                  new AttributeDefinition("QUERY_ID", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("HIVE_ID", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("USER", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("QUERY_START_TIME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("QUERY_END_TIME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("QUERY", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("TABLE_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("TABLE_LOCATION", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("SUCCESS", "BOOLEAN_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("FAILED", "BOOLEAN_TYPE", Multiplicity.REQUIRED, false, null),
+                  new AttributeDefinition("EXECUTION_ENGINE", "STRING_TYPE", Multiplicity.REQUIRED, false, null)
+                  });
+      
+      // TODO - assess these
+      /*
+       * Not sure what to do with these attributes - wouldn't tables and columns be linked to
+       * Hive Structure instances?
+       * 
+      ArrayList<SourceTables> sourceTables;
+      ArrayList<QueryColumns> queryColumns;
+      ArrayList<WhereClause> whereClause;
+      ArrayList<CreateColumns> createColumns;
+      ArrayList<GroupBy> groupBy;
+      ArrayList<GroupBy> orderBy;*/
+      
+      ts.defineClassType(lineageClassTypeDef);
+      
+    } catch (ClassNotFoundException e) {
+      e.printStackTrace();
+    } catch (MetadataException e) {
+      e.printStackTrace();
+    }
+    
+    return false;
+  }
 }

metadata-bridge-parent/metadata-bridge-hive/pom.xml

@@ -8,9 +8,9 @@
   <artifactId>metadata-bridge-hive</artifactId>
   <dependencies>
   	<dependency>
-		<groupId>org.apache.hadoop.metadata</groupId>
-		<artifactId>metadata-bridge-core</artifactId>
-		<version>${project.version}</version>
+  		<groupId>com.google.code.gson</groupId>
+  		<artifactId>gson</artifactId>
+  		<version>2.2.2</version>
   	</dependency>
   	<dependency>
 		<groupId>org.apache.hive</groupId>

metadata-bridge-parent/metadata-bridge-hive/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/Hook.java

 package org.apache.hadoop.metadata.bridge.hivelineage.hook;
 
 
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.gson.Gson;
-
 import java.io.DataOutputStream;
-import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLEncoder;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
 import java.util.Set;
 
 import javax.net.ssl.HttpsURLConnection;
@@ -24,22 +17,16 @@ import javax.net.ssl.X509TrustManager;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.exec.ExplainTask;
-import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.tez.TezTask;
 import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
 import org.apache.hadoop.hive.ql.hooks.HookContext;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.util.StringUtils;
-//import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-//import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-//import org.apache.hadoop.yarn.client.api.TimelineClient;
-//import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
+import com.google.gson.Gson;
 
 /**
  * DGC Hook sends query + plan info to DGCCollector Service. To enable (hadoop 2.4 and up) set
@@ -50,6 +37,10 @@ public class Hook implements ExecuteWithHookContext {
 
   private static final Log LOG = LogFactory.getLog(Hook.class.getName());
   private HiveLineageBean hlb;
+  
+  private static final String METADATA_HOST = "localhost";
+  private static final int METADATA_PORT = 20810;
+  private static final String METADATA_PATH = "/entities/submit/HiveLineage";
 
   @Override
   public void run(HookContext hookContext) throws Exception {
@@ -76,7 +67,6 @@ public class Hook implements ExecuteWithHookContext {
             }
             if (numMrJobs > 0) {
             	executionEngine="mr";
-
             }
             hiveId = sess.getSessionId();
            
@@ -168,7 +158,7 @@ public class Hook implements ExecuteWithHookContext {
         }
  
   public void fireAndForget(Configuration conf, HiveLineageBean hookData, String queryId) throws Exception {
-			String postUri = "http://167.69.111.50:20810/HiveHookCollector/HookServlet"; 		
+			String postUri = String.format("http://%s:%i%s%s", METADATA_HOST, METADATA_PORT, METADATA_PATH); 		
 	  		if (conf.getTrimmed("hadoop.metadata.hive.hook.uri") != null) {
 		  		postUri = conf.getTrimmed("hadoop.metadata.hive.hook.uri");
 	  		} 

metadata-bridge-parent/metadata-bridge-hive/src/test/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/HiveLineageInfoTest.java

+package org.apache.hadoop.metadata.bridge.hivelineage.hook;
+
+import org.apache.hadoop.hive.ql.parse.ParseException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+import com.google.gson.Gson;
+
+public class HiveLineageInfoTest {
+	public static String parseQuery(String query) throws SemanticException,
+			ParseException {
+		HiveLineageInfo lep = new HiveLineageInfo();
+		lep.getLineageInfo(query);
+		Gson gson = new Gson();
+		String jsonOut = gson.toJson(lep.getHLBean());
+		return jsonOut;
+	}
+}