Merge branch 'master' of https://github.com/hortonworks/metadata.git

bridge/pom.xml

@@ -23,5 +23,10 @@
   		<version>4.10</version>
   		<scope>test</scope>
   	</dependency>
+  	<dependency>
+  		<groupId>org.apache.hadoop.metadata</groupId>
+  		<artifactId>metadata-typesystem</artifactId>
+  		<version>0.1-incubating-SNAPSHOT</version>
+  	</dependency>
   </dependencies>
 </project>
\ No newline at end of file

bridge/src/main/java/org/apache/hadoop/metadata/bridge/Bridge.java

+package org.apache.hadoop.metadata.bridge;
+
+import org.apache.hadoop.metadata.types.AttributeDefinition;
+import org.apache.hadoop.metadata.types.ClassType;
+import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
+import org.apache.hadoop.metadata.types.TypeSystem;
+
+import com.google.common.collect.ImmutableList;
+
+public interface Bridge {
+	
+	boolean defineBridgeTypes(TypeSystem ts);
+	
+}

bridge/src/main/java/org/apache/hadoop/metadata/bridge/BridgeAssistant.java

+package org.apache.hadoop.metadata.bridge;
+
+import org.apache.hadoop.metadata.types.AttributeDefinition;
+import org.apache.hadoop.metadata.types.ClassType;
+import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
+import org.apache.hadoop.metadata.types.TypeSystem;
+
+import com.google.common.collect.ImmutableList;
+
+public class BridgeAssistant {
+
+	protected HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);}
+
+}

bridge/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java

+package org.apache.hadoop.metadata.bridge;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+public class BridgeException extends MetaException {
+
+	/**
+	 * 
+	 */
+	private static final long serialVersionUID = -384401342591560473L;
+
+}

bridge/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java

+package org.apache.hadoop.metadata.bridge;
+
+import org.apache.hadoop.metadata.types.TypeSystem;
+
+
+public class BridgeManager {
+	TypeSystem ts;
+	
+	BridgeManager(TypeSystem ts){
+		this.ts = ts;
+	}
+	
+	
+}

bridge/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/HiveLineageBridge.java

+package org.apache.hadoop.metadata.bridge.hivelineage;
+
+import org.apache.hadoop.metadata.bridge.Bridge;
+import org.apache.hadoop.metadata.types.TypeSystem;
+
+public class HiveLineageBridge implements Bridge {
+
+	@Override
+	public boolean defineBridgeTypes(TypeSystem ts) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+}

typesystem/src/main/java/org/apache/hadoop/metadata/bridge/hive/HiveMetaImporter.java → bridge/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveMetaImporter.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.bridge.hive;
+package org.apache.hadoop.metadata.bridge.hivestructure;
 
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -29,8 +11,9 @@ import org.apache.hadoop.hive.metastore.api.UnknownDBException;
 import org.apache.hadoop.hive.metastore.api.UnknownTableException;
 import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.MetadataService;
-import org.apache.hadoop.metadata.Struct;
-import org.apache.hadoop.metadata.types.StructType;
+import org.apache.hadoop.metadata.Referenceable;
+import org.apache.hadoop.metadata.storage.RepositoryException;
+import org.apache.hadoop.metadata.types.ClassType;
 import org.apache.thrift.TException;
 /*
  * Initial pass at one time importer TODO - needs re-write
@@ -38,7 +21,7 @@ import org.apache.thrift.TException;
 
 
 public class HiveMetaImporter {
-/*	
+	
 	private static HiveMetaStoreClient msc;
 	private static MetadataService ms;
 	
@@ -55,25 +38,48 @@ public class HiveMetaImporter {
 	}
 	
 	public static boolean fullImport(){
+		try{
+			databasesImport();
+			for (String dbName : msc.getAllDatabases()){
+				tablesImport(dbName);
+				for(String tbName : msc.getAllTables(dbName)){
+					fieldsImport(dbName,tbName);
+				}
+			return true;
+			}
+		}catch(MetaException me){
+			me.printStackTrace();
+		}catch(RepositoryException re){
+			re.printStackTrace();
+		}
+		
 		return false;
 	}
 	
-	public static boolean databaseImport() throws MetaException{
-        StructType structType = null;
+	public static boolean databasesImport() throws MetaException, RepositoryException{
+		ClassType classType = null;
 		try {
-			structType = (StructType) ms.getTypeSystem().getDataType(HiveBridge.STRUCT_TYPE_DB);
+			classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.DB_CLASS_TYPE);
 		} catch (MetadataException e1) {
 			e1.printStackTrace();
 		}
 		for(String dbName : msc.getAllDatabases()){
+			databaseImport(dbName);
+		}
+		return true;
+	}
+	
+	public static boolean databaseImport(String dbName) throws MetaException, RepositoryException{
 			try {
 				Database db = msc.getDatabase(dbName);
-				Struct s = new Struct(structType.getName());
-				s.set("DESC", db.getDescription());
-				s.set("DB_LOCATION_URI", db.getLocationUri());
-				s.set("NAME", db.getName());
-				if(db.isSetOwnerType()){s.set("OWNER_TYPE", db.getOwnerType());}
-				if(db.isSetOwnerName()){s.set("OWNER_NAME", db.getOwnerName());}
+				Referenceable dbRef = new Referenceable(HiveStructureBridge.DB_CLASS_TYPE);
+				dbRef.set("DESC", db.getDescription());
+				dbRef.set("DB_LOCATION_URI", db.getLocationUri());
+				dbRef.set("NAME", db.getName());
+				if(db.isSetOwnerType()){dbRef.set("OWNER_TYPE", db.getOwnerType());}
+				if(db.isSetOwnerName()){dbRef.set("OWNER_NAME", db.getOwnerName());}
+				
+				ms.getRepository().create(dbRef);
 			} catch (NoSuchObjectException e) {
 				// TODO Auto-generated catch block
 				e.printStackTrace();
@@ -81,29 +87,35 @@ public class HiveMetaImporter {
 				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
-			
-		}
 		return true;
 	}
 	
-	public static boolean tableImport(String dbName) throws MetaException{
-		StructType structType = null;
+	public static boolean tablesImport(String dbName) throws MetaException, RepositoryException{
+		ClassType classType = null;
 		try {
-			structType = (StructType) ms.getTypeSystem().getDataType(HiveBridge.STRUCT_TYPE_TB);
+			classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.TB_CLASS_TYPE);
 		} catch (MetadataException e1) {
 			e1.printStackTrace();
 		}
 		for(String tbName : msc.getAllTables(dbName)){
+		tableImport(dbName, tbName);
+		}
+		return true;
+	}
+	
+	public static boolean tableImport(String dbName, String tbName) throws MetaException, RepositoryException{
 			try {
 				Table tb = msc.getTable(dbName, tbName);
-				Struct s = new Struct(structType.getName());
-				s.set("CREATE_TIME", tb.getCreateTime());
-				s.set("LAST_ACCESS_TIME", tb.getLastAccessTime());
-				s.set("OWNER", tb.getOwner());
-				s.set("TBL_NAME", tb.getTableName());
-				s.set("TBL_TYPE", tb.getTableType());
-				if(tb.isSetViewExpandedText()){s.set("VIEW_EXPANDED_TEXT", tb.getViewExpandedText());}
-				if(tb.isSetViewOriginalText()){s.set("VIEW_ORIGINAL_TEXT", tb.getViewOriginalText());}
+				Referenceable tbRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE);
+				tbRef.set("CREATE_TIME", tb.getCreateTime());
+				tbRef.set("LAST_ACCESS_TIME", tb.getLastAccessTime());
+				tbRef.set("OWNER", tb.getOwner());
+				tbRef.set("TBL_NAME", tb.getTableName());
+				tbRef.set("TBL_TYPE", tb.getTableType());
+				if(tb.isSetViewExpandedText()){tbRef.set("VIEW_EXPANDED_TEXT", tb.getViewExpandedText());}
+				if(tb.isSetViewOriginalText()){tbRef.set("VIEW_ORIGINAL_TEXT", tb.getViewOriginalText());}
+
+				ms.getRepository().create(tbRef);
 			} catch (NoSuchObjectException e) {
 				// TODO Auto-generated catch block
 				e.printStackTrace();
@@ -111,25 +123,49 @@ public class HiveMetaImporter {
 				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
-			
-		}
 		return true;
 	}
 	
-	public static boolean fieldImport(String dbName, String tbName) throws MetaException{
-		StructType structType = null;
+	public static boolean fieldsImport (String dbName, String tbName) throws MetaException, RepositoryException{
+		ClassType classType = null;
 		try {
-			structType = (StructType) ms.getTypeSystem().getDataType(HiveBridge.STRUCT_TYPE_FD);
+			classType = ms.getTypeSystem().getDataType(ClassType.class, HiveStructureBridge.FD_CLASS_TYPE);
 		} catch (MetadataException e1) {
 			e1.printStackTrace();
 		}
 		try {
 			for(FieldSchema fs : msc.getFields(dbName, tbName)){
-				Struct s = new Struct(structType.getName());
-				if(fs.isSetComment()){s.set("COMMENT", fs.getName());}
-				s.set("COLUMN_NAME", fs.getName());
-				s.set("TYPE_NAME", fs.getType());
-				
+				Referenceable fdRef = new Referenceable(HiveStructureBridge.FD_CLASS_TYPE);
+				if(fs.isSetComment()){fdRef.set("COMMENT", fs.getName());}
+				fdRef.set("COLUMN_NAME", fs.getName());
+				fdRef.set("TYPE_NAME", fs.getType());
+
+				ms.getRepository().create(fdRef);
+			}
+		} catch (UnknownTableException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (UnknownDBException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (TException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		return true;
+	}
+	
+	public static boolean fieldImport(String dbName, String tbName, String fdName) throws MetaException{
+		try {
+			for(FieldSchema fs : msc.getFields(dbName, tbName)){
+				if (fs.getName().equals(fs)){
+				Referenceable fdRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE);
+				if(fs.isSetComment()){fdRef.set("COMMENT", fs.getName());}
+				fdRef.set("COLUMN_NAME", fs.getName());
+				fdRef.set("TYPE_NAME", fs.getType());
+				//SaveObject to MS Backend
+				return true;
+				}
 			}
 		} catch (UnknownTableException e) {
 			// TODO Auto-generated catch block
@@ -143,5 +179,5 @@ public class HiveMetaImporter {
 		}
 		return true;
 	}
-	*/
+	
 }

typesystem/src/main/java/org/apache/hadoop/metadata/bridge/hive/HiveBridge.java → bridge/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.bridge.hive;
+package org.apache.hadoop.metadata.bridge.hivestructure;
 
 import java.util.ArrayList;
 
+import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.bridge.Bridge;
-import org.apache.hadoop.metadata.bridge.BridgeLoad;
+import org.apache.hadoop.metadata.bridge.BridgeAssistant;
 import org.apache.hadoop.metadata.types.AttributeDefinition;
+import org.apache.hadoop.metadata.types.ClassType;
+import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
 import org.apache.hadoop.metadata.types.Multiplicity;
-import org.apache.hadoop.metadata.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.bridge.Bridge;
-import org.apache.hadoop.metadata.bridge.BridgeLoad;
-import org.apache.hadoop.metadata.types.AttributeDefinition;
-import org.apache.hadoop.metadata.types.Multiplicity;
-import org.apache.hadoop.metadata.types.StructTypeDefinition;
+import org.apache.hadoop.metadata.types.TypeSystem;
 
-public class HiveBridge implements Bridge {
-	
-	public static final String STRUCT_TYPE_DB = "HIVE_DB_STRUCT";
-    public static final String STRUCT_TYPE_TB = "HIVE_TB_STRUCT";
-    public static final String STRUCT_TYPE_FD = "HIVE_FD_STRUCT";
-	
-	public ArrayList<StructTypeDefinition> getStructTypeDefinitions() {
-		ArrayList<StructTypeDefinition> stds = new ArrayList<StructTypeDefinition>();
-		stds.add(new StructTypeDefinition(STRUCT_TYPE_DB,
-		        //new AttributeDefinition("DB_ID", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
-		        new AttributeDefinition[]{
-			        new AttributeDefinition("DESC", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
-			        new AttributeDefinition("DB_LOCATION_URI", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
-					new AttributeDefinition("NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
-					new AttributeDefinition("OWNER_TYPE", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
-					new AttributeDefinition("OWNER_NAME", "STRING_TYPE", Multiplicity.OPTIONAL, false, null)
-			        }
-				)
-		);
-		stds.add(new StructTypeDefinition(STRUCT_TYPE_TB, 
-				//new AttributeDefinition("TBL_ID", "INT_TYPE", Multiplicity.REQUIRED, false, null),
-				
+
+public class HiveStructureBridge extends BridgeAssistant implements Bridge{
+
+	static final String DB_CLASS_TYPE = "HiveDatabase";
+	static final String TB_CLASS_TYPE = "HiveTable";
+	static final String FD_CLASS_TYPE = "HiveField";
+	@Override
+	public boolean defineBridgeTypes(TypeSystem ts) {
+		ArrayList<HierarchicalTypeDefinition<?>> al = new ArrayList<HierarchicalTypeDefinition<?>>();
+		
+		try{
+		HierarchicalTypeDefinition<ClassType> databaseClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",DB_CLASS_TYPE, null,
+			new AttributeDefinition[]{
+		        new AttributeDefinition("DESC", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
+		        new AttributeDefinition("DB_LOCATION_URI", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+				new AttributeDefinition("NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+				new AttributeDefinition("OWNER_TYPE", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
+				new AttributeDefinition("OWNER_NAME", "STRING_TYPE", Multiplicity.OPTIONAL, false, null)
+		    }
+        );
+		
+		HierarchicalTypeDefinition<ClassType> tableClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",TB_CLASS_TYPE, null,
 				new AttributeDefinition[]{
-					new AttributeDefinition("CREATE_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null),
-					new AttributeDefinition("LAST_ACCESS_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null),
-					new AttributeDefinition("OWNER", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
-					new AttributeDefinition("TBL_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
-					new AttributeDefinition("TBL_TYPE", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
-					new AttributeDefinition("VIEW_EXPANDED_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
-					new AttributeDefinition("VIEW_ORIGINAL_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null)
-					}
-				)
+				new AttributeDefinition("CREATE_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null),
+				new AttributeDefinition("LAST_ACCESS_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null),
+				new AttributeDefinition("OWNER", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+				new AttributeDefinition("TBL_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+				new AttributeDefinition("TBL_TYPE", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+				new AttributeDefinition("VIEW_EXPANDED_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
+				new AttributeDefinition("VIEW_ORIGINAL_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null)
+				}
 		);
-		stds.add(new StructTypeDefinition(STRUCT_TYPE_FD,
-				//new AttributeDefinition("CD_ID", "INT_TYPE", Multiplicity.REQUIRED, false, null),
+		
+		HierarchicalTypeDefinition<ClassType> columnClassTypeDef = new HierarchicalTypeDefinition<ClassType>("ClassType",FD_CLASS_TYPE, null,
 				new AttributeDefinition[]{
-					new AttributeDefinition("COMMENT", "INT_TYPE", Multiplicity.OPTIONAL, false, null),
-					new AttributeDefinition("COLUMN_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
-					new AttributeDefinition("TYPE_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null)
-					}
-				)
+				new AttributeDefinition("COMMENT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
+				new AttributeDefinition("COLUMN_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
+				new AttributeDefinition("TYPE_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null)
+				}
 		);
-		return stds;
-	}
+		
+		}catch(ClassNotFoundException e){
+			e.printStackTrace();
+		}
+		
+		for (HierarchicalTypeDefinition htd : al){
+			try {
+				ts.defineClassType(htd);
+			} catch (MetadataException e) {
+				System.out.println(htd.hierarchicalMetaTypeName + "could not be added to the type system");
+				e.printStackTrace();
+			}
+		}
+		
+		
 
-	public boolean submitLoad(BridgeLoad load) {
-		// TODO Auto-generated method stub
 		return false;
 	}
-}
+
\ No newline at end of file
+}

typesystem/src/main/java/org/apache/hadoop/metadata/bridge/Bridge.java

-package org.apache.hadoop.metadata.bridge;
-
-import java.util.ArrayList;
-
-import org.apache.hadoop.metadata.types.StructType;
-import org.apache.hadoop.metadata.types.StructTypeDefinition;
-	/*
-	 * Interface for creating Bridges
-	 */
-
-public interface Bridge {
-
-	ArrayList<StructTypeDefinition> structs = new ArrayList<StructTypeDefinition>();
-	
-	public ArrayList<StructTypeDefinition> getStructTypeDefinitions();
-	
-	public boolean submitLoad(BridgeLoad load);
-}

typesystem/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java

-package org.apache.hadoop.metadata.bridge;
-
-public class BridgeException extends Exception {
-
-}

typesystem/src/main/java/org/apache/hadoop/metadata/bridge/BridgeListener.java

-package org.apache.hadoop.metadata.bridge;
-
-public class BridgeListener {
-	/*
-	 * Used to receive Json Payloaded Details from the remote bridge components
-	 */
-	
-}

typesystem/src/main/java/org/apache/hadoop/metadata/bridge/BridgeLoad.java

-package org.apache.hadoop.metadata.bridge;
-
-public class BridgeLoad {
-	/*
-	 * The BridgeLoad is a bean that the details of the JSON Payload from the Listner are converted into for a unified lense for the Bridge.
-	 */
-	
-	String loadType;
-	String timeStamp;
-	
-}

typesystem/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java

-package org.apache.hadoop.metadata.bridge;
-	/*
-	 * This will be the primary service for the Bridges (Will handle pushing type definitions into the type system and pushing entities to the repository system for Bridges)
-	 */
-
-import org.apache.hadoop.metadata.ITypedStruct;
-import org.apache.hadoop.metadata.ITypedStruct;
-import org.apache.hadoop.metadata.types.TypeSystem;
-
-import com.google.gson.JsonObject;
-
-public class BridgeManager {
-
-	private TypeSystem ts;
-	
-	public BridgeManager(TypeSystem ts){
-		this.ts = ts;
-	}
-		
-	protected BridgeLoad convertToBridgeLoad(JsonObject jo){
-		return null;
-	}
-	
-	public ITypedStruct convertToIStruct(BridgeLoad bl, Bridge b){
-		return null;
-	}
-	
-}