Skip to content

A
atlas
Commit 98d436c2 by Dennis Fusaro
Options

Moving Bridge to it's own project

parent b8308a31
Showing with 38 additions and 297 deletions
bridge/pom.xml
...@@ -23,5 +23,10 @@ ...@@ -23,5 +23,10 @@
<version>4.10</version> <version>4.10</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies> </dependencies>
</project> </project>
\ No newline at end of file
typesystem/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java → bridge/src/main/java/org/apache/hadoop/metadata/bridge/Bridge.java
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
public class BridgeException extends Exception { import org.apache.hadoop.metadata.types.TypeSystem;
public interface Bridge {
boolean defineBridgeTypes(TypeSystem ts);
} }
bridge/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java 0 → 100644
package org.apache.hadoop.metadata.bridge;
import org.apache.hadoop.metadata.types.TypeSystem;
public class BridgeManager {
TypeSystem ts;
BridgeManager(TypeSystem ts){
this.ts = ts;
}
}
bridge/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java 0 → 100644
package org.apache.hadoop.metadata.bridge.hivestructure;
import org.apache.hadoop.metadata.bridge.Bridge;
import org.apache.hadoop.metadata.types.TypeSystem;
public class HiveStructureBridge implements Bridge {
@Override
public boolean defineBridgeTypes(TypeSystem ts) {
// TODO Auto-generated method stub
return false;
}
}
typesystem/src/main/java/org/apache/hadoop/metadata/bridge/Bridge.java deleted 100644 → 0
package org.apache.hadoop.metadata.bridge;
import java.util.ArrayList;
import org.apache.hadoop.metadata.types.StructType;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
/*
* Interface for creating Bridges
*/
public interface Bridge {
ArrayList<StructTypeDefinition> structs = new ArrayList<StructTypeDefinition>();
public ArrayList<StructTypeDefinition> getStructTypeDefinitions();
public boolean submitLoad(BridgeLoad load);
}
typesystem/src/main/java/org/apache/hadoop/metadata/bridge/BridgeListener.java deleted 100644 → 0
package org.apache.hadoop.metadata.bridge;
public class BridgeListener {
/*
* Used to receive Json Payloaded Details from the remote bridge components
*/
}
typesystem/src/main/java/org/apache/hadoop/metadata/bridge/BridgeLoad.java deleted 100644 → 0
package org.apache.hadoop.metadata.bridge;
public class BridgeLoad {
/*
* The BridgeLoad is a bean that the details of the JSON Payload from the Listner are converted into for a unified lense for the Bridge.
*/
String loadType;
String timeStamp;
}
typesystem/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java deleted 100644 → 0
package org.apache.hadoop.metadata.bridge;
/*
* This will be the primary service for the Bridges (Will handle pushing type definitions into the type system and pushing entities to the repository system for Bridges)
*/
import org.apache.hadoop.metadata.ITypedStruct;
import org.apache.hadoop.metadata.ITypedStruct;
import org.apache.hadoop.metadata.types.TypeSystem;
import com.google.gson.JsonObject;
public class BridgeManager {
private TypeSystem ts;
public BridgeManager(TypeSystem ts){
this.ts = ts;
}
protected BridgeLoad convertToBridgeLoad(JsonObject jo){
return null;
}
public ITypedStruct convertToIStruct(BridgeLoad bl, Bridge b){
return null;
}
}
typesystem/src/main/java/org/apache/hadoop/metadata/bridge/hive/HiveBridge.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.bridge.hive;
import java.util.ArrayList;
import org.apache.hadoop.metadata.bridge.Bridge;
import org.apache.hadoop.metadata.bridge.BridgeLoad;
import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.bridge.Bridge;
import org.apache.hadoop.metadata.bridge.BridgeLoad;
import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
public class HiveBridge implements Bridge {
public static final String STRUCT_TYPE_DB = "HIVE_DB_STRUCT";
public static final String STRUCT_TYPE_TB = "HIVE_TB_STRUCT";
public static final String STRUCT_TYPE_FD = "HIVE_FD_STRUCT";
public ArrayList<StructTypeDefinition> getStructTypeDefinitions() {
ArrayList<StructTypeDefinition> stds = new ArrayList<StructTypeDefinition>();
stds.add(new StructTypeDefinition(STRUCT_TYPE_DB,
//new AttributeDefinition("DB_ID", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition[]{
new AttributeDefinition("DESC", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("DB_LOCATION_URI", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("OWNER_TYPE", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("OWNER_NAME", "STRING_TYPE", Multiplicity.OPTIONAL, false, null)
}
)
);
stds.add(new StructTypeDefinition(STRUCT_TYPE_TB,
//new AttributeDefinition("TBL_ID", "INT_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition[]{
new AttributeDefinition("CREATE_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("LAST_ACCESS_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("OWNER", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("TBL_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("TBL_TYPE", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("VIEW_EXPANDED_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("VIEW_ORIGINAL_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, false, null)
}
)
);
stds.add(new StructTypeDefinition(STRUCT_TYPE_FD,
//new AttributeDefinition("CD_ID", "INT_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition[]{
new AttributeDefinition("COMMENT", "INT_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("COLUMN_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("TYPE_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null)
}
)
);
return stds;
}
public boolean submitLoad(BridgeLoad load) {
// TODO Auto-generated method stub
return false;
}
}
\ No newline at end of file
typesystem/src/main/java/org/apache/hadoop/metadata/bridge/hive/HiveMetaImporter.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.bridge.hive;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.MetadataService;
import org.apache.hadoop.metadata.Struct;
import org.apache.hadoop.metadata.types.StructType;
import org.apache.thrift.TException;
/*
* Initial pass at one time importer TODO - needs re-write
*/
public class HiveMetaImporter {
/*
private static HiveMetaStoreClient msc;
private static MetadataService ms;
public HiveMetaImporter(MetadataService ms){
try {
this.ms = ms;
msc = new HiveMetaStoreClient(new HiveConf());
} catch (MetaException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static boolean fullImport(){
return false;
}
public static boolean databaseImport() throws MetaException{
StructType structType = null;
try {
structType = (StructType) ms.getTypeSystem().getDataType(HiveBridge.STRUCT_TYPE_DB);
} catch (MetadataException e1) {
e1.printStackTrace();
}
for(String dbName : msc.getAllDatabases()){
try {
Database db = msc.getDatabase(dbName);
Struct s = new Struct(structType.getName());
s.set("DESC", db.getDescription());
s.set("DB_LOCATION_URI", db.getLocationUri());
s.set("NAME", db.getName());
if(db.isSetOwnerType()){s.set("OWNER_TYPE", db.getOwnerType());}
if(db.isSetOwnerName()){s.set("OWNER_NAME", db.getOwnerName());}
} catch (NoSuchObjectException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (TException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return true;
}
public static boolean tableImport(String dbName) throws MetaException{
StructType structType = null;
try {
structType = (StructType) ms.getTypeSystem().getDataType(HiveBridge.STRUCT_TYPE_TB);
} catch (MetadataException e1) {
e1.printStackTrace();
}
for(String tbName : msc.getAllTables(dbName)){
try {
Table tb = msc.getTable(dbName, tbName);
Struct s = new Struct(structType.getName());
s.set("CREATE_TIME", tb.getCreateTime());
s.set("LAST_ACCESS_TIME", tb.getLastAccessTime());
s.set("OWNER", tb.getOwner());
s.set("TBL_NAME", tb.getTableName());
s.set("TBL_TYPE", tb.getTableType());
if(tb.isSetViewExpandedText()){s.set("VIEW_EXPANDED_TEXT", tb.getViewExpandedText());}
if(tb.isSetViewOriginalText()){s.set("VIEW_ORIGINAL_TEXT", tb.getViewOriginalText());}
} catch (NoSuchObjectException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (TException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return true;
}
public static boolean fieldImport(String dbName, String tbName) throws MetaException{
StructType structType = null;
try {
structType = (StructType) ms.getTypeSystem().getDataType(HiveBridge.STRUCT_TYPE_FD);
} catch (MetadataException e1) {
e1.printStackTrace();
}
try {
for(FieldSchema fs : msc.getFields(dbName, tbName)){
Struct s = new Struct(structType.getName());
if(fs.isSetComment()){s.set("COMMENT", fs.getName());}
s.set("COLUMN_NAME", fs.getName());
s.set("TYPE_NAME", fs.getType());
}
} catch (UnknownTableException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (UnknownDBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (TException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return true;
}
*/
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment