Skip to content

A
atlas
Commit 953ceabb by Ballistar13
Options

Finshed BridgeManager Unit Tests

Rigged BridgeManager and Bridge Properties together.
parent adca9c0c
Showing with 147 additions and 95 deletions
common/src/main/java/org/apache/hadoop/metadata/bridge/AEnitityBean.java
package org.apache.hadoop.metadata.bridge;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.Map.Entry;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.Referenceable;
import org.apache.hadoop.metadata.types.AttributeInfo;
public abstract class AEnitityBean {
public final Referenceable convertToReferencable() throws IllegalArgumentException, IllegalAccessException{
Referenceable selfAware = new Referenceable(this.getClass().getSimpleName());
for(Field f : this.getClass().getFields()){
selfAware.set(f.getName(), f.get(this));
}
return selfAware;
}
public final <t extends AEnitityBean>Object convertFromITypedReferenceable(ITypedReferenceableInstance instance) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, BridgeException{
if(!instance.getTypeName().equals(this.getClass().getSimpleName())){
throw new BridgeException("ReferenceableInstance type not the same as bean");
}
Object retObj = this.getClass().newInstance();
for (Entry<String, AttributeInfo> e : instance.fieldMapping().fields.entrySet()){
try {
String convertedName = e.getKey().substring(0, 1).toUpperCase()+e.getKey().substring(1);
this.getClass().getMethod("set"+convertedName, Class.forName(e.getValue().dataType().getName())).invoke(this, instance.get(e.getKey()));
} catch (MetadataException | ClassNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
return retObj;
}
}
metadata-bridge-parent/metadata-bridge-core/pom.xml
......@@ -44,9 +44,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId>
<version>0.5.2</version>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/ABridge.java
package org.apache.hadoop.metadata.bridge;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
import javax.inject.Inject;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.Referenceable;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.storage.RepositoryException;
import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.AttributeInfo;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.Multiplicity;
......@@ -22,11 +26,10 @@ import com.google.common.collect.ImmutableList;
public abstract class ABridge implements IBridge {
protected ArrayList<Class<? extends AEnitityBean>> typeBeanClasses = new ArrayList<Class<? extends AEnitityBean>>();
MetadataRepository repo;
protected static final Logger LOG = LoggerFactory.getLogger("BridgeLogger");
protected static final Logger LOG = BridgeManager.LOG;
protected HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);}
public ArrayList<Class<? extends AEnitityBean>> getTypeBeanClasses() {
......@@ -38,13 +41,13 @@ public abstract class ABridge implements IBridge {
this.repo = repo;
}
public <t extends AEnitityBean>Object get(String id) throws RepositoryException {
public AEnitityBean get(String id) throws RepositoryException {
// get from the system by id (?)
ITypedReferenceableInstance ref = repo.getEntityDefinition(id);
// turn into a HiveLineageBean
try {
Class<AEnitityBean> c = getTypeBeanInListByName(ref.getTypeName());
return c.newInstance().convertFromITypedReferenceable(ref);
return this.convertFromITypedReferenceable(ref, getTypeBeanInListByName(ref.getTypeName()));
} catch (BridgeException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
// TODO Auto-generated catch block
e.printStackTrace();
......@@ -57,7 +60,7 @@ public abstract class ABridge implements IBridge {
ClassType type = TypeSystem.getInstance().getDataType(ClassType.class, bean.getClass().getSimpleName());
ITypedReferenceableInstance refBean = null;
try {
refBean = type.convert(bean.convertToReferencable(), Multiplicity.REQUIRED);
refBean = type.convert(this.convertToReferencable(bean), Multiplicity.REQUIRED);
String id = repo.createEntity(refBean, type.getName());
return id;
} catch (IllegalArgumentException | IllegalAccessException e) {
......@@ -99,6 +102,33 @@ public abstract class ABridge implements IBridge {
}
throw new BridgeException("No EntityBean Definition Found");
}
protected final <T extends AEnitityBean> Referenceable convertToReferencable(T o ) throws IllegalArgumentException, IllegalAccessException{
Referenceable selfAware = new Referenceable(o.getClass().getSimpleName());
for(Field f : o.getClass().getFields()){
selfAware.set(f.getName(), f.get(o));
}
return selfAware;
}
protected final <T extends AEnitityBean>T convertFromITypedReferenceable(ITypedReferenceableInstance instance, Class<? extends AEnitityBean> c) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, BridgeException{
if(!instance.getTypeName().equals(c.getSimpleName())){
throw new BridgeException("ReferenceableInstance type not the same as bean");
}
Object retObj = this.getClass().newInstance();
for (Entry<String, AttributeInfo> e : instance.fieldMapping().fields.entrySet()){
try {
String convertedName = e.getKey().substring(0, 1).toUpperCase()+e.getKey().substring(1);
this.getClass().getMethod("set"+convertedName, Class.forName(e.getValue().dataType().getName())).invoke(this, instance.get(e.getKey()));
} catch (MetadataException | ClassNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
return (T)retObj;
}
}
metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java
......@@ -5,7 +5,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException;
public class BridgeException extends MetaException {
public BridgeException(String msg) {
super(string);
super(msg);
}
/**
......
metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java
package org.apache.hadoop.metadata.bridge;
//TODO - Create Index Annotation Framework for BeanConverter
//TODO - Enhance Bean Conversion to handled nested objects
//TODO - Enhance Bean COnversion to handle Collections
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Properties;
import javax.inject.Inject;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.types.AttributeDefinition;
......@@ -15,31 +23,37 @@ import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BridgeManager {
TypeSystem ts;
MetadataRepository rs;
ArrayList<ABridge> activeBridges;
private final static String bridgeFileDefault = "bridge-manager.properties";
public static final Logger LOG = LoggerFactory.getLogger("BridgeLogger");
@Inject
BridgeManager(MetadataRepository rs){
BridgeManager(MetadataRepository rs) throws ConfigurationException, ClassNotFoundException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException{
this.ts = TypeSystem.getInstance();
if(System.getProperty("bridgeManager.propsFile") != null | !System.getProperty("bridgeManager.propsFile").isEmpty()){
this.rs = rs;
if(System.getProperty("bridgeManager.propsFile") != null && System.getProperty("bridgeManager.propsFile").length() != 0){
setActiveBridges(System.getProperty("bridgeManager.propsFile"));
}else{
setActiveBridges(System.getProperty(bridgeFileDefault));
setActiveBridges(bridgeFileDefault);
}
for (ABridge bridge : activeBridges){
try {
this.loadTypes(bridge, ts);
} catch (MetadataException e) {
// TODO Auto-generated catch block
BridgeManager.LOG.error(e.getMessage(), e);
e.printStackTrace();
}
}
// Handle some kind of errors - waiting on errors concept from typesystem
}
public ArrayList<ABridge> getActiveBridges(){
......@@ -47,37 +61,30 @@ public class BridgeManager {
}
private void setActiveBridges(String bridgePropFileName){
if(bridgePropFileName == null | bridgePropFileName.isEmpty()){
if(bridgePropFileName == null || bridgePropFileName.isEmpty()){
bridgePropFileName = BridgeManager.bridgeFileDefault;
}
ArrayList<ABridge> aBList = new ArrayList<ABridge>();
Properties props = new Properties();
InputStream configStm = this.getClass().getResourceAsStream(bridgePropFileName);
PropertiesConfiguration config = new PropertiesConfiguration();
try {
ABridge.LOG.info("Loading : Active Bridge List");
props.load(configStm);
String[] activeBridgeList = ((String)props.get("BridgeManager.activeBridges")).split(",");
ABridge.LOG.info("Loaded : Active Bridge List");
ABridge.LOG.info("First Loaded :" + activeBridgeList[0]);
BridgeManager.LOG.info("Loading : Active Bridge List");
config.load(bridgePropFileName);
String[] activeBridgeList = ((String)config.getProperty("BridgeManager.activeBridges")).split(",");
BridgeManager.LOG.info("Loaded : Active Bridge List");
BridgeManager.LOG.info("First Loaded :" + activeBridgeList[0]);
for (String s : activeBridgeList){
Class<?> bridgeCls = (Class<?>) Class.forName(s);
if(bridgeCls.isAssignableFrom(ABridge.class)){
aBList.add((ABridge) bridgeCls.newInstance());
if(ABridge.class.isAssignableFrom(bridgeCls)){
System.out.println( s +" is able to be instaciated");
aBList.add((ABridge) bridgeCls.getConstructor(MetadataRepository.class).newInstance(rs));
}
}
} catch (IOException e) {
ABridge.LOG.error(e.getMessage(), e);
e.printStackTrace();
} catch (InstantiationException e) {
ABridge.LOG.error(e.getMessage(), e);
e.printStackTrace();
} catch (IllegalAccessException e) {
ABridge.LOG.error(e.getMessage(), e);
e.printStackTrace();
} catch (ClassNotFoundException e) {
ABridge.LOG.error(e.getMessage(), e);
} catch (InstantiationException | ConfigurationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException | ClassNotFoundException e) {
BridgeManager.LOG.error(e.getMessage(), e);
e.printStackTrace();
}
this.activeBridges = aBList;
......@@ -87,7 +94,7 @@ public class BridgeManager {
}
private final boolean loadTypes(ABridge bridge, TypeSystem ts) throws MetadataException{
for (Class<AEnitityBean> clazz : bridge.getTypeBeanClasses()){
for (Class<? extends AEnitityBean> clazz : bridge.getTypeBeanClasses()){
ts.defineClassType(BridgeManager.convertEntityBeanToClassTypeDefinition(clazz));
}
return false;
......@@ -101,13 +108,13 @@ public class BridgeManager {
try {
attDefAL.add(BridgeManager.convertFieldtoAttributeDefiniton(f));
} catch (MetadataException e) {
ABridge.LOG.error("Class " + class1.getName() + " cannot be converted to TypeDefinition");
BridgeManager.LOG.error("Class " + class1.getName() + " cannot be converted to TypeDefinition");
e.printStackTrace();
}
}
HierarchicalTypeDefinition<ClassType> typeDef = new HierarchicalTypeDefinition<>(ClassType.class, class1.getSimpleName(),
null, (AttributeDefinition[])attDefAL.toArray());
null, (AttributeDefinition[])attDefAL.toArray(new AttributeDefinition[0]));
return typeDef;
}
......
metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java
......@@ -2,8 +2,11 @@ package org.apache.hadoop.metadata.bridge.hivestructure;
import java.util.ArrayList;
import javax.inject.Inject;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.bridge.ABridge;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
......@@ -13,10 +16,17 @@ import org.apache.hadoop.metadata.types.TypeSystem;
public class HiveStructureBridge extends ABridge{
static final String DB_CLASS_TYPE = "HiveDatabase";
static final String TB_CLASS_TYPE = "HiveTable";
static final String FD_CLASS_TYPE = "HiveField";
@Override
@Inject
protected HiveStructureBridge(MetadataRepository repo) {
super(repo);
// TODO Auto-generated constructor stub
}
public boolean defineBridgeTypes(TypeSystem ts) {
ArrayList<HierarchicalTypeDefinition<?>> al = new ArrayList<HierarchicalTypeDefinition<?>>();
// TODO
......
metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java
......@@ -52,7 +52,7 @@ public class HiveLineageResource {
@Produces(MediaType.APPLICATION_JSON)
public JsonElement getById(@PathParam("id") String id) throws RepositoryException {
// get the lineage bean
HiveLineage hlb = bridge.get(id);
HiveLineage hlb = (HiveLineage) bridge.get(id);
// turn it into a JsonTree & return
return new Gson().toJsonTree(hlb);
}
......
metadata-bridge-parent/metadata-bridge-core/src/main/resources/bridge-manager.properties
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
#
BridgeManager.activebridges=org.apache.hadoop.metadata.bridge.hivelineage
\ No newline at end of file
BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.hivelineage.HiveLineageBridge
\ No newline at end of file
metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/BridgeManagerTest.java 0 → 100644
package org.apache.hadoop.metadata.bridge;
import javax.inject.Inject;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.testng.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@Guice(modules = RepositoryMetadataModule.class)
public class BridgeManagerTest{
@Inject
MetadataRepository repo;
@Test
public void testLoadPropertiesFile() throws Exception {
BridgeManager bm = new BridgeManager(repo);
System.out.println(bm.getActiveBridges().size());
Assert.assertEquals(bm.activeBridges.get(0).getClass().getSimpleName(),"HiveLineageBridge");
}
@Test
public void testBeanConvertion(){
//Tests Conversion of Bean to Type
}
@Test
public void testIRefConvertion(){
//Tests Conversion of IRef cast to Bean
}
}
metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/TestGenericBridges.java 0 → 100644
package org.apache.hadoop.metadata.bridge;
public class TestGenericBridges {
//TODO Build Generic Tests for non-lineage Bridge
}
metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/hivelineage/TestHiveLineageBridge.java
......@@ -10,6 +10,7 @@ import javax.inject.Inject;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.bridge.hivelineage.hook.HiveLineage;
import org.apache.hadoop.metadata.repository.MetadataRepository;
......@@ -47,7 +48,7 @@ public class TestHiveLineageBridge {
}
@Test(priority = 1, enabled = false)
public void testCreate() throws RepositoryException {
public void testCreate() throws MetadataException {
// add the lineage bean to the repo
oneId = bridge.create(hlb);
......@@ -57,7 +58,7 @@ public class TestHiveLineageBridge {
@Test(priority = 2, enabled = false)
public void testGet() throws RepositoryException, IOException {
HiveLineage bean = bridge.get(oneId);
Object bean = bridge.get(oneId);
Assert.assertEquals(hlb, bean);
}
......
metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/test/BridgeManagerTest.java deleted 100644 → 0
package org.apache.hadoop.metadata.bridge.test;
import org.testng.annotations.Test;
public class BridgeManagerTest{
@Test
public void testLoadPropertiesFile() throws Exception {
}
}
metadata-bridge-parent/metadata-bridge-core/src/test/resources/bridge-manager.properties 0 → 100644
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
#
BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.hivelineage.HiveLineageBridge
\ No newline at end of file
metadata-bridge-parent/metadata-bridge-core/src/test/resources/test-bridge-manager.properties
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
#
BridgeManager.activebridges=org.apache.hadoop.metadata.bridge.hivelineage
\ No newline at end of file
BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.HiveLineage
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment