Skip to content

A
atlas
Commit 2ee66639 by Venkatesh Seetharam
Options

BUG-33250 Apache Incubation Related changes- take 2. Contributed by Venkatesh Seetharam

parent ee143338
Showing with 877 additions and 681 deletions
addons/falcontypes/pom.xml → addons/falcon-bridge/pom.xml
...@@ -20,21 +20,27 @@ ...@@ -20,21 +20,27 @@
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<!--
<parent>
<artifactId>metadata-governance</artifactId>
<groupId>org.apache.hadoop.metadata</groupId>
<version>0.1-incubating-SNAPSHOT</version>
</parent>
-->
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>metadata-governance</artifactId>
<groupId>org.apache.hadoop.metadata</groupId>
<version>0.1-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>falcon-bridge</artifactId>
<description>Apache Metadata Falcon Bridge Module</description>
<name>Apache Metadata Falcon Bridge</name>
<packaging>jar</packaging>
<artifactId>metadata-falcontypes</artifactId> <properties>
<falcon.version>0.6.0.2.2.0.0-2041</falcon.version>
</properties>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.falcon</groupId> <groupId>org.apache.falcon</groupId>
<artifactId>falcon-client</artifactId> <artifactId>falcon-client</artifactId>
<version>${falcon.version}</version>
</dependency> </dependency>
<!-- falcon-client depends on jersey-client in provided scope. Hence explicit dependency --> <!-- falcon-client depends on jersey-client in provided scope. Hence explicit dependency -->
......
addons/hivetypes/pom.xml → addons/hive-bridge/pom.xml
...@@ -20,32 +20,60 @@ ...@@ -20,32 +20,60 @@
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<!--
<parent>
<artifactId>metadata-governance</artifactId>
<groupId>org.apache.hadoop.metadata</groupId>
<version>0.1-incubating-SNAPSHOT</version>
</parent>
-->
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>metadata-governance</artifactId>
<groupId>org.apache.hadoop.metadata</groupId>
<version>0.1-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>hive-bridge</artifactId>
<description>Apache Metadata Hive Bridge Module</description>
<name>Apache Metadata Hive Bridge</name>
<packaging>jar</packaging>
<artifactId>metadata-hivetypes</artifactId> <properties>
<hive.version>0.14.0</hive.version>
</properties>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId> <artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId> <artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.calcite</groupId> <groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId> <artifactId>calcite-avatica</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.calcite</groupId> <groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId> <artifactId>calcite-core</artifactId>
...@@ -56,26 +84,45 @@ ...@@ -56,26 +84,45 @@
<artifactId>gson</artifactId> <artifactId>gson</artifactId>
<version>2.2.2</version> <version>2.2.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId> <groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId> <artifactId>metadata-typesystem</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId> <groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-repository</artifactId> <artifactId>metadata-repository</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId> <artifactId>hadoop-client</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>log4j</groupId> <groupId>log4j</groupId>
<artifactId>log4j</artifactId> <artifactId>log4j</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.testng</groupId> <groupId>org.testng</groupId>
<artifactId>testng</artifactId> <artifactId>testng</artifactId>
</dependency> </dependency>
</dependencies> </dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<configuration>
<excludes>
<exclude>resources/*-site.xml</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>
addons/hivetypes/src/main/java/org/apache/hadoop/metadata/hivetypes/HiveImporter.java → addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hivetypes/HiveImporter.java
...@@ -28,15 +28,15 @@ import org.apache.hadoop.hive.metastore.api.Partition; ...@@ -28,15 +28,15 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.metadata.ITypedReferenceableInstance; import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.ITypedStruct; import org.apache.hadoop.metadata.typesystem.ITypedStruct;
import org.apache.hadoop.metadata.MetadataException; import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.Referenceable; import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.Struct; import org.apache.hadoop.metadata.typesystem.Struct;
import org.apache.hadoop.metadata.repository.MetadataRepository; import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.apache.hadoop.metadata.typesystem.persistence.IRepository; import org.apache.hadoop.metadata.repository.IRepository;
import org.apache.hadoop.metadata.typesystem.persistence.Id; import org.apache.hadoop.metadata.typesystem.persistence.Id;
import org.apache.hadoop.metadata.typesystem.persistence.RepositoryException; import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.types.IDataType; import org.apache.hadoop.metadata.typesystem.types.IDataType;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity; import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.StructType; import org.apache.hadoop.metadata.typesystem.types.StructType;
......
metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeTypeBootstrapper.java → addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeTypeBootstrapper.java
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* <p/> * <p/>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* <p/> * <p/>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
import org.apache.hadoop.metadata.MetadataException; import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition; import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType; import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition; import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity; import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem; import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.inject.Inject; import javax.inject.Inject;
import javax.inject.Singleton; import javax.inject.Singleton;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Map; import java.util.Map;
@Singleton @Singleton
public class BridgeTypeBootstrapper { public class BridgeTypeBootstrapper {
private static final Logger LOG = LoggerFactory.getLogger(BridgeTypeBootstrapper.class); private static final Logger LOG = LoggerFactory.getLogger(BridgeTypeBootstrapper.class);
private final Map<Class, IBridge> bridges; private final Map<Class, IBridge> bridges;
private boolean isSetup = false; private boolean isSetup = false;
@Inject @Inject
BridgeTypeBootstrapper(Map<Class, IBridge> bridges) BridgeTypeBootstrapper(Map<Class, IBridge> bridges)
throws MetadataException { throws MetadataException {
this.bridges = bridges; this.bridges = bridges;
} }
public final static HierarchicalTypeDefinition<ClassType> public final static HierarchicalTypeDefinition<ClassType>
convertEntityBeanToClassTypeDefinition( convertEntityBeanToClassTypeDefinition(
Class<? extends AEntityBean> class1) { Class<? extends AEntityBean> class1) {
ArrayList<AttributeDefinition> attDefAL = new ArrayList<AttributeDefinition>(); ArrayList<AttributeDefinition> attDefAL = new ArrayList<AttributeDefinition>();
for (Field f : class1.getFields()) { for (Field f : class1.getFields()) {
try { try {
attDefAL.add(BridgeTypeBootstrapper.convertFieldtoAttributeDefiniton(f)); attDefAL.add(BridgeTypeBootstrapper.convertFieldtoAttributeDefiniton(f));
} catch (MetadataException e) { } catch (MetadataException e) {
BridgeManager.LOG.error("Class " + class1.getName() BridgeManager.LOG.error("Class " + class1.getName()
+ " cannot be converted to TypeDefinition"); + " cannot be converted to TypeDefinition");
e.printStackTrace(); e.printStackTrace();
} }
} }
HierarchicalTypeDefinition<ClassType> typeDef = new HierarchicalTypeDefinition<>( HierarchicalTypeDefinition<ClassType> typeDef = new HierarchicalTypeDefinition<>(
ClassType.class, class1.getSimpleName(), null, ClassType.class, class1.getSimpleName(), null,
(AttributeDefinition[]) attDefAL (AttributeDefinition[]) attDefAL
.toArray(new AttributeDefinition[0])); .toArray(new AttributeDefinition[0]));
return typeDef; return typeDef;
} }
public final static AttributeDefinition convertFieldtoAttributeDefiniton( public final static AttributeDefinition convertFieldtoAttributeDefiniton(
Field f) throws MetadataException { Field f) throws MetadataException {
return new AttributeDefinition(f.getName(), return new AttributeDefinition(f.getName(),
f.getType().getSimpleName().toLowerCase(), Multiplicity.REQUIRED, false, null); f.getType().getSimpleName().toLowerCase(), Multiplicity.REQUIRED, false, null);
} }
public synchronized boolean bootstrap() throws MetadataException { public synchronized boolean bootstrap() throws MetadataException {
if (isSetup) if (isSetup)
return false; return false;
else { else {
LOG.info("Bootstrapping types"); LOG.info("Bootstrapping types");
_bootstrap(); _bootstrap();
isSetup = true; isSetup = true;
LOG.info("Bootstrapping complete"); LOG.info("Bootstrapping complete");
return true; return true;
} }
} }
private void _bootstrap() throws MetadataException { private void _bootstrap() throws MetadataException {
TypeSystem ts = TypeSystem.getInstance(); TypeSystem ts = TypeSystem.getInstance();
for (IBridge bridge : bridges.values()) { for (IBridge bridge : bridges.values()) {
LOG.info("Registering bridge, %s", bridge.getClass().getSimpleName()); LOG.info("Registering bridge, %s", bridge.getClass().getSimpleName());
loadTypes(bridge, ts); loadTypes(bridge, ts);
} }
} }
private final boolean loadTypes(IBridge bridge, TypeSystem ts) private final boolean loadTypes(IBridge bridge, TypeSystem ts)
throws MetadataException { throws MetadataException {
for (Class<? extends AEntityBean> clazz : bridge.getTypeBeanClasses()) { for (Class<? extends AEntityBean> clazz : bridge.getTypeBeanClasses()) {
LOG.info("Registering %s", clazz.getSimpleName()); LOG.info("Registering %s", clazz.getSimpleName());
ts.defineClassType(BridgeTypeBootstrapper ts.defineClassType(BridgeTypeBootstrapper
.convertEntityBeanToClassTypeDefinition(clazz)); .convertEntityBeanToClassTypeDefinition(clazz));
} }
return false; return false;
} }
} }
metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/module/BridgeModule.java → addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/module/BridgeModule.java
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* <p/> * <p/>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* <p/> * <p/>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.metadata.bridge.module; package org.apache.hadoop.metadata.bridge.module;
import com.google.inject.AbstractModule; import com.google.inject.AbstractModule;
import com.google.inject.Scopes; import com.google.inject.Scopes;
import com.google.inject.multibindings.MapBinder; import com.google.inject.multibindings.MapBinder;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.RepositoryMetadataModule; import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.bridge.BridgeTypeBootstrapper; import org.apache.hadoop.metadata.bridge.BridgeTypeBootstrapper;
import org.apache.hadoop.metadata.bridge.IBridge; import org.apache.hadoop.metadata.bridge.IBridge;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
public class BridgeModule extends AbstractModule { public class BridgeModule extends AbstractModule {
public static final Logger LOG = LoggerFactory public static final Logger LOG = LoggerFactory
.getLogger(BridgeModule.class); .getLogger(BridgeModule.class);
@Override @Override
protected void configure() { protected void configure() {
install(new RepositoryMetadataModule()); install(new RepositoryMetadataModule());
// make sure the BridgeTypeBootstrapper is only ever created once // make sure the BridgeTypeBootstrapper is only ever created once
bind(BridgeTypeBootstrapper.class).in(Scopes.SINGLETON); bind(BridgeTypeBootstrapper.class).in(Scopes.SINGLETON);
// Load the configured bridge classes and add them to the map binder // Load the configured bridge classes and add them to the map binder
MapBinder<Class, IBridge> mapbinder = MapBinder.newMapBinder(binder(), MapBinder<Class, IBridge> mapbinder = MapBinder.newMapBinder(binder(),
Class.class, IBridge.class); Class.class, IBridge.class);
String propsURI = System.getProperty("bridgeManager.propsFile", String propsURI = System.getProperty("bridgeManager.propsFile",
"bridge-manager.properties"); "bridge-manager.properties");
List<Class<? extends IBridge>> bridges = getBridgeClasses(propsURI); List<Class<? extends IBridge>> bridges = getBridgeClasses(propsURI);
for (Class<? extends IBridge> bridgeClass : bridges) { for (Class<? extends IBridge> bridgeClass : bridges) {
mapbinder.addBinding(bridgeClass).to(bridgeClass).in(Scopes.SINGLETON); mapbinder.addBinding(bridgeClass).to(bridgeClass).in(Scopes.SINGLETON);
} }
} }
/* /*
* Get the bridge classes from the configuration file * Get the bridge classes from the configuration file
*/ */
private List<Class<? extends IBridge>> getBridgeClasses( private List<Class<? extends IBridge>> getBridgeClasses(
String bridgePropFileName) { String bridgePropFileName) {
List<Class<? extends IBridge>> aBList = new ArrayList<Class<? extends IBridge>>(); List<Class<? extends IBridge>> aBList = new ArrayList<Class<? extends IBridge>>();
PropertiesConfiguration config = new PropertiesConfiguration(); PropertiesConfiguration config = new PropertiesConfiguration();
try { try {
LOG.info("Loading : Active Bridge List"); LOG.info("Loading : Active Bridge List");
config.load(bridgePropFileName); config.load(bridgePropFileName);
String[] activeBridgeList = ((String) config String[] activeBridgeList = ((String) config
.getProperty("BridgeManager.activeBridges")).split(","); .getProperty("BridgeManager.activeBridges")).split(",");
LOG.info("Loaded : Active Bridge List"); LOG.info("Loaded : Active Bridge List");
for (String s : activeBridgeList) { for (String s : activeBridgeList) {
Class<? extends IBridge> bridgeCls = (Class<? extends IBridge>) Class Class<? extends IBridge> bridgeCls = (Class<? extends IBridge>) Class
.forName(s); .forName(s);
aBList.add(bridgeCls); aBList.add(bridgeCls);
} }
} catch (ConfigurationException | IllegalArgumentException } catch (ConfigurationException | IllegalArgumentException
| SecurityException | ClassNotFoundException e) { | SecurityException | ClassNotFoundException e) {
LOG.error(e.getMessage(), e); LOG.error(e.getMessage(), e);
e.printStackTrace(); e.printStackTrace();
} }
return aBList; return aBList;
} }
} }
metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java → addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* <p/> * <p/>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* <p/> * <p/>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.metadata.web.resources; package org.apache.hadoop.metadata.web.resources;
import org.apache.hadoop.metadata.bridge.hivelineage.HiveLineageBridge; import org.apache.hadoop.metadata.bridge.hivelineage.HiveLineageBridge;
import javax.inject.Singleton; import javax.inject.Singleton;
//@Path("bridge/hive") //@Path("bridge/hive")
@Singleton @Singleton
public class HiveLineageResource { public class HiveLineageResource {
private final HiveLineageBridge bridge = null; private final HiveLineageBridge bridge = null;
/* /*
//@Inject //@Inject
public HiveLineageResource(HiveLineageBridge bridge) { public HiveLineageResource(HiveLineageBridge bridge) {
this.bridge = bridge; this.bridge = bridge;
} }
//@Inject //@Inject
public HiveLineageResource(Map<Class<? extends IBridge>, IBridge> bridges) { public HiveLineageResource(Map<Class<? extends IBridge>, IBridge> bridges) {
this.bridge = (HiveLineageBridge) bridges.get(HiveLineageBridge.class); this.bridge = (HiveLineageBridge) bridges.get(HiveLineageBridge.class);
} }
@GET @GET
@Path("/{id}") @Path("/{id}")
@Produces(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON)
public JsonElement getById(@PathParam("id") String id) throws RepositoryException { public JsonElement getById(@PathParam("id") String id) throws RepositoryException {
// get the lineage bean // get the lineage bean
HiveLineage hlb = (HiveLineage) bridge.get(id); HiveLineage hlb = (HiveLineage) bridge.get(id);
// turn it into a JsonTree & return // turn it into a JsonTree & return
return new Gson().toJsonTree(hlb); return new Gson().toJsonTree(hlb);
} }
@GET @GET
@Produces(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON)
public JsonElement list() throws RepositoryException { public JsonElement list() throws RepositoryException {
// make a new JsonArray to be returned // make a new JsonArray to be returned
JsonArray ja = new JsonArray(); JsonArray ja = new JsonArray();
// iterate over each item returned by the hive bridge's list() method // iterate over each item returned by the hive bridge's list() method
for (String s: bridge.list()) { for (String s: bridge.list()) {
// they are GUIDs so make them into JsonPrimitives // they are GUIDs so make them into JsonPrimitives
ja.add(new JsonPrimitive(s)); ja.add(new JsonPrimitive(s));
} }
return ja; return ja;
} }
@POST @POST
@Consumes(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON)
public JsonElement addLineage(@Context HttpServletRequest request) public JsonElement addLineage(@Context HttpServletRequest request)
throws IOException, MetadataException { throws IOException, MetadataException {
// create a reader // create a reader
try (Reader reader = new InputStreamReader(request.getInputStream())) { try (Reader reader = new InputStreamReader(request.getInputStream())) {
// deserialize // deserialize
HiveLineage bean = new Gson().fromJson(reader, HiveLineage.class); HiveLineage bean = new Gson().fromJson(reader, HiveLineage.class);
String id = bridge.create(bean); String id = bridge.create(bean);
JsonObject jo = new JsonObject(); JsonObject jo = new JsonObject();
jo.addProperty("id", id); jo.addProperty("id", id);
return jo; return jo;
} }
} }
*/ */
} }
metadata-bridge-parent/metadata-bridge-core/src/main/resources/bridge-manager.properties → addons/metadata-bridge-parent/metadata-bridge-core/src/main/resources/bridge-manager.properties
# #
# Licensed to the Apache Software Foundation (ASF) under one # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file # or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information # distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file # regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the # to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance # "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at # with the License. You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths) #BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
# #
BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.hivelineage.HiveLineageBridge BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.hivelineage.HiveLineageBridge
\ No newline at end of file
metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/BridgeManagerTest.java → addons/metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/BridgeManagerTest.java
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* <p/> * <p/>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* <p/> * <p/>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
import org.apache.hadoop.metadata.RepositoryMetadataModule; import org.apache.hadoop.metadata.RepositoryMetadataModule;
import org.apache.hadoop.metadata.repository.MetadataRepository; import org.apache.hadoop.metadata.repository.MetadataRepository;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import javax.inject.Inject; import javax.inject.Inject;
@Guice(modules = RepositoryMetadataModule.class) @Guice(modules = RepositoryMetadataModule.class)
public class BridgeManagerTest { public class BridgeManagerTest {
@Inject @Inject
MetadataRepository repo; MetadataRepository repo;
@Test(enabled = false) @Test(enabled = false)
public void testLoadPropertiesFile() throws Exception { public void testLoadPropertiesFile() throws Exception {
BridgeManager bm = new BridgeManager(repo); BridgeManager bm = new BridgeManager(repo);
System.out.println(bm.getActiveBridges().size()); System.out.println(bm.getActiveBridges().size());
Assert.assertEquals(bm.activeBridges.get(0).getClass().getSimpleName(), Assert.assertEquals(bm.activeBridges.get(0).getClass().getSimpleName(),
"HiveLineageBridge"); "HiveLineageBridge");
} }
@Test @Test
public void testBeanConvertion() { public void testBeanConvertion() {
//Tests Conversion of Bean to Type //Tests Conversion of Bean to Type
} }
@Test @Test
public void testIRefConvertion() { public void testIRefConvertion() {
//Tests Conversion of IRef cast to Bean //Tests Conversion of IRef cast to Bean
} }
} }
metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/TestBridgeModule.java → addons/metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/TestBridgeModule.java
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* <p/> * <p/>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* <p/> * <p/>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
import org.apache.hadoop.metadata.bridge.module.BridgeModule; import org.apache.hadoop.metadata.bridge.module.BridgeModule;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@Guice(modules = {BridgeModule.class}) @Guice(modules = {BridgeModule.class})
public class TestBridgeModule { public class TestBridgeModule {
@Test @Test
public void loadAnything() { public void loadAnything() {
// if it makes it here, the BridgeModule loaded successfully // if it makes it here, the BridgeModule loaded successfully
Assert.assertTrue(true); Assert.assertTrue(true);
} }
} }
metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/TestGenericBridges.java → addons/metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/TestGenericBridges.java
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* <p/> * <p/>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* <p/> * <p/>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.metadata.bridge; package org.apache.hadoop.metadata.bridge;
public class TestGenericBridges { public class TestGenericBridges {
//TODO Build Generic Tests for non-lineage Bridge //TODO Build Generic Tests for non-lineage Bridge
} }
metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/hivelineage/TestHiveLineageBridge.java → addons/metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/hivelineage/TestHiveLineageBridge.java
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* <p/> * <p/>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* <p/> * <p/>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.metadata.bridge.hivelineage; package org.apache.hadoop.metadata.bridge.hivelineage;
import com.google.gson.Gson; import com.google.gson.Gson;
import org.apache.commons.collections.IteratorUtils; import org.apache.commons.collections.IteratorUtils;
import org.apache.hadoop.metadata.MetadataException; import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.bridge.BridgeTypeBootstrapper; import org.apache.hadoop.metadata.bridge.BridgeTypeBootstrapper;
import org.apache.hadoop.metadata.bridge.hivelineage.hook.HiveLineage; import org.apache.hadoop.metadata.bridge.hivelineage.hook.HiveLineage;
import org.apache.hadoop.metadata.bridge.module.BridgeModule; import org.apache.hadoop.metadata.bridge.module.BridgeModule;
import org.apache.hadoop.metadata.repository.RepositoryException; import org.apache.hadoop.metadata.repository.RepositoryException;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import javax.inject.Inject; import javax.inject.Inject;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.util.List; import java.util.List;
@Guice(modules = {BridgeModule.class}) @Guice(modules = {BridgeModule.class})
public class TestHiveLineageBridge { public class TestHiveLineageBridge {
@Inject @Inject
HiveLineageBridge bridge; HiveLineageBridge bridge;
@Inject @Inject
BridgeTypeBootstrapper bootstrapper; BridgeTypeBootstrapper bootstrapper;
HiveLineage hlb; HiveLineage hlb;
// the id of one.json in the repo (test #1) // the id of one.json in the repo (test #1)
String oneId; String oneId;
private HiveLineage loadHiveLineageBean(String path) throws IOException { private HiveLineage loadHiveLineageBean(String path) throws IOException {
return new Gson().fromJson(new InputStreamReader(this.getClass().getResourceAsStream(path)), return new Gson().fromJson(new InputStreamReader(this.getClass().getResourceAsStream(path)),
HiveLineage.class); HiveLineage.class);
} }
@BeforeClass @BeforeClass
public void bootstrap() throws IOException, MetadataException { public void bootstrap() throws IOException, MetadataException {
bootstrapper.bootstrap(); bootstrapper.bootstrap();
hlb = loadHiveLineageBean("/one.json"); hlb = loadHiveLineageBean("/one.json");
} }
@Test(priority = 1, enabled = false) @Test(priority = 1, enabled = false)
public void testCreate() throws MetadataException { public void testCreate() throws MetadataException {
// add the lineage bean to the repo // add the lineage bean to the repo
oneId = bridge.create(hlb); oneId = bridge.create(hlb);
// make sure this actually did worked // make sure this actually did worked
Assert.assertNotNull(oneId); Assert.assertNotNull(oneId);
} }
@Test(priority = 2, enabled = false) @Test(priority = 2, enabled = false)
public void testGet() throws RepositoryException, IOException { public void testGet() throws RepositoryException, IOException {
Object bean = bridge.get(oneId); Object bean = bridge.get(oneId);
Assert.assertEquals(hlb, bean); Assert.assertEquals(hlb, bean);
} }
@Test(priority = 3, enabled = false) @Test(priority = 3, enabled = false)
public void testList() throws RepositoryException { public void testList() throws RepositoryException {
List<String> list = IteratorUtils.toList(bridge.list().iterator()); List<String> list = IteratorUtils.toList(bridge.list().iterator());
Assert.assertEquals(list.size(), 1); Assert.assertEquals(list.size(), 1);
Assert.assertEquals(list.get(0), oneId); Assert.assertEquals(list.get(0), oneId);
} }
} }
metadata-bridge-parent/metadata-bridge-core/src/test/resources/bridge-manager.properties → addons/metadata-bridge-parent/metadata-bridge-core/src/test/resources/bridge-manager.properties
# #
# Licensed to the Apache Software Foundation (ASF) under one # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file # or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information # distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file # regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the # to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance # "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at # with the License. You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths) #BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
# #
BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.hivelineage.HiveLineageBridge BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.hivelineage.HiveLineageBridge
\ No newline at end of file
metadata-bridge-parent/metadata-bridge-core/src/test/resources/test-bridge-manager.properties → addons/metadata-bridge-parent/metadata-bridge-core/src/test/resources/test-bridge-manager.properties
# #
# Licensed to the Apache Software Foundation (ASF) under one # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file # or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information # distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file # regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the # to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance # "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at # with the License. You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths) #BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
# #
BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.HiveLineage BridgeManager.activeBridges=org.apache.hadoop.metadata.bridge.HiveLineage
\ No newline at end of file
metadata-bridge-parent/pom.xml → addons/metadata-bridge-parent/pom.xml
...@@ -24,12 +24,12 @@ ...@@ -24,12 +24,12 @@
<groupId>org.apache.hadoop.metadata</groupId> <groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-governance</artifactId> <artifactId>metadata-governance</artifactId>
<version>0.1-incubating-SNAPSHOT</version> <version>0.1-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent> </parent>
<artifactId>metadata-bridge-parent</artifactId> <artifactId>metadata-bridge-parent</artifactId>
<packaging>pom</packaging> <packaging>pom</packaging>
<modules> <modules>
<module>metadata-bridge-core</module> <module>metadata-bridge-core</module>
<module>metadata-bridge-hive</module> <module>metadata-bridge-hive</module>
<module>metadata-bridge</module>
</modules> </modules>
</project> </project>
\ No newline at end of file
metadata-bridge-parent/metadata-bridge/.gitignore deleted 100644 → 0
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
target
\ No newline at end of file
metadata-bridge-parent/metadata-bridge/pom.xml deleted 100644 → 0
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-bridge-parent</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>metadata-bridge</artifactId>
<packaging>pom</packaging>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>metadata-bridge-core</artifactId>
<version>${project.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>metadata-bridge-hive</artifactId>
<version>${project.version}</version>
<optional>true</optional>
</dependency>
</dependencies>
</project>
\ No newline at end of file
pom.xml
...@@ -80,8 +80,6 @@ ...@@ -80,8 +80,6 @@
<tinkerpop.version>2.5.0</tinkerpop.version> <tinkerpop.version>2.5.0</tinkerpop.version>
<titan.version>0.5.3</titan.version> <titan.version>0.5.3</titan.version>
<hadoop.version>2.5.0</hadoop.version> <hadoop.version>2.5.0</hadoop.version>
<hive.version>0.14.0</hive.version>
<falcon.version>0.6.0.2.2.0.0-2041</falcon.version>
<!-- scala versions --> <!-- scala versions -->
<scala.version>2.10.4</scala.version> <scala.version>2.10.4</scala.version>
...@@ -206,7 +204,6 @@ ...@@ -206,7 +204,6 @@
<module>webapp</module> <module>webapp</module>
<module>docs</module> <module>docs</module>
<module>tools</module> <module>tools</module>
<module>metadata-bridge-parent</module>
</modules> </modules>
<repositories> <repositories>
...@@ -267,39 +264,6 @@ ...@@ -267,39 +264,6 @@
<dependencyManagement> <dependencyManagement>
<dependencies> <dependencies>
<!-- hive -->
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency> <dependency>
<groupId>org.apache.calcite</groupId> <groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId> <artifactId>calcite-avatica</artifactId>
...@@ -313,12 +277,6 @@ ...@@ -313,12 +277,6 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.falcon</groupId>
<artifactId>falcon-client</artifactId>
<version>${falcon.version}</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId> <groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId> <artifactId>jsp-2.1</artifactId>
<version>6.0.0</version> <version>6.0.0</version>
...@@ -932,7 +890,6 @@ ...@@ -932,7 +890,6 @@
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.7.2</version> <version>2.7.2</version>
<configuration> <configuration>
<skipTests>true</skipTests>
<forkMode>always</forkMode> <forkMode>always</forkMode>
</configuration> </configuration>
<dependencies> <dependencies>
......
src/build/checkstyle-noframes.xsl 0 → 100644
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
<xsl:output method="html" indent="yes"/>
<xsl:decimal-format decimal-separator="." grouping-separator=","/>
<xsl:key name="files" match="file" use="@name"/>
<!-- Checkstyle XML Style Sheet by Stephane Bailliez <sbailliez@apache.org> -->
<!-- Part of the Checkstyle distribution found at http://checkstyle.sourceforge.net -->
<!-- Usage (generates checkstyle_report.html): -->
<!-- <checkstyle failonviolation="false" config="${check.config}"> -->
<!-- <fileset dir="${src.dir}" includes="**/*.java"/> -->
<!-- <formatter type="xml" toFile="${doc.dir}/checkstyle_report.xml"/> -->
<!-- </checkstyle> -->
<!-- <style basedir="${doc.dir}" destdir="${doc.dir}" -->
<!-- includes="checkstyle_report.xml" -->
<!-- style="${doc.dir}/checkstyle-noframes-sorted.xsl"/> -->
<xsl:template match="checkstyle">
<html>
<head>
<style type="text/css">
.bannercell {
border: 0px;
padding: 0px;
}
body {
margin-left: 10;
margin-right: 10;
font:normal 80% arial,helvetica,sanserif;
background-color:#FFFFFF;
color:#000000;
}
.a td {
background: #efefef;
}
.b td {
background: #fff;
}
th, td {
text-align: left;
vertical-align: top;
}
th {
font-weight:bold;
background: #ccc;
color: black;
}
table, th, td {
font-size:100%;
border: none
}
table.log tr td, tr th {
}
h2 {
font-weight:bold;
font-size:140%;
margin-bottom: 5;
}
h3 {
font-size:100%;
font-weight:bold;
background: #525D76;
color: white;
text-decoration: none;
padding: 5px;
margin-right: 2px;
margin-left: 2px;
margin-bottom: 0;
}
</style>
</head>
<body>
<a name="top"></a>
<!-- jakarta logo -->
<table border="0" cellpadding="0" cellspacing="0" width="100%">
<tr>
<td class="bannercell" rowspan="2">
<!--a href="http://jakarta.apache.org/">
<img src="http://jakarta.apache.org/images/jakarta-logo.gif" alt="http://jakarta.apache.org" align="left" border="0"/>
</a-->
</td>
<td class="text-align:right">
<h2>CheckStyle Audit</h2>
</td>
</tr>
<tr>
<td class="text-align:right">Designed for use with
<a href='http://checkstyle.sourceforge.net/'>CheckStyle</a>
and<a href='http://jakarta.apache.org'>Ant</a>.
</td>
</tr>
</table>
<hr size="1"/>
<!-- Summary part -->
<xsl:apply-templates select="." mode="summary"/>
<hr size="1" width="100%" align="left"/>
<!-- Package List part -->
<xsl:apply-templates select="." mode="filelist"/>
<hr size="1" width="100%" align="left"/>
<!-- For each package create its part -->
<xsl:apply-templates
select="file[@name and generate-id(.) = generate-id(key('files', @name))]"/>
<hr size="1" width="100%" align="left"/>
</body>
</html>
</xsl:template>
<xsl:template match="checkstyle" mode="filelist">
<h3>Files</h3>
<table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
<tr>
<th>Name</th>
<th>Errors</th>
</tr>
<xsl:for-each
select="file[@name and generate-id(.) = generate-id(key('files', @name))]">
<xsl:sort data-type="number" order="descending"
select="count(key('files', @name)/error)"/>
<xsl:variable name="errorCount" select="count(error)"/>
<tr>
<xsl:call-template name="alternated-row"/>
<td>
<a href="#f-{@name}">
<xsl:value-of select="@name"/>
</a>
</td>
<td>
<xsl:value-of select="$errorCount"/>
</td>
</tr>
</xsl:for-each>
</table>
</xsl:template>
<xsl:template match="file">
<a name="f-{@name}"></a>
<h3>File
<xsl:value-of select="@name"/>
</h3>
<table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
<tr>
<th>Error Description</th>
<th>Line</th>
</tr>
<xsl:for-each select="key('files', @name)/error">
<xsl:sort data-type="number" order="ascending" select="@line"/>
<tr>
<xsl:call-template name="alternated-row"/>
<td>
<xsl:value-of select="@message"/>
</td>
<td>
<xsl:value-of select="@line"/>
</td>
</tr>
</xsl:for-each>
</table>
<a href="#top">Back to top</a>
</xsl:template>
<xsl:template match="checkstyle" mode="summary">
<h3>Summary</h3>
<xsl:variable name="fileCount"
select="count(file[@name and generate-id(.) = generate-id(key('files', @name))])"/>
<xsl:variable name="errorCount" select="count(file/error)"/>
<table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
<tr>
<th>Files</th>
<th>Errors</th>
</tr>
<tr>
<xsl:call-template name="alternated-row"/>
<td>
<xsl:value-of select="$fileCount"/>
</td>
<td>
<xsl:value-of select="$errorCount"/>
</td>
</tr>
</table>
</xsl:template>
<xsl:template name="alternated-row">
<xsl:attribute name="class">
<xsl:if test="position() mod 2 = 1">a</xsl:if>
<xsl:if test="position() mod 2 = 0">b</xsl:if>
</xsl:attribute>
</xsl:template>
</xsl:stylesheet>
src/build/checkstyle.xml
...@@ -23,7 +23,7 @@ ...@@ -23,7 +23,7 @@
<!-- <!--
Checkstyle configuration for Falcon that is based on the sun_checks.xml file Checkstyle configuration for Metadata that is based on the sun_checks.xml file
that is bundled with Checkstyle and includes checks for: that is bundled with Checkstyle and includes checks for:
- the Java Language Specification at - the Java Language Specification at
...@@ -210,7 +210,7 @@ ...@@ -210,7 +210,7 @@
This generates too many false-positives on wrapped 'throws' clauses This generates too many false-positives on wrapped 'throws' clauses
to be really useful. Disabled for now. to be really useful. Disabled for now.
Falcon style is: Metadata style is:
* Spaces, not tabs. * Spaces, not tabs.
* Indent by four spaces. * Indent by four spaces.
* Indent by four spaces when wrapping a line. * Indent by four spaces when wrapping a line.
......
src/build/findbugs-exclude.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<FindBugsFilter>
<!--
Disable encoding as this might give an impression that DGI code base is
"Internationalization" ready, but we haven't done anything consciously to guarantee that.
-->
<Match>
<Bug pattern="DM_DEFAULT_ENCODING"/>
</Match>
</FindBugsFilter>
tools/pom.xml
...@@ -148,11 +148,6 @@ ...@@ -148,11 +148,6 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId> <groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId> <artifactId>gson</artifactId>
</dependency> </dependency>
......
webapp/pom.xml
...@@ -268,7 +268,7 @@ ...@@ -268,7 +268,7 @@
<artifactId>maven-jetty-plugin</artifactId> <artifactId>maven-jetty-plugin</artifactId>
<version>${jetty.version}</version> <version>${jetty.version}</version>
<configuration> <configuration>
<skip>${skipTests}</skip> <skip>${skipITs}</skip>
<!--only skip int tests --> <!--only skip int tests -->
<connectors> <connectors>
<!-- <!--
...@@ -292,6 +292,10 @@ ...@@ -292,6 +292,10 @@
<useTestClasspath>true</useTestClasspath> <useTestClasspath>true</useTestClasspath>
<systemProperties> <systemProperties>
<systemProperty> <systemProperty>
<name>metadata.log.dir</name>
<value>${project.build.directory}/logs</value>
</systemProperty>
<systemProperty>
<name>keystore.file</name> <name>keystore.file</name>
<value> <value>
${project.build.directory}/../../webapp/target/metadata.keystore ${project.build.directory}/../../webapp/target/metadata.keystore
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment