Skip to content

A
atlas
Commit 043d2aa6 by Shwetha GS
Options

ATLAS-987 Atlas hooks should avoid adding dependent libraries to component…

ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags)
parent b9ce7a11
Showing with 1162 additions and 42 deletions
addons/falcon-bridge-shim/pom.xml 0 → 100755
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.8-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>falcon-bridge-shim</artifactId>
<description>Apache Atlas Falcon Bridge Shim Module</description>
<name>Apache Atlas Falcon Bridge Shim</name>
<packaging>jar</packaging>
<properties>
<falcon.version>0.8</falcon.version>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
</properties>
<dependencies>
<!-- Logging -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.falcon</groupId>
<artifactId>falcon-common</artifactId>
<version>${falcon.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>
addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java 0 → 100755
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.falcon.service;
import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
import org.apache.falcon.FalconException;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.service.ConfigurationChangeListener;
import org.apache.falcon.service.FalconService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Falcon hook used for atlas entity registration.
*/
public class AtlasService implements FalconService, ConfigurationChangeListener {
private static final Logger LOG = LoggerFactory.getLogger(AtlasService.class);
private static final String ATLAS_PLUGIN_TYPE = "falcon";
private static final String ATLAS_FALCON_HOOK_IMPL_CLASSNAME = "org.apache.atlas.falcon.hook.AtlasService";
private AtlasPluginClassLoader atlasPluginClassLoader = null;
private FalconService falconServiceImpl = null;
private ConfigurationChangeListener configChangeListenerImpl = null;
public AtlasService() {
this.initialize();
}
@Override
public String getName() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasService.getName()");
}
String ret = null;
try {
activatePluginClassLoader();
ret = falconServiceImpl.getName();
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasService.getName()");
}
return ret;
}
@Override
public void init() throws FalconException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasService.init()");
}
try {
activatePluginClassLoader();
falconServiceImpl.init();
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasService.init()");
}
}
@Override
public void destroy() throws FalconException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasService.destroy()");
}
try {
activatePluginClassLoader();
falconServiceImpl.destroy();
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasService.destroy()");
}
}
@Override
public void onAdd(Entity entity) throws FalconException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasService.onAdd(" + entity + ")");
}
try {
activatePluginClassLoader();
configChangeListenerImpl.onAdd(entity);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasService.onAdd(" + entity + ")");
}
}
@Override
public void onRemove(Entity entity) throws FalconException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasService.onRemove(" + entity + ")");
}
try {
activatePluginClassLoader();
configChangeListenerImpl.onRemove(entity);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasService.onRemove(" + entity + ")");
}
}
@Override
public void onChange(Entity entity, Entity entity1) throws FalconException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasService.onChange(" + entity + ", " + entity1 + ")");
}
try {
activatePluginClassLoader();
configChangeListenerImpl.onChange(entity, entity1);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasService.onChange(" + entity + ", " + entity1 + ")");
}
}
@Override
public void onReload(Entity entity) throws FalconException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasService.onReload(" + entity + ")");
}
try {
activatePluginClassLoader();
configChangeListenerImpl.onReload(entity);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasService.onReload(" + entity + ")");
}
}
private void initialize() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasService.initialize()");
}
try {
atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass());
Class<?> cls = Class.forName(ATLAS_FALCON_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader);
activatePluginClassLoader();
Object atlasService = cls.newInstance();
falconServiceImpl = (FalconService) atlasService;
configChangeListenerImpl = (ConfigurationChangeListener) atlasService;
} catch (Exception excp) {
LOG.error("Error instantiating Atlas hook implementation", excp);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasService.initialize()");
}
}
private void activatePluginClassLoader() {
if (atlasPluginClassLoader != null) {
atlasPluginClassLoader.activate();
}
}
private void deactivatePluginClassLoader() {
if (atlasPluginClassLoader != null) {
atlasPluginClassLoader.deactivate();
}
}
}
addons/falcon-bridge/pom.xml
...@@ -99,13 +99,13 @@ ...@@ -99,13 +99,13 @@
<artifactId>maven-dependency-plugin</artifactId> <artifactId>maven-dependency-plugin</artifactId>
<executions> <executions>
<execution> <execution>
<id>copy-hook-dependencies</id> <id>copy-hook</id>
<phase>package</phase> <phase>package</phase>
<goals> <goals>
<goal>copy</goal> <goal>copy</goal>
</goals> </goals>
<configuration> <configuration>
<outputDirectory>${project.build.directory}/dependency/hook/falcon</outputDirectory> <outputDirectory>${project.build.directory}/dependency/hook/falcon/atlas-falcon-plugin-impl</outputDirectory>
<overWriteReleases>false</overWriteReleases> <overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots> <overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer> <overWriteIfNewer>true</overWriteIfNewer>
...@@ -195,6 +195,46 @@ ...@@ -195,6 +195,46 @@
<artifactId>kafka-clients</artifactId> <artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version> <version>${kafka.version}</version>
</artifactItem> </artifactItem>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</artifactItem>
<artifactItem>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>
<execution>
<id>copy-hook-shim</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependency/hook/falcon</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>falcon-bridge-shim</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
<version>${project.version}</version>
</artifactItem>
</artifactItems> </artifactItems>
</configuration> </configuration>
</execution> </execution>
......
addons/hive-bridge-shim/pom.xml 0 → 100755
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.8-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>hive-bridge-shim</artifactId>
<description>Apache Atlas Hive Bridge Shim Module</description>
<name>Apache Atlas Hive Bridge Shim</name>
<packaging>jar</packaging>
<properties>
<hive.version>1.2.1</hive.version>
<calcite.version>0.9.2-incubating</calcite.version>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
</properties>
<dependencies>
<!-- Logging -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>
addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java 0 → 100755
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.hive.hook;
import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
import org.apache.hadoop.hive.ql.hooks.HookContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Hive hook used for atlas entity registration.
*/
public class HiveHook implements ExecuteWithHookContext {
private static final Logger LOG = LoggerFactory.getLogger(HiveHook.class);
private static final String ATLAS_PLUGIN_TYPE = "hive";
private static final String ATLAS_HIVE_HOOK_IMPL_CLASSNAME = "org.apache.atlas.hive.hook.HiveHook";
private AtlasPluginClassLoader atlasPluginClassLoader = null;
private ExecuteWithHookContext hiveHookImpl = null;
public HiveHook() {
this.initialize();
}
@Override
public void run(final HookContext hookContext) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HiveHook.run(" + hookContext + ")");
}
try {
activatePluginClassLoader();
hiveHookImpl.run(hookContext);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== HiveHook.run(" + hookContext + ")");
}
}
private void initialize() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HiveHook.initialize()");
}
try {
atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass());
@SuppressWarnings("unchecked")
Class<ExecuteWithHookContext> cls = (Class<ExecuteWithHookContext>) Class
.forName(ATLAS_HIVE_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader);
activatePluginClassLoader();
hiveHookImpl = cls.newInstance();
} catch (Exception excp) {
LOG.error("Error instantiating Atlas hook implementation", excp);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== HiveHook.initialize()");
}
}
private void activatePluginClassLoader() {
if (atlasPluginClassLoader != null) {
atlasPluginClassLoader.activate();
}
}
private void deactivatePluginClassLoader() {
if (atlasPluginClassLoader != null) {
atlasPluginClassLoader.deactivate();
}
}
}
addons/hive-bridge/pom.xml
...@@ -168,25 +168,7 @@ ...@@ -168,25 +168,7 @@
<goal>copy</goal> <goal>copy</goal>
</goals> </goals>
<configuration> <configuration>
<artifactItems> <outputDirectory>${project.build.directory}/dependency/hook/hive/atlas-hive-plugin-impl</outputDirectory>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<overWrite>true</overWrite>
<outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory>
</artifactItem>
</artifactItems>
</configuration>
</execution>
<execution>
<id>copy-hook-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory>
<overWriteReleases>false</overWriteReleases> <overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots> <overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer> <overWriteIfNewer>true</overWriteIfNewer>
...@@ -271,6 +253,46 @@ ...@@ -271,6 +253,46 @@
<artifactId>kafka-clients</artifactId> <artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version> <version>${kafka.version}</version>
</artifactItem> </artifactItem>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</artifactItem>
<artifactItem>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>
<execution>
<id>copy-hook-shim</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>hive-bridge-shim</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
<version>${project.version}</version>
</artifactItem>
</artifactItems> </artifactItems>
</configuration> </configuration>
</execution> </execution>
......
addons/sqoop-bridge-shim/pom.xml 0 → 100755
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.8-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>sqoop-bridge-shim</artifactId>
<description>Apache Atlas Sqoop Bridge Shim Module</description>
<name>Apache Atlas Sqoop Bridge Shim</name>
<packaging>jar</packaging>
<properties>
<sqoop.version>1.4.6.2.3.99.0-195</sqoop.version>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
</properties>
<dependencies>
<!-- Logging -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.sqoop</groupId>
<artifactId>sqoop</artifactId>
<version>${sqoop.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
</project>
addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.sqoop.hook;
import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
import org.apache.sqoop.SqoopJobDataPublisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Sqoop hook used for atlas entity registration.
*/
public class SqoopHook extends SqoopJobDataPublisher {
private static final Logger LOG = LoggerFactory.getLogger(SqoopHook.class);
private static final String ATLAS_PLUGIN_TYPE = "sqoop";
private static final String ATLAS_SQOOP_HOOK_IMPL_CLASSNAME = "org.apache.atlas.sqoop.hook.SqoopHook";
private AtlasPluginClassLoader atlasPluginClassLoader = null;
private SqoopJobDataPublisher sqoopHookImpl = null;
public SqoopHook() {
this.initialize();
}
@Override
public void publish(SqoopJobDataPublisher.Data data) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> SqoopHook.run(" + data + ")");
}
try {
activatePluginClassLoader();
sqoopHookImpl.publish(data);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== SqoopHook.run(" + data + ")");
}
}
private void initialize() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HiveHook.initialize()");
}
try {
atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass());
@SuppressWarnings("unchecked")
Class<SqoopJobDataPublisher> cls = (Class<SqoopJobDataPublisher>) Class
.forName(ATLAS_SQOOP_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader);
activatePluginClassLoader();
sqoopHookImpl = cls.newInstance();
} catch (Exception excp) {
LOG.error("Error instantiating Atlas hook implementation", excp);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== HiveHook.initialize()");
}
}
private void activatePluginClassLoader() {
if (atlasPluginClassLoader != null) {
atlasPluginClassLoader.activate();
}
}
private void deactivatePluginClassLoader() {
if (atlasPluginClassLoader != null) {
atlasPluginClassLoader.deactivate();
}
}
}
addons/sqoop-bridge/pom.xml
...@@ -170,29 +170,16 @@ ...@@ -170,29 +170,16 @@
<goal>copy</goal> <goal>copy</goal>
</goals> </goals>
<configuration> <configuration>
<outputDirectory>${project.build.directory}/dependency/hook/sqoop/atlas-sqoop-plugin-impl</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<artifactItems> <artifactItems>
<artifactItem> <artifactItem>
<groupId>${project.groupId}</groupId> <groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId> <artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
<overWrite>true</overWrite>
<outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory>
</artifactItem> </artifactItem>
</artifactItems>
</configuration>
</execution>
<execution>
<id>copy-hook-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<artifactItems>
<artifactItem> <artifactItem>
<groupId>${project.groupId}</groupId> <groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId> <artifactId>${project.artifactId}</artifactId>
...@@ -273,6 +260,46 @@ ...@@ -273,6 +260,46 @@
<artifactId>kafka-clients</artifactId> <artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version> <version>${kafka.version}</version>
</artifactItem> </artifactItem>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</artifactItem>
<artifactItem>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>
<execution>
<id>copy-hook-shim</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>sqoop-bridge-shim</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
<version>${project.version}</version>
</artifactItem>
</artifactItems> </artifactItems>
</configuration> </configuration>
</execution> </execution>
......
addons/storm-bridge-shim/pom.xml 0 → 100755
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.8-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>storm-bridge-shim</artifactId>
<description>Apache Atlas Storm Bridge Shim Module</description>
<name>Apache Atlas Storm Bridge Shim</name>
<packaging>jar</packaging>
<properties>
<storm.version>1.0.0</storm.version>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
</properties>
<dependencies>
<!-- Logging -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<type>jar</type>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>
addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.storm.hook;
import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
import org.apache.storm.ISubmitterHook;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.generated.TopologyInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
/**
* Storm hook used for atlas entity registration.
*/
public class StormAtlasHook implements ISubmitterHook {
private static final Logger LOG = LoggerFactory.getLogger(StormAtlasHook.class);
private static final String ATLAS_PLUGIN_TYPE = "storm";
private static final String ATLAS_STORM_HOOK_IMPL_CLASSNAME = "org.apache.atlas.storm.hook.StormAtlasHook";
private AtlasPluginClassLoader atlasPluginClassLoader = null;
private ISubmitterHook stormHook = null;
public StormAtlasHook() {
this.initialize();
}
@Override
public void notify(TopologyInfo topologyInfo, Map stormConf, StormTopology stormTopology)
throws IllegalAccessException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> StormAtlasHook.notify(" + topologyInfo + ", " + stormConf + ", " + stormTopology + ")");
}
try {
activatePluginClassLoader();
stormHook.notify(topologyInfo, stormConf, stormTopology);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== StormAtlasHook.notify(" + topologyInfo + ", " + stormConf + ", " + stormTopology + ")");
}
}
private void initialize() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> StormAtlasHook.initialize()");
}
try {
atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass());
@SuppressWarnings("unchecked")
Class<ISubmitterHook> cls = (Class<ISubmitterHook>) Class
.forName(ATLAS_STORM_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader);
activatePluginClassLoader();
stormHook = cls.newInstance();
} catch (Exception excp) {
LOG.error("Error instantiating Atlas hook implementation", excp);
} finally {
deactivatePluginClassLoader();
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== StormAtlasHook.initialize()");
}
}
private void activatePluginClassLoader() {
if (atlasPluginClassLoader != null) {
atlasPluginClassLoader.activate();
}
}
private void deactivatePluginClassLoader() {
if (atlasPluginClassLoader != null) {
atlasPluginClassLoader.deactivate();
}
}
}
addons/storm-bridge/pom.xml
...@@ -139,13 +139,13 @@ ...@@ -139,13 +139,13 @@
<artifactId>maven-dependency-plugin</artifactId> <artifactId>maven-dependency-plugin</artifactId>
<executions> <executions>
<execution> <execution>
<id>copy-hook-dependencies</id> <id>copy-hook</id>
<phase>package</phase> <phase>package</phase>
<goals> <goals>
<goal>copy</goal> <goal>copy</goal>
</goals> </goals>
<configuration> <configuration>
<outputDirectory>${project.build.directory}/dependency/hook/storm</outputDirectory> <outputDirectory>${project.build.directory}/dependency/hook/storm/atlas-storm-plugin-impl</outputDirectory>
<overWriteReleases>false</overWriteReleases> <overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots> <overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer> <overWriteIfNewer>true</overWriteIfNewer>
...@@ -315,6 +315,46 @@ ...@@ -315,6 +315,46 @@
<artifactId>hbase-common</artifactId> <artifactId>hbase-common</artifactId>
<version>${hbase.version}</version> <version>${hbase.version}</version>
</artifactItem> </artifactItem>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</artifactItem>
<artifactItem>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>
<execution>
<id>copy-hook-shim</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependency/hook/storm</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>storm-bridge-shim</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
<version>${project.version}</version>
</artifactItem>
</artifactItems> </artifactItems>
</configuration> </configuration>
</execution> </execution>
......
plugin-classloader/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.8-incubating-SNAPSHOT</version>
</parent>
<artifactId>atlas-plugin-classloader</artifactId>
<description>Apache Atlas Plugin Classloader</description>
<name>Apache Atlas Plugin Classloader</name>
<packaging>jar</packaging>
<properties>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
</properties>
<dependencies>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
</dependencies>
</project>
plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.plugin.classloader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URI;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
/**
* AtlasPluginClassLoaderUtil used by AtlasPluginClassLoader.
*/
final class AtlasPluginClassLoaderUtil {
private static final Logger LOG = LoggerFactory.getLogger(AtlasPluginClassLoaderUtil.class);
private static final String ATLAS_PLUGIN_LIBDIR = "atlas-%-plugin-impl";
private AtlasPluginClassLoaderUtil(){ }
public static URL[] getFilesInDirectories(String[] libDirs) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasPluginClassLoaderUtil.getFilesInDirectories()");
}
List<URL> ret = new ArrayList<URL>();
for (String libDir : libDirs) {
getFilesInDirectory(libDir, ret);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasPluginClassLoaderUtil.getFilesInDirectories(): " + ret.size() + " files");
}
return ret.toArray(new URL[]{});
}
private static void getFilesInDirectory(String dirPath, List<URL> files) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasPluginClassLoaderUtil.getPluginFiles()");
}
if (dirPath != null) {
try {
File[] dirFiles = new File(dirPath).listFiles();
if (dirFiles != null) {
for (File dirFile : dirFiles) {
try {
URL jarPath = dirFile.toURI().toURL();
if (LOG.isDebugEnabled()) {
LOG.debug(
"getFilesInDirectory('" + dirPath + "'): adding " + dirFile.getAbsolutePath());
}
files.add(jarPath);
} catch (Exception excp) {
LOG.warn("getFilesInDirectory('" + dirPath + "'): failed to get URI for file " + dirFile
.getAbsolutePath(), excp);
}
}
}
} catch (Exception excp) {
LOG.warn("getFilesInDirectory('" + dirPath + "'): error", excp);
}
} else {
LOG.warn("getFilesInDirectory('" + dirPath + "'): could not find directory in path " + dirPath);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasPluginClassLoaderUtil.getFilesInDirectory(" + dirPath + ")");
}
}
public static String getPluginImplLibPath(String pluginType, Class<?> pluginClass) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasPluginClassLoaderUtil.getPluginImplLibPath for Class (" + pluginClass.getName() + ")");
}
URI uri = pluginClass.getProtectionDomain().getCodeSource().getLocation().toURI();
Path path = Paths.get(URI.create(uri.toString()));
String ret = path.getParent().toString() + File.separatorChar + ATLAS_PLUGIN_LIBDIR.replaceAll("%", pluginType);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasPluginClassLoaderUtil.getPluginImplLibPath for Class " + pluginClass.getName() + "): "
+ ret + ")");
}
return ret;
}
}
plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.plugin.classloader;
import org.testng.Assert;
import org.testng.annotations.Test;
public class AtlasPluginClassLoaderTest {
@Test
public void testClassLoader() throws Exception {
String cls = "org.apache.atlas.service.Services";
try {
loadClass(cls, null);
Assert.fail("Expected ClassNotFoundException");
} catch (ClassNotFoundException e) {
//expected
}
AtlasPluginClassLoader classLoader = new AtlasPluginClassLoader("../common/target");
classLoader.activate();
//org.apache.atlas.service.Services class should be loadable now
//should also load org.apache.atlas.service.Service
Class<?> servicesCls = loadClass(cls, null);
loadClass("org.apache.atlas.service.Service", servicesCls.getClassLoader());
//Fall back to current class loader should also work
loadClass(AtlasPluginClassLoaderUtil.class.getName(), null);
classLoader.deactivate();
//After disable, class loading should fail again
try {
loadClass(cls, null);
Assert.fail("Expected ClassNotFoundException");
} catch (ClassNotFoundException e) {
//expected
}
}
private Class<?> loadClass(String cls, ClassLoader classLoader) throws ClassNotFoundException {
if (classLoader == null) {
classLoader = Thread.currentThread().getContextClassLoader();
}
return Class.forName(cls, true, classLoader);
}
}
pom.xml
...@@ -378,6 +378,7 @@ ...@@ -378,6 +378,7 @@
<log4j.version>1.2.17</log4j.version> <log4j.version>1.2.17</log4j.version>
<akka.version>2.3.7</akka.version> <akka.version>2.3.7</akka.version>
<spray.version>1.3.1</spray.version> <spray.version>1.3.1</spray.version>
<gson.version>2.5</gson.version>
<guava.version>14.0</guava.version> <guava.version>14.0</guava.version>
<fastutil.version>6.5.16</fastutil.version> <fastutil.version>6.5.16</fastutil.version>
<guice.version>4.0</guice.version> <guice.version>4.0</guice.version>
...@@ -470,12 +471,17 @@ ...@@ -470,12 +471,17 @@
<module>catalog</module> <module>catalog</module>
<module>dashboardv2</module> <module>dashboardv2</module>
<module>webapp</module> <module>webapp</module>
<module>docs</module> <module>docs</module>
<module>addons/hdfs-model</module> <module>addons/hdfs-model</module>
<module>plugin-classloader</module>
<module>addons/hive-bridge-shim</module>
<module>addons/hive-bridge</module> <module>addons/hive-bridge</module>
<module>addons/falcon-bridge-shim</module>
<module>addons/falcon-bridge</module> <module>addons/falcon-bridge</module>
<module>addons/sqoop-bridge-shim</module>
<module>addons/sqoop-bridge</module> <module>addons/sqoop-bridge</module>
<module>addons/storm-bridge-shim</module>
<module>addons/storm-bridge</module> <module>addons/storm-bridge</module>
<module>distro</module> <module>distro</module>
...@@ -1111,6 +1117,30 @@ ...@@ -1111,6 +1117,30 @@
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>hive-bridge-shim</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>storm-bridge-shim</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>falcon-bridge-shim</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>sqoop-bridge-shim</artifactId>
<version>${project.version}</version>
</dependency>
<!--Scala dependencies--> <!--Scala dependencies-->
<dependency> <dependency>
<groupId>org.scala-lang</groupId> <groupId>org.scala-lang</groupId>
...@@ -1223,7 +1253,7 @@ ...@@ -1223,7 +1253,7 @@
<dependency> <dependency>
<groupId>com.google.code.gson</groupId> <groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId> <artifactId>gson</artifactId>
<version>2.5</version> <version>${gson.version}</version>
</dependency> </dependency>
<dependency> <dependency>
...@@ -1303,6 +1333,12 @@ ...@@ -1303,6 +1333,12 @@
</exclusion> </exclusion>
</exclusions> </exclusions>
</dependency> </dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-plugin-classloader</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
......
release-log.txt
...@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES: ...@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES: ALL CHANGES:
ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags)
ATLAS-993 If condition in DSL order by clause is not defined then dsl query fails (guptaneeru via shwethags) ATLAS-993 If condition in DSL order by clause is not defined then dsl query fails (guptaneeru via shwethags)
ATLAS-968 Set group information from UGI for Ldap authentication (nixonrodrigues via shwethags) ATLAS-968 Set group information from UGI for Ldap authentication (nixonrodrigues via shwethags)
ATLAS-584 Integrate CSRF prevention filter (kevalbhatt18 via shwethags) ATLAS-584 Integrate CSRF prevention filter (kevalbhatt18 via shwethags)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment