Skip to content

A
atlas
Commit 054dfb76 by Venkatesh Seetharam
Options

Remove falcon and metadata-bridge addon modules

parent fef50cee
Showing with 0 additions and 2427 deletions
addons/falcon-bridge/pom.xml deleted 100755 → 0
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.1-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>falcon-bridge</artifactId>
<description>Apache Atlas Falcon Bridge Module</description>
<name>Apache Atlas Falcon Bridge</name>
<packaging>jar</packaging>
<properties>
<falcon.version>0.6.0.2.2.0.0-2041</falcon.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.falcon</groupId>
<artifactId>falcon-client</artifactId>
<version>${falcon.version}</version>
</dependency>
<!-- falcon-client depends on jersey-client in provided scope. Hence explicit dependency -->
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
</dependency>
</dependencies>
</project>
addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.falcon;
import com.google.inject.Inject;
import org.apache.atlas.MetadataException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.lang.StringUtils;
import org.apache.falcon.client.FalconCLIException;
import org.apache.falcon.client.FalconClient;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.Interface;
import org.apache.falcon.entity.v0.cluster.Location;
import org.apache.falcon.entity.v0.cluster.Properties;
import org.apache.falcon.entity.v0.cluster.Property;
import org.apache.falcon.resource.EntityList;
import javax.xml.bind.JAXBException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class FalconImporter {
private static final TypeSystem typeSystem = TypeSystem.getInstance();
private final FalconClient client;
private final MetadataRepository repository;
@Inject
public FalconImporter(FalconClient client, MetadataRepository repo) {
this.client = client;
this.repository = repo;
}
private Entity getEntity(FalconClient client, EntityType type, String name)
throws FalconCLIException, JAXBException {
String entityStr = client.getDefinition(type.name(), name);
return (Entity) type.getUnmarshaller().unmarshal(new StringReader(entityStr));
}
public void importClusters() throws MetadataException {
try {
EntityList clusters =
client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null);
for (EntityList.EntityElement element : clusters.getElements()) {
Cluster cluster = (Cluster) getEntity(client, EntityType.CLUSTER, element.name);
Referenceable clusterRef = new Referenceable(FalconTypeSystem.DefinedTypes.CLUSTER.name());
clusterRef.set("name", cluster.getName());
if (cluster.getACL() != null) {
Struct acl = new Struct(FalconTypeSystem.DefinedTypes.ACL.name());
acl.set("owner", cluster.getACL().getOwner());
acl.set("group", cluster.getACL().getGroup());
acl.set("permission", cluster.getACL().getPermission());
StructType aclType =
typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.ACL.name());
clusterRef.set("acl", aclType.convert(acl, Multiplicity.REQUIRED));
}
if (StringUtils.isNotEmpty(cluster.getTags())) {
String[] parts = cluster.getTags().split(",");
List<ITypedInstance> tags = new ArrayList<>();
for (String part : parts) {
TraitType tagType =
typeSystem.getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name());
String[] kv = part.trim().split("=");
Struct tag = new Struct(FalconTypeSystem.DefinedTypes.TAG.name());
tag.set("name", kv[0]);
tag.set("value", kv[0]);
tags.add(tagType.convert(tag, Multiplicity.REQUIRED));
}
clusterRef.set("tags", tags);
}
if (cluster.getProperties() != null) {
clusterRef.set("properties", getMap(cluster.getProperties()));
}
if (cluster.getLocations() != null) {
List<ITypedInstance> locations = new ArrayList<>();
for (Location loc : cluster.getLocations().getLocations()) {
Struct location = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name());
EnumType locationType = typeSystem.getDataType(EnumType.class,
FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION_TYPE.name());
location.set("type", locationType.fromValue(loc.getName().toUpperCase()));
location.set("path", loc.getPath());
StructType type = typeSystem
.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name());
locations.add(type.convert(location, Multiplicity.REQUIRED));
}
clusterRef.set("locations", locations);
}
if (cluster.getInterfaces() != null) {
List<ITypedInstance> interfaces = new ArrayList<>();
for (Interface interfaceFld : cluster.getInterfaces().getInterfaces()) {
Struct interfaceStruct = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name());
interfaceStruct.set("type", interfaceFld.getType().name());
interfaceStruct.set("endpoint", interfaceFld.getEndpoint());
interfaceStruct.set("version", interfaceFld.getVersion());
StructType type = typeSystem
.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name());
interfaces.add(type.convert(interfaceStruct, Multiplicity.REQUIRED));
}
clusterRef.set("interfaces", interfaces);
}
repository.createEntity(clusterRef);
}
} catch (Exception e) {
throw new MetadataException(e);
}
}
private Map<String, String> getMap(Properties properties) {
Map<String, String> map = new HashMap();
for (Property property : properties.getProperties()) {
map.put(property.getName().trim(), property.getValue().trim());
}
return map;
}
}
addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.falcon;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.MetadataException;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
public class FalconTypeSystem {
public static final Logger LOG = LoggerFactory.getLogger(FalconTypeSystem.class);
public static final TypeSystem TYPE_SYSTEM = TypeSystem.getInstance();
private static FalconTypeSystem INSTANCE;
private List<StructTypeDefinition> structTypeDefinitions = new ArrayList<>();
private List<HierarchicalTypeDefinition<TraitType>> traitTypeDefinitions = new ArrayList<>();
private FalconTypeSystem() throws MetadataException {
HierarchicalTypeDefinition<ClassType> cluster = defineCluster();
//TODO define feed and process
TYPE_SYSTEM.defineTypes(ImmutableList.copyOf(structTypeDefinitions), ImmutableList.copyOf(traitTypeDefinitions),
ImmutableList.of(cluster));
}
public static FalconTypeSystem getInstance() throws MetadataException {
if (INSTANCE == null) {
synchronized (LOG) {
if (INSTANCE == null) {
INSTANCE = new FalconTypeSystem();
}
}
}
return INSTANCE;
}
private HierarchicalTypeDefinition<ClassType> defineCluster() throws MetadataException {
defineACL();
defineClusterInterface();
defineClusterLocation();
defineTags();
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("acl", DefinedTypes.ACL.name(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("tags", DefinedTypes.TAG.name(), Multiplicity.COLLECTION, false, null),
new AttributeDefinition("locations",
TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(),
Multiplicity.COLLECTION, false, null),
new AttributeDefinition("interfaces", DefinedTypes.CLUSTER_INTERFACE.name(), Multiplicity.COLLECTION,
false, null), new AttributeDefinition("properties",
TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(),
Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> cluster =
new HierarchicalTypeDefinition<>(ClassType.class, DefinedTypes.CLUSTER.name(),
ImmutableList.<String>of(), attributeDefinitions);
LOG.debug("Created definition for " + DefinedTypes.CLUSTER.name());
return cluster;
}
private HierarchicalTypeDefinition<TraitType> defineTags() {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("value", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null)};
HierarchicalTypeDefinition<TraitType> traitType =
new HierarchicalTypeDefinition<>(TraitType.class, DefinedTypes.TAG.name(), ImmutableList.<String>of(),
attributeDefinitions);
LOG.debug("Created definition for " + DefinedTypes.TAG.name());
traitTypeDefinitions.add(traitType);
return traitType;
}
private StructTypeDefinition defineClusterLocation() throws MetadataException {
EnumValue values[] = {new EnumValue("WORKING", 1), new EnumValue("STAGING", 2), new EnumValue("TEMP", 3),};
LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION_TYPE.name());
EnumTypeDefinition locationType = new EnumTypeDefinition(DefinedTypes.CLUSTER_LOCATION_TYPE.name(), values);
TYPE_SYSTEM.defineEnumType(locationType);
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("type", DefinedTypes.CLUSTER_LOCATION_TYPE.name(), Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("path", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),};
LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION.name());
StructTypeDefinition location =
new StructTypeDefinition(DefinedTypes.CLUSTER_LOCATION.name(), attributeDefinitions);
structTypeDefinitions.add(location);
return location;
}
private StructTypeDefinition defineClusterInterface() throws MetadataException {
EnumValue values[] = {new EnumValue("READONLY", 1), new EnumValue("WRITE", 2), new EnumValue("EXECUTE", 3),
new EnumValue("WORKFLOW", 4), new EnumValue("MESSAGING", 5), new EnumValue("REGISTRY", 6),};
LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE_TYPE.name());
EnumTypeDefinition interfaceType = new EnumTypeDefinition(DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), values);
TYPE_SYSTEM.defineEnumType(interfaceType);
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("type", DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), Multiplicity.REQUIRED,
false, null),
new AttributeDefinition("endpoint", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("version", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),};
LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE.name());
StructTypeDefinition interfaceEntity =
new StructTypeDefinition(DefinedTypes.CLUSTER_INTERFACE.name(), attributeDefinitions);
structTypeDefinitions.add(interfaceEntity);
return interfaceEntity;
}
public static enum DefinedTypes {
ACL,
TAG,
CLUSTER,
CLUSTER_INTERFACE,
CLUSTER_INTERFACE_TYPE,
CLUSTER_LOCATION,
CLUSTER_LOCATION_TYPE
}
private StructTypeDefinition defineACL() {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("group", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("permission", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),};
LOG.debug("Created definition for " + DefinedTypes.ACL.name());
StructTypeDefinition acl = new StructTypeDefinition(DefinedTypes.ACL.name(), attributeDefinitions);
structTypeDefinitions.add(acl);
return acl;
}
}
addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.falcon;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.falcon.client.FalconClient;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.Interface;
import org.apache.falcon.entity.v0.cluster.Interfaces;
import org.apache.falcon.entity.v0.cluster.Interfacetype;
import org.apache.falcon.entity.v0.cluster.Location;
import org.apache.falcon.entity.v0.cluster.Locations;
import org.apache.falcon.resource.EntityList;
import org.testng.annotations.Test;
import java.io.StringWriter;
import java.util.UUID;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class FalconImporterTest {
@Test
public void testImport() throws Exception {
MetadataRepository repo = mock(MetadataRepository.class);
FalconClient client = mock(FalconClient.class);
FalconTypeSystem.getInstance();
FalconImporter importer = new FalconImporter(client, repo);
when(client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null))
.thenReturn(getEntityList());
//TODO Set other fields in cluster
when(client.getDefinition(anyString(), anyString())).thenReturn(getCluster());
when(repo.createEntity(any(IReferenceableInstance.class), anyString()))
.thenReturn(UUID.randomUUID().toString());
importer.importClusters();
}
public EntityList getEntityList() {
EntityList.EntityElement[] entities = new EntityList.EntityElement[2];
entities[0] = new EntityList.EntityElement();
entities[0].name = "c1";
entities[1] = new EntityList.EntityElement();
entities[1].name = "c2";
return new EntityList(entities);
}
private Interface getInterface(Interfacetype type, String endpoint) {
Interface clusterInterface = new Interface();
clusterInterface.setEndpoint(endpoint);
clusterInterface.setType(type);
clusterInterface.setVersion("2.2");
return clusterInterface;
}
public String getCluster() throws Exception {
Cluster cluster = new Cluster();
cluster.setName(RandomStringUtils.randomAlphabetic(10));
cluster.setColo(RandomStringUtils.randomAlphabetic(5));
cluster.setTags("owner=xyz,team=abc");
Interfaces interfaces = new Interfaces();
Interface clusterInterface = new Interface();
clusterInterface.setEndpoint("hdfs://localhost:8030");
clusterInterface.setType(Interfacetype.WRITE);
clusterInterface.setVersion("2.2");
interfaces.getInterfaces().add(getInterface(Interfacetype.WRITE, "hdfs://localhost:8030"));
interfaces.getInterfaces().add(getInterface(Interfacetype.READONLY, "hdfs://localhost:8030"));
interfaces.getInterfaces().add(getInterface(Interfacetype.EXECUTE, "http://localhost:8040"));
cluster.setInterfaces(interfaces);
Locations locations = new Locations();
locations.getLocations().add(getLocation());
cluster.setLocations(locations);
StringWriter writer = new StringWriter();
EntityType.CLUSTER.getMarshaller().marshal(cluster, writer);
return writer.toString();
}
public Location getLocation() {
Location location = new Location();
location.setName("staging");
location.setPath("/staging");
return location;
}
}
addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.falcon;
import org.apache.atlas.MetadataException;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.junit.Assert;
import org.testng.annotations.Test;
public class FalconTypeSystemTest {
@Test
public void testTypeSystem() throws MetadataException {
FalconTypeSystem.getInstance();
Assert.assertNotNull(
TypeSystem.getInstance().getDataType(ClassType.class, FalconTypeSystem.DefinedTypes.CLUSTER.name()));
Assert.assertNotNull(
TypeSystem.getInstance().getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name()));
}
}
addons/metadata-bridge-parent/metadata-bridge-core/.gitignore deleted 100755 → 0
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
/test-output/
addons/metadata-bridge-parent/metadata-bridge-core/pom.xml deleted 100755 → 0
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-bridge-parent</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>atlas-bridge-core</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>0.14.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.1.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-multibindings</artifactId>
<version>3.0</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/ABridge.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.MetadataException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
public abstract class ABridge implements IBridge {
protected static final Logger LOG = BridgeManager.LOG;
protected ArrayList<Class<? extends AEntityBean>> typeBeanClasses = new ArrayList<Class<? extends AEntityBean>>();
MetadataRepository repo;
protected ABridge(MetadataRepository repo) {
this.repo = repo;
}
protected HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, ImmutableList<String> superTypes,
AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);
}
public ArrayList<Class<? extends AEntityBean>> getTypeBeanClasses() {
return typeBeanClasses;
}
public AEntityBean get(String id) throws RepositoryException {
// get from the system by id (?)
ITypedReferenceableInstance ref = repo.getEntityDefinition(id);
// turn into a HiveLineageBean
try {
Class<AEntityBean> c = getTypeBeanInListByName(ref.getTypeName());
return this.convertFromITypedReferenceable(ref, getTypeBeanInListByName(ref.getTypeName()));
} catch (BridgeException | InstantiationException | IllegalAccessException |
IllegalArgumentException | InvocationTargetException | NoSuchMethodException |
SecurityException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
public String create(AEntityBean bean) throws MetadataException {
ClassType type = TypeSystem.getInstance().getDataType(ClassType.class, bean.getClass().getSimpleName());
ITypedReferenceableInstance refBean = null;
try {
refBean = type.convert(this.convertToReferencable(bean), Multiplicity.REQUIRED);
String id = repo.createEntity(refBean);
return id;
} catch (IllegalArgumentException | IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
throw new MetadataException("Cannot create entity");
}
public Iterable<String> list() throws RepositoryException {
List<String> returnList = null;
for (Class c : typeBeanClasses) {
List<String> inBetweenList = repo.getEntityList(c.getSimpleName());
try {
returnList.addAll(inBetweenList);
} catch (NullPointerException e) {
returnList = inBetweenList;
}
}
return returnList;
}
protected final boolean containsType(String s) {
for (Class c : typeBeanClasses) {
if (c.getSimpleName().equals(s)) {
return true;
}
}
return false;
}
protected final Class<AEntityBean> getTypeBeanInListByName(String s) throws BridgeException {
if (containsType(s)) {
for (Class c : typeBeanClasses) {
if (c.getSimpleName().equals(s)) {
return c;
}
}
} else {
throw new BridgeException("No EntityBean Definition Found");
}
throw new BridgeException("No EntityBean Definition Found");
}
protected final <T extends AEntityBean> Referenceable convertToReferencable(T o)
throws IllegalArgumentException, IllegalAccessException {
Referenceable selfAware = new Referenceable(o.getClass().getSimpleName());
// TODO - support non-primitive types and deep inspection
for (Field f : o.getClass().getFields()) {
selfAware.set(f.getName(), f.get(o));
}
return selfAware;
}
protected final <T extends AEntityBean> T convertFromITypedReferenceable(ITypedReferenceableInstance instance,
Class<? extends AEntityBean> c)
throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException,
NoSuchMethodException, SecurityException, BridgeException {
if (!instance.getTypeName().equals(c.getSimpleName())) {
throw new BridgeException("ReferenceableInstance type not the same as bean");
}
Object retObj = this.getClass().newInstance();
for (Entry<String, AttributeInfo> e : instance.fieldMapping().fields.entrySet()) {
try {
String convertedName = e.getKey().substring(0, 1).toUpperCase() + e.getKey().substring(1);
this.getClass().getMethod("set" + convertedName, Class.forName(e.getValue().dataType().getName()))
.invoke(this, instance.get(e.getKey()));
} catch (MetadataException | ClassNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
return (T) retObj;
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/AEntityBean.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
public abstract class AEntityBean {
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
import org.apache.hadoop.hive.metastore.api.MetaException;
public class BridgeException extends MetaException {
/**
*
*/
private static final long serialVersionUID = -384401342591560473L;
public BridgeException(String msg) {
super(msg);
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
//TODO - Create Index Annotation Framework for BeanConverter
//TODO - Enhance Bean Conversion to handled nested objects
//TODO - Enhance Bean COnversion to handle Collections
import org.apache.atlas.MetadataException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
public class BridgeManager {
public static final Logger LOG = LoggerFactory.getLogger("BridgeLogger");
private final static String bridgeFileDefault = "bridge-manager.properties";
TypeSystem ts;
MetadataRepository rs;
ArrayList<ABridge> activeBridges;
@Inject
BridgeManager(MetadataRepository rs)
throws ConfigurationException, ClassNotFoundException, InstantiationException, IllegalAccessException,
IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException {
this.ts = TypeSystem.getInstance();
this.rs = rs;
if (System.getProperty("bridgeManager.propsFile") != null
&& System.getProperty("bridgeManager.propsFile").length() != 0) {
setActiveBridges(System.getProperty("bridgeManager.propsFile"));
} else {
setActiveBridges(bridgeFileDefault);
}
for (ABridge bridge : activeBridges) {
try {
this.loadTypes(bridge, ts);
} catch (MetadataException e) {
BridgeManager.LOG.error(e.getMessage(), e);
e.printStackTrace();
}
}
}
public final static HierarchicalTypeDefinition<ClassType> convertEntityBeanToClassTypeDefinition(
Class<? extends AEntityBean> class1) {
ArrayList<AttributeDefinition> attDefAL = new ArrayList<AttributeDefinition>();
for (Field f : class1.getFields()) {
try {
attDefAL.add(BridgeManager.convertFieldtoAttributeDefiniton(f));
} catch (MetadataException e) {
BridgeManager.LOG.error("Class " + class1.getName() +
" cannot be converted to TypeDefinition");
e.printStackTrace();
}
}
HierarchicalTypeDefinition<ClassType> typeDef =
new HierarchicalTypeDefinition<>(ClassType.class, class1.getSimpleName(), null,
(AttributeDefinition[]) attDefAL.toArray(new AttributeDefinition[0]));
return typeDef;
}
public final static AttributeDefinition convertFieldtoAttributeDefiniton(Field f) throws MetadataException {
return new AttributeDefinition(f.getName(), f.getType().getSimpleName(), Multiplicity.REQUIRED, false, null);
}
public ArrayList<ABridge> getActiveBridges() {
return this.activeBridges;
}
private void setActiveBridges(String bridgePropFileName) {
if (bridgePropFileName == null || bridgePropFileName.isEmpty()) {
bridgePropFileName = BridgeManager.bridgeFileDefault;
}
ArrayList<ABridge> aBList = new ArrayList<ABridge>();
PropertiesConfiguration config = new PropertiesConfiguration();
try {
BridgeManager.LOG.info("Loading : Active Bridge List");
config.load(bridgePropFileName);
String[] activeBridgeList = ((String) config.getProperty("BridgeManager.activeBridges")).split(",");
BridgeManager.LOG.info("Loaded : Active Bridge List");
BridgeManager.LOG.info("First Loaded :" + activeBridgeList[0]);
for (String s : activeBridgeList) {
Class<?> bridgeCls = (Class<?>) Class.forName(s);
if (ABridge.class.isAssignableFrom(bridgeCls)) {
System.out.println(s + " is able to be instaciated");
aBList.add((ABridge) bridgeCls.getConstructor(MetadataRepository.class).newInstance(rs));
}
}
} catch (InstantiationException | ConfigurationException | IllegalAccessException |
IllegalArgumentException | InvocationTargetException | NoSuchMethodException |
SecurityException | ClassNotFoundException e) {
BridgeManager.LOG.error(e.getMessage(), e);
e.printStackTrace();
}
this.activeBridges = aBList;
}
private final boolean loadTypes(ABridge bridge, TypeSystem ts) throws MetadataException {
for (Class<? extends AEntityBean> clazz : bridge.getTypeBeanClasses()) {
ts.defineClassType(BridgeManager.convertEntityBeanToClassTypeDefinition(clazz));
}
return false;
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeTypeBootstrapper.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
import org.apache.atlas.MetadataException;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Map;
@Singleton
public class BridgeTypeBootstrapper {
private static final Logger LOG = LoggerFactory.getLogger(BridgeTypeBootstrapper.class);
private final Map<Class, IBridge> bridges;
private boolean isSetup = false;
@Inject
BridgeTypeBootstrapper(Map<Class, IBridge> bridges) throws MetadataException {
this.bridges = bridges;
}
public final static HierarchicalTypeDefinition<ClassType> convertEntityBeanToClassTypeDefinition(
Class<? extends AEntityBean> class1) {
ArrayList<AttributeDefinition> attDefAL = new ArrayList<AttributeDefinition>();
for (Field f : class1.getFields()) {
try {
attDefAL.add(BridgeTypeBootstrapper.convertFieldtoAttributeDefiniton(f));
} catch (MetadataException e) {
BridgeManager.LOG.error("Class " + class1.getName() + " cannot be converted to TypeDefinition");
e.printStackTrace();
}
}
HierarchicalTypeDefinition<ClassType> typeDef =
new HierarchicalTypeDefinition<>(ClassType.class, class1.getSimpleName(), null,
(AttributeDefinition[]) attDefAL.toArray(new AttributeDefinition[0]));
return typeDef;
}
public final static AttributeDefinition convertFieldtoAttributeDefiniton(Field f) throws MetadataException {
return new AttributeDefinition(f.getName(), f.getType().getSimpleName().toLowerCase(), Multiplicity.REQUIRED,
false, null);
}
public synchronized boolean bootstrap() throws MetadataException {
if (isSetup) {
return false;
} else {
LOG.info("Bootstrapping types");
_bootstrap();
isSetup = true;
LOG.info("Bootstrapping complete");
return true;
}
}
private void _bootstrap() throws MetadataException {
TypeSystem ts = TypeSystem.getInstance();
for (IBridge bridge : bridges.values()) {
LOG.info("Registering bridge, %s", bridge.getClass().getSimpleName());
loadTypes(bridge, ts);
}
}
private final boolean loadTypes(IBridge bridge, TypeSystem ts) throws MetadataException {
for (Class<? extends AEntityBean> clazz : bridge.getTypeBeanClasses()) {
LOG.info("Registering %s", clazz.getSimpleName());
ts.defineClassType(BridgeTypeBootstrapper.convertEntityBeanToClassTypeDefinition(clazz));
}
return false;
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/IBridge.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
import java.util.ArrayList;
public interface IBridge {
ArrayList<Class<? extends AEntityBean>> getTypeBeanClasses();
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/HiveLineageBridge.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge.hivelineage;
import org.apache.atlas.bridge.ABridge;
import org.apache.atlas.bridge.hivelineage.hook.HiveLineage;
import org.apache.atlas.repository.MetadataRepository;
import javax.inject.Inject;
public class HiveLineageBridge extends ABridge {
@Inject
HiveLineageBridge(MetadataRepository mr) {
super(mr);
this.typeBeanClasses.add(HiveLineage.class);
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/HiveLineage.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge.hivelineage.hook;
import org.apache.atlas.bridge.AEntityBean;
import java.io.Serializable;
import java.util.ArrayList;
public class HiveLineage extends AEntityBean implements Serializable {
/**
*
*/
private static final long serialVersionUID = 1L;
public String queryId;
public String hiveId;
public String user;
public String queryStartTime;
public String queryEndTime;
public String query;
public String tableName;
public String tableLocation;
public boolean success;
public boolean failed;
public String executionEngine;
ArrayList<SourceTables> sourceTables;
ArrayList<QueryColumns> queryColumns;
ArrayList<WhereClause> whereClause;
ArrayList<CreateColumns> createColumns;
ArrayList<GroupBy> groupBy;
ArrayList<GroupBy> orderBy;
public String getQueryId() {
return this.queryId;
}
public void setQueryId(String queryId) {
this.queryId = queryId;
}
public String getExecutionEngine() {
return this.executionEngine;
}
public void setExecutionEngine(String executionEngine) {
this.executionEngine = executionEngine;
}
public String getHiveId() {
return this.hiveId;
}
public void setHiveId(String hiveId) {
this.hiveId = hiveId;
}
public boolean getSuccess() {
return this.success;
}
public void setSuccess(boolean success) {
this.success = success;
}
public boolean getFailed() {
return this.failed;
}
public void setFailed(boolean failed) {
this.failed = failed;
}
public String getTableName() {
return this.tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public String getTableLocation() {
return this.tableLocation;
}
public void setTableLocation(String tableLocation) {
this.tableLocation = tableLocation;
}
public String getUser() {
return this.user;
}
public void setUser(String user) {
this.user = user;
}
public String getQueryStartTime() {
return this.queryStartTime;
}
public void setQueryStartTime(String queryStartTime) {
this.queryStartTime = queryStartTime;
}
public String getQueryEndTime() {
return this.queryEndTime;
}
public void setQueryEndTime(String queryEndTime) {
this.queryEndTime = queryEndTime;
}
public String getQuery() {
return this.query;
}
public void setQuery(String query) {
this.query = query;
}
public ArrayList<SourceTables> getSourceTables() {
return this.sourceTables;
}
public void setSourceTables(ArrayList<SourceTables> sourceTables) {
this.sourceTables = sourceTables;
}
public ArrayList<QueryColumns> getQueryColumns() {
return this.queryColumns;
}
public void setQueryColumns(ArrayList<QueryColumns> queryColumns) {
this.queryColumns = queryColumns;
}
public ArrayList<WhereClause> getWhereClause() {
return this.whereClause;
}
public void setWhereClause(ArrayList<WhereClause> whereClause) {
this.whereClause = whereClause;
}
public ArrayList<GroupBy> getGroupBy() {
return this.groupBy;
}
public void setGroupBy(ArrayList<GroupBy> groupBy) {
this.groupBy = groupBy;
}
public ArrayList<CreateColumns> getCreateColumns() {
return this.createColumns;
}
public void setCreateColumns(ArrayList<CreateColumns> createColumns) {
this.createColumns = createColumns;
}
public class SourceTables {
public String tableName;
public String tableAlias;
public String databaseName;
public String getTableName() {
return this.tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public String getTableAlias() {
return this.tableAlias;
}
public void setTableAlias(String tableAlias) {
this.tableAlias = tableAlias;
}
public String getDatabaseName() {
return this.databaseName;
}
public void setDatabaseName(String databaseName) {
this.databaseName = databaseName;
}
}
public class QueryColumns {
public String tbAliasOrName;
public String columnName;
public String columnAlias;
public String columnFunction;
public String getTbAliasOrName() {
return this.tbAliasOrName;
}
public void setTbAliasOrName(String tbAliasOrName) {
this.tbAliasOrName = tbAliasOrName;
}
public String getColumnName() {
return this.columnName;
}
public void setColumnName(String columnName) {
this.columnName = columnName;
}
public String getColumnAlias() {
return this.columnAlias;
}
public void setColumnAlias(String columnAlias) {
this.columnAlias = columnAlias;
}
public String getColumnFunction() {
return this.columnFunction;
}
public void setColumnFunction(String columnFunction) {
this.columnFunction = columnFunction;
}
}
public class GroupBy {
public String tbAliasOrName;
public String columnName;
public String getTbAliasOrName() {
return this.tbAliasOrName;
}
public void setTbAliasOrName(String tbAliasOrName) {
this.tbAliasOrName = tbAliasOrName;
}
public String getColumnName() {
return this.columnName;
}
public void setColumnName(String columnName) {
this.columnName = columnName;
}
}
public class WhereClause {
public String tbAliasOrName;
public String columnCondition;
public String columnName;
public String columnOperator;
public String columnValue;
public String getColumnCondition() {
return this.columnCondition;
}
public void setColumnCondition(String columnCondition) {
this.columnCondition = columnCondition;
}
public String getTbAliasOrName() {
return this.tbAliasOrName;
}
public void setTbAliasOrName(String tbAliasOrName) {
this.tbAliasOrName = tbAliasOrName;
}
public String getColumnName() {
return this.columnName;
}
public void setColumnName(String columnName) {
this.columnName = columnName;
}
public String getColumnOperator() {
return this.columnOperator;
}
public void setColumnOperator(String columnOperator) {
this.columnOperator = columnOperator;
}
public String getColumnValue() {
return this.columnValue;
}
public void setColumnValue(String columnValue) {
this.columnValue = columnValue;
}
}
public class CreateColumns {
public String columnName;
public String columnType;
public String getColumnName() {
return this.columnName;
}
public void setColumnName(String columnName) {
this.columnName = columnName;
}
public String getColumnType() {
return this.columnType;
}
public void setColumnType(String columnType) {
this.columnType = columnType;
}
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveMetaImporter.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge.hivestructure;
import org.apache.atlas.MetadataException;
import org.apache.atlas.repository.IRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.thrift.TException;
/*
* Initial pass at one time importer TODO - needs re-write
*/
public class HiveMetaImporter {
private static HiveMetaStoreClient msc;
private static IRepository repo;
public HiveMetaImporter(IRepository repo) {
try {
this.repo = repo;
msc = new HiveMetaStoreClient(new HiveConf());
// TODO Get hive-site.conf from class path first
} catch (MetaException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static boolean fullImport() {
try {
databasesImport();
for (String dbName : msc.getAllDatabases()) {
tablesImport(dbName);
for (String tbName : msc.getAllTables(dbName)) {
fieldsImport(dbName, tbName);
}
return true;
}
} catch (MetaException me) {
me.printStackTrace();
} catch (RepositoryException re) {
re.printStackTrace();
}
return false;
}
public static boolean databasesImport() throws MetaException, RepositoryException {
ClassType classType = null;
try {
classType = TypeSystem.getInstance().getDataType(ClassType.class, HiveStructureBridge.DB_CLASS_TYPE);
} catch (MetadataException e1) {
e1.printStackTrace();
}
for (String dbName : msc.getAllDatabases()) {
databaseImport(dbName);
}
return true;
}
public static boolean databaseImport(String dbName) throws MetaException, RepositoryException {
try {
Database db = msc.getDatabase(dbName);
Referenceable dbRef = new Referenceable(HiveStructureBridge.DB_CLASS_TYPE);
dbRef.set("DESC", db.getDescription());
dbRef.set("DB_LOCATION_URI", db.getLocationUri());
dbRef.set("NAME", db.getName());
if (db.isSetOwnerType()) {
dbRef.set("OWNER_TYPE", db.getOwnerType());
}
if (db.isSetOwnerName()) {
dbRef.set("OWNER_NAME", db.getOwnerName());
}
repo.create(dbRef);
} catch (NoSuchObjectException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (TException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return true;
}
public static boolean tablesImport(String dbName) throws MetaException, RepositoryException {
ClassType classType = null;
try {
classType = TypeSystem.getInstance().getDataType(ClassType.class, HiveStructureBridge.TB_CLASS_TYPE);
} catch (MetadataException e1) {
e1.printStackTrace();
}
for (String tbName : msc.getAllTables(dbName)) {
tableImport(dbName, tbName);
}
return true;
}
public static boolean tableImport(String dbName, String tbName) throws MetaException, RepositoryException {
try {
Table tb = msc.getTable(dbName, tbName);
Referenceable tbRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE);
tbRef.set("CREATE_TIME", tb.getCreateTime());
tbRef.set("LAST_ACCESS_TIME", tb.getLastAccessTime());
tbRef.set("OWNER", tb.getOwner());
tbRef.set("TBL_NAME", tb.getTableName());
tbRef.set("TBL_TYPE", tb.getTableType());
if (tb.isSetViewExpandedText()) {
tbRef.set("VIEW_EXPANDED_TEXT", tb.getViewExpandedText());
}
if (tb.isSetViewOriginalText()) {
tbRef.set("VIEW_ORIGINAL_TEXT", tb.getViewOriginalText());
}
repo.create(tbRef);
} catch (NoSuchObjectException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (TException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return true;
}
public static boolean fieldsImport(String dbName, String tbName) throws MetaException, RepositoryException {
ClassType classType = null;
try {
classType = TypeSystem.getInstance().getDataType(ClassType.class, HiveStructureBridge.FD_CLASS_TYPE);
} catch (MetadataException e1) {
e1.printStackTrace();
}
try {
for (FieldSchema fs : msc.getFields(dbName, tbName)) {
Referenceable fdRef = new Referenceable(HiveStructureBridge.FD_CLASS_TYPE);
if (fs.isSetComment()) {
fdRef.set("COMMENT", fs.getName());
}
fdRef.set("COLUMN_NAME", fs.getName());
fdRef.set("TYPE_NAME", fs.getType());
repo.create(fdRef);
}
} catch (UnknownTableException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (UnknownDBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (TException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return true;
}
public static boolean fieldImport(String dbName, String tbName, String fdName) throws MetaException {
try {
for (FieldSchema fs : msc.getFields(dbName, tbName)) {
if (fs.getName().equals(fs)) {
Referenceable fdRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE);
if (fs.isSetComment()) {
fdRef.set("COMMENT", fs.getName());
}
fdRef.set("COLUMN_NAME", fs.getName());
fdRef.set("TYPE_NAME", fs.getType());
//SaveObject to MS Backend
return true;
}
}
} catch (UnknownTableException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (UnknownDBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (TException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return true;
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge.hivestructure;
import org.apache.atlas.MetadataException;
import org.apache.atlas.bridge.ABridge;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import javax.inject.Inject;
import java.util.ArrayList;
public class HiveStructureBridge extends ABridge {
static final String DB_CLASS_TYPE = "HiveDatabase";
static final String TB_CLASS_TYPE = "HiveTable";
static final String FD_CLASS_TYPE = "HiveField";
@Inject
protected HiveStructureBridge(MetadataRepository repo) {
super(repo);
// TODO Auto-generated constructor stub
}
public boolean defineBridgeTypes(TypeSystem ts) {
ArrayList<HierarchicalTypeDefinition<?>> al = new ArrayList<HierarchicalTypeDefinition<?>>();
// TODO
//convert to helper methods
// Add to arrayList
try {
HierarchicalTypeDefinition<ClassType> databaseClassTypeDef =
new HierarchicalTypeDefinition<ClassType>("ClassType", DB_CLASS_TYPE, null,
new AttributeDefinition[]{
new AttributeDefinition("DESC", "STRING_TYPE", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("DB_LOCATION_URI", "STRING_TYPE", Multiplicity.REQUIRED,
false, null),
new AttributeDefinition("NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("OWNER_TYPE", "STRING_TYPE", Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("OWNER_NAME", "STRING_TYPE", Multiplicity.OPTIONAL, false,
null)});
HierarchicalTypeDefinition<ClassType> tableClassTypeDef =
new HierarchicalTypeDefinition<ClassType>("ClassType", TB_CLASS_TYPE, null,
new AttributeDefinition[]{
new AttributeDefinition("CREATE_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("LAST_ACCESS_TIME", "LONG_TYPE", Multiplicity.REQUIRED,
false, null),
new AttributeDefinition("OWNER", "STRING_TYPE", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("TBL_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("TBL_TYPE", "STRING_TYPE", Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("VIEW_EXPANDED_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL,
false, null),
new AttributeDefinition("VIEW_ORIGINAL_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL,
false, null)});
HierarchicalTypeDefinition<ClassType> columnClassTypeDef =
new HierarchicalTypeDefinition<ClassType>("ClassType", FD_CLASS_TYPE, null,
new AttributeDefinition[]{
new AttributeDefinition("COMMENT", "STRING_TYPE", Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("COLUMN_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("TYPE_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false,
null)});
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
for (HierarchicalTypeDefinition htd : al) {
try {
ts.defineClassType(htd);
} catch (MetadataException e) {
System.out.println(htd.hierarchicalMetaTypeName + "could not be added to the type system");
e.printStackTrace();
}
}
return false;
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/module/BridgeModule.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge.module;
import com.google.inject.AbstractModule;
import com.google.inject.Scopes;
import com.google.inject.multibindings.MapBinder;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.bridge.BridgeTypeBootstrapper;
import org.apache.atlas.bridge.IBridge;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
public class BridgeModule extends AbstractModule {
public static final Logger LOG = LoggerFactory.getLogger(BridgeModule.class);
@Override
protected void configure() {
install(new RepositoryMetadataModule());
// make sure the BridgeTypeBootstrapper is only ever created once
bind(BridgeTypeBootstrapper.class).in(Scopes.SINGLETON);
// Load the configured bridge classes and add them to the map binder
MapBinder<Class, IBridge> mapbinder = MapBinder.newMapBinder(binder(), Class.class, IBridge.class);
String propsURI = System.getProperty("bridgeManager.propsFile", "bridge-manager.properties");
List<Class<? extends IBridge>> bridges = getBridgeClasses(propsURI);
for (Class<? extends IBridge> bridgeClass : bridges) {
mapbinder.addBinding(bridgeClass).to(bridgeClass).in(Scopes.SINGLETON);
}
}
/*
* Get the bridge classes from the configuration file
*/
private List<Class<? extends IBridge>> getBridgeClasses(String bridgePropFileName) {
List<Class<? extends IBridge>> aBList = new ArrayList<Class<? extends IBridge>>();
PropertiesConfiguration config = new PropertiesConfiguration();
try {
LOG.info("Loading : Active Bridge List");
config.load(bridgePropFileName);
String[] activeBridgeList = ((String) config.getProperty("BridgeManager.activeBridges")).split(",");
LOG.info("Loaded : Active Bridge List");
for (String s : activeBridgeList) {
Class<? extends IBridge> bridgeCls = (Class<? extends IBridge>) Class.forName(s);
aBList.add(bridgeCls);
}
} catch (ConfigurationException | IllegalArgumentException
| SecurityException | ClassNotFoundException e) {
LOG.error(e.getMessage(), e);
e.printStackTrace();
}
return aBList;
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.resources;
import org.apache.atlas.bridge.hivelineage.HiveLineageBridge;
import javax.inject.Singleton;
//@Path("bridge/hive")
@Singleton
public class HiveLineageResource {
private final HiveLineageBridge bridge = null;
/*
//@Inject
public HiveLineageResource(HiveLineageBridge bridge) {
this.bridge = bridge;
}
//@Inject
public HiveLineageResource(Map<Class<? extends IBridge>, IBridge> bridges) {
this.bridge = (HiveLineageBridge) bridges.get(HiveLineageBridge.class);
}
@GET
@Path("/{id}")
@Produces(MediaType.APPLICATION_JSON)
public JsonElement getById(@PathParam("id") String id) throws RepositoryException {
// get the lineage bean
HiveLineage hlb = (HiveLineage) bridge.get(id);
// turn it into a JsonTree & return
return new Gson().toJsonTree(hlb);
}
@GET
@Produces(MediaType.APPLICATION_JSON)
public JsonElement list() throws RepositoryException {
// make a new JsonArray to be returned
JsonArray ja = new JsonArray();
// iterate over each item returned by the hive bridge's list() method
for (String s: bridge.list()) {
// they are GUIDs so make them into JsonPrimitives
ja.add(new JsonPrimitive(s));
}
return ja;
}
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public JsonElement addLineage(@Context HttpServletRequest request)
throws IOException, MetadataException {
// create a reader
try (Reader reader = new InputStreamReader(request.getInputStream())) {
// deserialize
HiveLineage bean = new Gson().fromJson(reader, HiveLineage.class);
String id = bridge.create(bean);
JsonObject jo = new JsonObject();
jo.addProperty("id", id);
return jo;
}
}
*/
}
addons/metadata-bridge-parent/metadata-bridge-core/src/main/resources/bridge-manager.properties deleted 100755 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
#
BridgeManager.activeBridges=org.apache.atlas.bridge.hivelineage.HiveLineageBridge
\ No newline at end of file
addons/metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/BridgeManagerTest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.MetadataRepository;
import org.testng.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
@Guice(modules = RepositoryMetadataModule.class)
public class BridgeManagerTest {
@Inject
MetadataRepository repo;
@Test(enabled = false)
public void testLoadPropertiesFile() throws Exception {
BridgeManager bm = new BridgeManager(repo);
System.out.println(bm.getActiveBridges().size());
Assert.assertEquals(bm.activeBridges.get(0).getClass().getSimpleName(), "HiveLineageBridge");
}
@Test
public void testBeanConvertion() {
//Tests Conversion of Bean to Type
}
@Test
public void testIRefConvertion() {
//Tests Conversion of IRef cast to Bean
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/TestBridgeModule.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
import org.apache.atlas.bridge.module.BridgeModule;
import org.testng.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@Guice(modules = {BridgeModule.class})
public class TestBridgeModule {
@Test
public void loadAnything() {
// if it makes it here, the BridgeModule loaded successfully
Assert.assertTrue(true);
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/TestGenericBridges.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge;
public class TestGenericBridges {
//TODO Build Generic Tests for non-lineage Bridge
}
addons/metadata-bridge-parent/metadata-bridge-core/src/test/java/org/apache/hadoop/metadata/bridge/hivelineage/TestHiveLineageBridge.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge.hivelineage;
import com.google.gson.Gson;
import org.apache.atlas.MetadataException;
import org.apache.atlas.bridge.BridgeTypeBootstrapper;
import org.apache.atlas.bridge.hivelineage.hook.HiveLineage;
import org.apache.atlas.bridge.module.BridgeModule;
import org.apache.atlas.repository.RepositoryException;
import org.apache.commons.collections.IteratorUtils;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.List;
@Guice(modules = {BridgeModule.class})
public class TestHiveLineageBridge {
@Inject
HiveLineageBridge bridge;
@Inject
BridgeTypeBootstrapper bootstrapper;
HiveLineage hlb;
// the id of one.json in the repo (test #1)
String oneId;
private HiveLineage loadHiveLineageBean(String path) throws IOException {
return new Gson().fromJson(new InputStreamReader(this.getClass().getResourceAsStream(path)), HiveLineage.class);
}
@BeforeClass
public void bootstrap() throws IOException, MetadataException {
bootstrapper.bootstrap();
hlb = loadHiveLineageBean("/one.json");
}
@Test(priority = 1, enabled = false)
public void testCreate() throws MetadataException {
// add the lineage bean to the repo
oneId = bridge.create(hlb);
// make sure this actually did worked
Assert.assertNotNull(oneId);
}
@Test(priority = 2, enabled = false)
public void testGet() throws RepositoryException, IOException {
Object bean = bridge.get(oneId);
Assert.assertEquals(hlb, bean);
}
@Test(priority = 3, enabled = false)
public void testList() throws RepositoryException {
List<String> list = IteratorUtils.toList(bridge.list().iterator());
Assert.assertEquals(list.size(), 1);
Assert.assertEquals(list.get(0), oneId);
}
}
addons/metadata-bridge-parent/metadata-bridge-core/src/test/resources/bridge-manager.properties deleted 100755 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
#
BridgeManager.activeBridges=org.apache.atlas.bridge.hivelineage.HiveLineageBridge
\ No newline at end of file
addons/metadata-bridge-parent/metadata-bridge-core/src/test/resources/graph.properties deleted 100755 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
storage.backend=inmemory
# Graph Search Index
index.search.backend=elasticsearch
index.search.directory=target/data/es
index.search.elasticsearch.client-only=false
index.search.elasticsearch.local-mode=true
\ No newline at end of file
addons/metadata-bridge-parent/metadata-bridge-core/src/test/resources/one.json deleted 100755 → 0
{
"queryId": "a760104_20150106120303_036186d5-a991-4dfc-9ff2-05b072c7e711",
"hiveId": "90797386-3933-4ab0-ae68-a7baa7e155d4",
"user": "",
"queryStartTime": "1420563838114",
"queryEndTime": "1420563853806",
"query": "create table nyse_gss_count_dump as select count(nyse.stock_symbol) stock_symbol_count, stock_symbol from nyse_stocks nyse where (nyse.stock_symbol \u003d \u0027AET\u0027 or nyse.stock_symbol \u003d \u0027UNH\u0027 ) and nyse.stock_symbol \u003d \u0027T\u0027 GROUP by stock_symbol",
"tableName": "nyse_gss_count_dump",
"success": true,
"failed": false,
"executionEngine": "tez",
"sourceTables": [
{
"tableName": "nyse_stocks",
"tableAlias": "nyse"
}
],
"queryColumns": [
{
"tbAliasOrName": "nyse",
"columnName": "stock_symbol",
"columnAlias": "stock_symbol_count",
"columnFunction": "count"
},
{"columnName": "stock_symbol"}
],
"whereClause": [
{
"tbAliasOrName": "nyse",
"columnName": "stock_symbol",
"columnOperator": "\u003d",
"columnValue": "\u0027AET\u0027"
},
{
"tbAliasOrName": "nyse",
"columnName": "stock_symbol",
"columnOperator": "\u003d",
"columnValue": "\u0027UNH\u0027"
},
{
"tbAliasOrName": "nyse",
"columnName": "stock_symbol",
"columnOperator": "\u003d",
"columnValue": "\u0027T\u0027"
}
],
"groupBy": [{"columnName": "stock_symbol"}]
}
\ No newline at end of file
addons/metadata-bridge-parent/metadata-bridge-core/src/test/resources/test-bridge-manager.properties deleted 100755 → 0
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#BridgeManager.activebridges denotes which bridge defintions to load from the classpath (Comma seperated list of fully qualified class paths)
#
BridgeManager.activeBridges=org.apache.atlas.bridge.HiveLineage
\ No newline at end of file
addons/metadata-bridge-parent/metadata-bridge-core/src/test/resources/two.json deleted 100755 → 0
{
"queryId": "a760104_20150108124747_53cb7716-8756-4dfe-b746-4055f53e2895",
"hiveId": "1aebd95c-c7d5-4893-8c8c-c9ae098bdd5c",
"user": "",
"queryStartTime": "1420739257453",
"queryEndTime": "1420739277589",
"query": "create table nyse_gss_count_dump as select count(nyse.stock_symbol) stock_symbol_count, stock_symbol from nyse_stocks nyse where (nyse.stock_symbol \u003d \u0027AET\u0027 or nyse.stock_symbol \u003d \u0027UNH\u0027 ) and nyse.stock_symbol \u003d \u0027T\u0027 GROUP by stock_symbol",
"tableName": "nyse_gss_count_dump",
"success": true,
"failed": false,
"executionEngine": "tez",
"sourceTables": [
{
"tableName": "nyse_stocks",
"tableAlias": "nyse"
}
],
"queryColumns": [
{
"tbAliasOrName": "nyse",
"columnName": "stock_symbol",
"columnAlias": "stock_symbol_count",
"columnFunction": "count"
},
{"columnName": "stock_symbol"}
],
"whereClause": [
{
"tbAliasOrName": "nyse",
"columnName": "stock_symbol",
"columnOperator": "\u003d",
"columnValue": "\u0027AET\u0027"
},
{
"tbAliasOrName": "nyse",
"columnName": "stock_symbol",
"columnOperator": "\u003d",
"columnValue": "\u0027UNH\u0027"
},
{
"tbAliasOrName": "nyse",
"columnName": "stock_symbol",
"columnOperator": "\u003d",
"columnValue": "\u0027T\u0027"
}
],
"groupBy": [{"columnName": "stock_symbol"}]
}
\ No newline at end of file
addons/metadata-bridge-parent/pom.xml deleted 100755 → 0
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>atlas-bridge-parent</artifactId>
<packaging>pom</packaging>
<modules>
<module>atlas-bridge-core</module>
<module>atlas-bridge-hive</module>
</modules>
</project>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment