Skip to content

A
atlas
Commit 1bfda02a by Shwetha GS
Options

ATLAS-58 Make hive hook reliable (shwethags)

parent 016e36c0
Showing with 1451 additions and 1272 deletions
addons/hive-bridge/pom.xml
......@@ -33,7 +33,7 @@
<packaging>jar</packaging>
<properties>
<hive.version>1.2.0</hive.version>
<hive.version>1.2.1</hive.version>
<calcite.version>0.9.2-incubating</calcite.version>
</properties>
......@@ -98,6 +98,11 @@
<artifactId>atlas-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-notification</artifactId>
</dependency>
<!-- to bring up atlas server for integration tests -->
<dependency>
<groupId>org.apache.atlas</groupId>
......@@ -149,7 +154,7 @@
</configuration>
</execution>
<execution>
<id>copy</id>
<id>copy-hook</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
......@@ -209,6 +214,16 @@
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>atlas-notification</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>atlas-common</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
......@@ -228,6 +243,21 @@
<artifactId>scalap</artifactId>
<version>${scala.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-multibindings</artifactId>
<version>${guice.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_${scala.binary.version}</artifactId>
<version>${kafka.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>
......@@ -253,12 +283,16 @@
<useTestScope>true</useTestScope>
<systemProperties>
<systemProperty>
<name>log4j.configuration</name>
<value>atlas-log4j.xml</value>
</systemProperty>
<systemProperty>
<name>atlas.log.dir</name>
<value>${project.build.directory}/logs</value>
</systemProperty>
</systemProperties>
<stopKey>atlas-stop</stopKey>
<stopPort>41001</stopPort>
<stopPort>21001</stopPort>
</configuration>
<executions>
<execution>
......@@ -302,7 +336,6 @@
<configuration>
<generateProjectInfo>false</generateProjectInfo>
<generateReports>false</generateReports>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
......
addons/hive-bridge/src/bin/import-hive.sh
......@@ -67,7 +67,7 @@ done
METADATA_LOG_DIR="${METADATA_LOG_DIR:-$BASEDIR/logs}"
export METADATA_LOG_DIR
JAVA_PROPERTIES="$METADATA_OPTS -Datlas.log.dir=$METADATA_LOG_DIR -Datlas.log.file=import-hive.log"
JAVA_PROPERTIES="$METADATA_OPTS -Datlas.log.dir=$METADATA_LOG_DIR -Datlas.log.file=import-hive.log -Dlog4j.configuration=atlas-log4j.xml"
shift
while [[ ${1} =~ ^\-D ]]; do
......
client/pom.xml
......@@ -64,11 +64,6 @@
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
......
common/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>0.6-incubating-SNAPSHOT</version>
</parent>
<artifactId>atlas-common</artifactId>
<description>Apache Atlas Common</description>
<name>Apache Atlas Common</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
</dependencies>
</project>
common/src/main/java/org/apache/atlas/service/Service.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.service;
import org.apache.atlas.AtlasException;
/**
* Service interface to start any background jobs
*/
public interface Service {
void start() throws AtlasException;
void stop();
}
common/src/main/java/org/apache/atlas/service/Services.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.service;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Set;
/**
* Utility for starting and stopping all services
*/
@Singleton
public class Services {
public static final Logger LOG = LoggerFactory.getLogger(Services.class);
private final Set<Service> services;
@Inject
public Services(Set<Service> services) {
this.services = services;
}
public void start() {
try {
for (Service service : services) {
LOG.debug("Starting service {}", service.getClass().getName());
service.start();
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public void stop() {
for (Service service : services) {
LOG.debug("Stopping service {}", service.getClass().getName());
service.stop();
}
}
}
dashboard/public/modules/details/detailsResource.js
......@@ -19,7 +19,7 @@
'use strict';
angular.module('dgc.details').factory('DetailsResource', ['$resource', function($resource) {
return $resource('/api/atlas/entities/:id', {}, {
return $resource('/api/atlas/entity/:id', {}, {
get: {
method: 'GET',
transformResponse: function(data) {
......
dashboard/public/modules/search/searchController.js
......@@ -43,7 +43,7 @@ angular.module('dgc.search').controller('SearchController', ['$scope', '$locatio
}, function searchSuccess(response) {
$scope.resultCount = response.count;
$scope.results = response.results;
$scope.resultRows = $scope.results.rows;
$scope.resultRows = $scope.results;
$scope.totalItems = $scope.resultCount;
$scope.transformedResults = {};
$scope.dataTransitioned = false;
......@@ -59,7 +59,7 @@ angular.module('dgc.search').controller('SearchController', ['$scope', '$locatio
} else {
$scope.transformedResults = $scope.resultRows;
}
if ($scope.results.rows)
if ($scope.results)
$scope.searchMessage = $scope.resultCount + ' results matching your search query ' + $scope.query + ' were found';
else
$scope.searchMessage = '0 results matching your search query ' + $scope.query + ' were found';
......
distro/pom.xml
......@@ -57,6 +57,35 @@
</python.path.l>
</properties>
</profile>
<profile>
<id>dist</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<descriptors>
<descriptor>src/main/assemblies/standalone-package.xml</descriptor>
<descriptor>src/main/assemblies/src-package.xml</descriptor>
</descriptors>
<finalName>apache-atlas-${project.version}</finalName>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<build>
......@@ -69,6 +98,7 @@
<executions>
<execution>
<configuration>
<skip>${skipTests}</skip>
<executable>python</executable>
<workingDirectory>src/test/python</workingDirectory>
<arguments>
......@@ -86,26 +116,6 @@
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<inherited>false</inherited>
<configuration>
<descriptors>
<descriptor>src/main/assemblies/standalone-package.xml</descriptor>
<descriptor>src/main/assemblies/src-package.xml</descriptor>
</descriptors>
<finalName>apache-atlas-${project.version}</finalName>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
......
distro/src/bin/atlas_start.py
......@@ -24,7 +24,7 @@ import atlas_config as mc
METADATA_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=application.log"
METADATA_COMMAND_OPTS="-Datlas.home=%s"
METADATA_CONFIG_OPTS="-Datlas.conf=%s"
DEFAULT_JVM_OPTS="-Xmx1024m -Dlog4j.configuration=atlas-log4j.xml"
DEFAULT_JVM_OPTS="-Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml"
def main():
......
distro/src/conf/application.properties
......@@ -45,9 +45,16 @@ atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
######### Notification Configs #########
atlas.notification.embedded=true
atlas.notification.kafka.data=${sys:atlas.home}/data/kafka
atlas.kafka.data=${sys:atlas.home}/data/kafka
atlas.kafka.zookeeper.connect=localhost:9026
atlas.kafka.bootstrap.servers=localhost:9027
atlas.kafka.zookeeper.session.timeout.ms=400
atlas.kafka.zookeeper.sync.time.ms=20
atlas.kafka.auto.commit.interval.ms=1000
######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
......
distro/src/main/assemblies/standalone-package.xml
......@@ -50,13 +50,6 @@
</fileSet>
<fileSet>
<directory>../addons/hive-bridge/src/bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>../logs</directory>
<outputDirectory>logs</outputDirectory>
<directoryMode>0777</directoryMode>
......@@ -85,6 +78,13 @@
<!-- addons/hive -->
<fileSet>
<directory>../addons/hive-bridge/src/bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>../addons/hive-bridge/target/dependency/bridge</directory>
<outputDirectory>bridge</outputDirectory>
</fileSet>
......@@ -93,12 +93,6 @@
<directory>../addons/hive-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
<fileSet>
<directory>../addons/hive-bridge/target/site</directory>
<outputDirectory>docs/hive</outputDirectory>
</fileSet>
</fileSets>
<files>
......
distro/src/test/python/scripts/TestMetadata.py
......@@ -51,13 +51,13 @@ class TestMetadata(unittest.TestCase):
'org.apache.atlas.Main',
['-app', 'metadata_home/server/webapp/atlas'],
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib\\*:metadata_home/libext\\*',
['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m', '-Dlog4j.configuration=atlas-log4j.xml'], 'metadata_home/logs')
['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m', '-XX:MaxPermSize=512m', '-Dlog4j.configuration=atlas-log4j.xml'], 'metadata_home/logs')
else:
java_mock.assert_called_with(
'org.apache.atlas.Main',
['-app', 'metadata_home/server/webapp/atlas'],
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib/*:metadata_home/libext/*',
['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m', '-Dlog4j.configuration=atlas-log4j.xml'], 'metadata_home/logs')
['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m', '-XX:MaxPermSize=512m', '-Dlog4j.configuration=atlas-log4j.xml'], 'metadata_home/logs')
pass
def test_jar_java_lookups_fail(self):
......
docs/pom.xml
......@@ -40,18 +40,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
<version>3.3</version>
<dependencies>
<dependency>
<groupId>org.apache.maven.doxia</groupId>
<artifactId>doxia-module-twiki</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-ssh-external</artifactId>
<version>2.6</version>
</dependency>
</dependencies>
<executions>
<execution>
......@@ -62,45 +56,28 @@
</execution>
</executions>
<configuration>
<skip>${skipDocs}</skip>
<reportPlugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>2.3</version>
<reportSets>
<reportSet>
<reports>
<report>index</report>
<report>project-team</report>
<report>mailing-list</report>
<report>issue-tracking</report>
<report>license</report>
<report>scm</report>
</reports>
</reportSet>
</reportSets>
<configuration>
<dependencyDetailsEnabled>false</dependencyDetailsEnabled>
<dependencyLocationsEnabled>false</dependencyLocationsEnabled>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.7</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jxr-plugin</artifactId>
<version>2.1</version>
<configuration>
<aggregate>true</aggregate>
</configuration>
</plugin>
</reportPlugins>
<generateProjectInfo>false</generateProjectInfo>
<generateReports>false</generateReports>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>2.8.1</version>
<executions>
<execution>
<goals>
<goal>project-team</goal>
<goal>mailing-list</goal>
<goal>cim</goal>
<goal>issue-tracking</goal>
<goal>license</goal>
<goal>scm</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
docs/src/site/site.xml
......@@ -17,7 +17,7 @@
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Metadata and Governance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="Apache Atlas"
xmlns="http://maven.apache.org/DECORATION/1.3.0"
xsi:schemaLocation="http://maven.apache.org/DECORATION/1.3.0 http://maven.apache.org/xsd/decoration-1.3.0.xsd">
......@@ -100,21 +100,10 @@
<menu name="Documentation">
<!-- current points to latest release -->
<item name="current" href="./0.5.0-incubating/index.html"/>
<item name="current" href="./index.html"/>
<item name="0.5-incubating" href="./0.5.0-incubating/index.html"/>
</menu>
<menu name="Resources">
<item name="Overview" href="index.html"/>
<item name="Getting Started" href="./QuickStart.html"/>
<item name="Architecture" href="./Architecture.html"/>
<item name="Installation" href="./InstallationSteps.html"/>
<item name="Type System" href="./TypeSystem.html"/>
<item name="Configuration" href="./Configuration.html"/>
<item name="Security" href="./Security.html"/>
<item name="Rest API" href="./api/rest.html"/>
</menu>
<menu name="ASF">
<item name="How Apache Works" href="http://www.apache.org/foundation/how-it-works.html"/>
<item name="Foundation" href="http://www.apache.org/foundation/"/>
......
docs/src/site/twiki/Architecture.twiki
......@@ -2,5 +2,30 @@
---++ Introduction
---++ Metadata High Level Architecture - Overview
<img src="architecture.png" height="400" width="600" />
---++ Atlas High Level Architecture - Overview
<img src="images/twiki/architecture.png" height="400" width="600" />
---++ Bridges
External components like hive/sqoop/storm/falcon should model their taxonomy using typesystem and register the types with Atlas. For every entity created in this external component, the corresponding entity should be registered in Atlas as well.
This is typically done in a hook which runs in the external component and is called for every entity operation. Hook generally processes the entity asynchronously using a thread pool to avoid adding latency to the main operation.
The hook can then build the entity and register the entity using Atlas REST APIs. Howerver, any failure in APIs because of network issue etc can in result entity not registered in Atlas and hence inconsistent metadata.
Atlas exposes notification interface and can be used for reliable entity registration by hook as well. The hook can send notification message containing the list of entities to be registered. Atlas service contains hook consumer that listens to these messages and registers the entities.
Available bridges are:
* [[hive/Bridge-Hive][Hive Bridge]]
---++ Notification
Notification is used for reliable entity registration from hooks and for entity/type change notifications. Atlas, by default, provides kafka integration, but its possible to provide other implementations as well. Atlas service starts embedded kafka server by default.
Atlas also provides NotificationHookConsumer that runs in Atlas Service and listens to messages from hook and registers the entities in Atlas.
<img src="images/twiki/notification.png" height="100" width="150" />
addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki → docs/src/site/twiki/Bridge-Hive.twiki
---+ Hive Atlas Bridge
Hive metadata can be modelled in Atlas using its Type System. The default modelling is available in org.apache.atlas.hive.model.HiveDataModelGenerator. It defines the following types:
* hive_resource_type(EnumType) - [JAR, FILE, ARCHIVE]
* hive_principal_type(EnumType) - [USER, ROLE, GROUP]
* hive_function_type(EnumType) - [JAVA]
* hive_order(StructType) - [col, order]
* hive_resourceuri(StructType) - [resourceType, uri]
* hive_serde(StructType) - [name, serializationLib, parameters]
* hive_process(ClassType) - [name, startTime, endTime, userName, sourceTableNames, targetTableNames, queryText, queryPlan, queryId, queryGraph]
* hive_function(ClassType) - [functionName, dbName, className, ownerName, ownerType, createTime, functionType, resourceUris]
* hive_type(ClassType) - [name, type1, type2, fields]
* hive_partition(ClassType) - [values, dbName, tableName, createTime, lastAccessTime, sd, parameters]
* hive_storagedesc(ClassType) - [cols, location, inputFormat, outputFormat, compressed, numBuckets, serdeInfo, bucketCols, sortCols, parameters, storedAsSubDirectories]
* hive_index(ClassType) - [indexName, indexHandlerClass, dbName, createTime, lastAccessTime, origTableName, indexTableName, sd, parameters, deferredRebuild]
* hive_role(ClassType) - [roleName, createTime, ownerName]
* hive_column(ClassType) - [name, type, comment]
* hive_db(ClassType) - [name, description, locationUri, parameters, ownerName, ownerType]
* hive_table(ClassType) - [name, dbName, owner, createTime, lastAccessTime, retention, sd, partitionKeys, columns, parameters, viewOriginalText, viewExpandedText, tableType, temporary]
---++ Hive Model
The default hive modelling is available in org.apache.atlas.hive.model.HiveDataModelGenerator. It defines the following types:
<verbatim>
hive_object_type(EnumType) - values [GLOBAL, DATABASE, TABLE, PARTITION, COLUMN]
hive_resource_type(EnumType) - values [JAR, FILE, ARCHIVE]
hive_principal_type(EnumType) - values [USER, ROLE, GROUP]
hive_db(ClassType) - super types [Referenceable] - attributes [name, clusterName, description, locationUri, parameters, ownerName, ownerType]
hive_order(StructType) - attributes [col, order]
hive_resourceuri(StructType) - attributes [resourceType, uri]
hive_serde(StructType) - attributes [name, serializationLib, parameters]
hive_type(ClassType) - super types [] - attributes [name, type1, type2, fields]
hive_storagedesc(ClassType) - super types [Referenceable] - attributes [cols, location, inputFormat, outputFormat, compressed, numBuckets, serdeInfo, bucketCols, sortCols, parameters, storedAsSubDirectories]
hive_role(ClassType) - super types [] - attributes [roleName, createTime, ownerName]
hive_column(ClassType) - super types [Referenceable] - attributes [name, type, comment]
hive_table(ClassType) - super types [DataSet] - attributes [tableName, db, owner, createTime, lastAccessTime, comment, retention, sd, partitionKeys, columns, parameters, viewOriginalText, viewExpandedText, tableType, temporary]
hive_partition(ClassType) - super types [Referenceable] - attributes [values, table, createTime, lastAccessTime, sd, columns, parameters]
hive_process(ClassType) - super types [Process] - attributes [startTime, endTime, userName, operationType, queryText, queryPlan, queryId, queryGraph]
</verbatim>
The entities are created and de-duped using unique qualified name. They provide namespace and can be used for querying as well:
hive_db - attribute qualifiedName - clustername.dbname
hive_table - attribute name - clustername.dbname.tablename
hive_partition - attribute qualifiedName - clustername.dbname.tablename.partitionvalues
hive_process - attribute qualifiedName - queryText
---++ Importing Hive Metadata
org.apache.atlas.hive.bridge.HiveMetaStoreBridge imports the hive metadata into Atlas using the typesystem defined in org.apache.atlas.hive.model.HiveDataModelGenerator. import-hive.sh command can be used to facilitate this.
Set-up the following configs in hive-site.xml of your hive set-up and set environment variable HIVE_CONFIG to the
hive conf directory:
org.apache.atlas.hive.bridge.HiveMetaStoreBridge imports the hive metadata into Atlas using the model defined in org.apache.atlas.hive.model.HiveDataModelGenerator. import-hive.sh command can be used to facilitate this.
Set-up the following configs in hive-site.xml of your hive set-up and set environment variable HIVE_CONFIG to the hive conf directory:
* Atlas endpoint - Add the following property with the Atlas endpoint for your set-up
<verbatim>
<property>
......@@ -34,20 +41,20 @@ hive conf directory:
</property>
</verbatim>
Usage: <dgi package>/bin/import-hive.sh. The logs are in <dgi package>/logs/import-hive.log
Usage: <atlas package>/bin/hive/import-hive.sh. The logs are in <atlas package>/logs/import-hive.log
---++ Hive Hook
Hive supports listeners on hive command execution using hive hooks. This is used to add/update/remove entities in Atlas using the model defined in org.apache.atlas.hive.model.HiveDataModelGenerator.
The hook submits the request to a thread pool executor to avoid blocking the command execution. Follow the these instructions in your hive set-up to add hive hook for Atlas:
* Add org.apache.atlas.hive.hook.HiveHook as post execution hook in hive-site.xml
The hook submits the request to a thread pool executor to avoid blocking the command execution. The thread submits the entities as message to the notification server and atlas server reads these messages and registers the entities.
Follow these instructions in your hive set-up to add hive hook for Atlas:
* Set-up atlas hook and atlas endpoint in hive-site.xml:
<verbatim>
<property>
<name>hive.exec.post.hooks</name>
<value>org.apache.atlas.hive.hook.HiveHook</value>
</property>
</verbatim>
* Add the following properties in hive-ste.xml with the Atlas endpoint for your set-up
<verbatim>
<property>
<name>atlas.rest.address</name>
......@@ -58,15 +65,19 @@ The hook submits the request to a thread pool executor to avoid blocking the com
<value>primary</value>
</property>
</verbatim>
* Add 'export HIVE_AUX_JARS_PATH=<dgi package>/hook/hive' in hive-env.sh
* Add 'export HIVE_AUX_JARS_PATH=<atlas package>/hook/hive' in hive-env.sh
* Copy <atlas package>/conf/application.properties to hive conf directory <hive package>/conf
The following properties in hive-site.xml control the thread pool details:
The following properties in hive-site.xml control the thread pool and notification details:
* atlas.hook.hive.synchronous - boolean, true to run the hook synchronously. default false
* atlas.hook.hive.numRetries - number of retries for notification failure. default 3
* atlas.hook.hive.minThreads - core number of threads. default 5
* atlas.hook.hive.maxThreads - maximum number of threads. default 5
* atlas.hook.hive.keepAliveTime - keep alive time in msecs. default 10
* atlas.hook.hive.synchronous - boolean, true to run the hook synchronously. default false
Refer [[Configuration][Configuration]] for notification related configurations
---++ Limitations
* Since database name, table name and column names are case insensitive in hive, the corresponding names in entities are lowercase. So, any search APIs should use lowercase while querying on the entity names
* Only the following hive operations are captured by hive hook currently - create database, create table, create view, CTAS, load, import, export, query, alter table rename and alter view rename
* Only the following hive operations are captured by hive hook currently - create database, create table, create view, CTAS, load, import, export, query, alter table rename and alter view rename
\ No newline at end of file
docs/src/site/twiki/Configuration.twiki
......@@ -49,6 +49,26 @@ atlas.lineage.hive.process.outputs.name=outputs
atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
</verbatim>
---+++ Notification Configs
Refer http://kafka.apache.org/documentation.html#configuration for kafka configuration. All kafka configs should be prefixed with 'atlas.kafka.'
<verbatim>
atlas.notification.embedded=true
atlas.kafka.data=${sys:atlas.home}/data/kafka
atlas.kafka.zookeeper.connect=localhost:9026
atlas.kafka.bootstrap.servers=localhost:9027
atlas.kafka.zookeeper.session.timeout.ms=400
atlas.kafka.zookeeper.sync.time.ms=20
atlas.kafka.auto.commit.interval.ms=1000
</verbatim>
---+++ Client Configs
<verbatim>
atlas.client.readTimeoutMSecs=60000
atlas.client.connectTimeoutMSecs=60000
</verbatim>
---+++ Security Properties
---++++ SSL config
......
docs/src/site/twiki/InstallationSteps.twiki
......@@ -14,7 +14,7 @@ Once the build successfully completes, artifacts can be packaged for deployment.
<verbatim>
mvn clean package -DskipTests -DskipCheck=true
mvn clean package -Pdist
</verbatim>
......
docs/src/site/twiki/QuickStart.twiki
......@@ -8,12 +8,12 @@ instance graph below.
---+++ Example Type Definitions
<img src="guide-class-diagram.png"/>
<img src="images/twiki/guide-class-diagram.png"/>
---+++ Example Instance Graph
<img src="guide-instance-graph.png"/>
<img src="images/twiki/guide-instance-graph.png"/>
---++ Running the example
......
docs/src/site/twiki/TypeSystem.twiki
......@@ -5,10 +5,10 @@
---++ Overview
---+++ Data Types Overview
<img src="data-types.png" height="400" width="600" />
<img src="images/twiki/data-types.png" height="400" width="600" />
---+++ Types Instances Overview
<img src="types-instances.png" height="400" width="600" />
<img src="images/twiki/types-instances.png" height="400" width="600" />
---++ Details
......
docs/src/site/twiki/index.twiki
......@@ -47,6 +47,8 @@ allows integration with the whole enterprise data ecosystem.
* [[Search][Search]]
* [[security][security]]
* [[Configuration][Configuration]]
* Bridges
* [[Bridge-Hive][Hive Bridge]]
---++ API Documentation
......@@ -56,4 +58,4 @@ allows integration with the whole enterprise data ecosystem.
#LicenseInfo
---+ Licensing Information
Metadata (Atlas) is distributed under [[http://www.apache.org/licenses/LICENSE-2.0][Apache License 2.0]].
Atlas is distributed under [[http://www.apache.org/licenses/LICENSE-2.0][Apache License 2.0]].
notification/pom.xml
......@@ -27,7 +27,7 @@
<version>0.6-incubating-SNAPSHOT</version>
</parent>
<artifactId>atlas-notification</artifactId>
<description>Apache Atlas Client</description>
<description>Apache Atlas Notification</description>
<name>Apache Atlas Notification</name>
<packaging>jar</packaging>
......@@ -39,6 +39,11 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
......@@ -53,8 +58,8 @@
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-multibindings</artifactId>
</dependency>
<dependency>
......
notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java
......@@ -30,6 +30,7 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.notification.NotificationConsumer;
import org.apache.atlas.notification.NotificationException;
import org.apache.atlas.notification.NotificationInterface;
import org.apache.atlas.service.Service;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationConverter;
import org.apache.kafka.clients.consumer.ConsumerConfig;
......@@ -49,6 +50,9 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -57,13 +61,11 @@ import java.util.Properties;
import java.util.concurrent.Future;
@Singleton
public class KafkaNotification extends NotificationInterface {
public class KafkaNotification extends NotificationInterface implements Service {
public static final Logger LOG = LoggerFactory.getLogger(KafkaNotification.class);
public static final String PROPERTY_PREFIX = NotificationInterface.PROPERTY_PREFIX + ".kafka";
public static final String PROPERTY_PREFIX = "atlas.kafka";
private static final int ATLAS_ZK_PORT = 9026;
private static final int ATLAS_KAFKA_PORT = 9027;
private static final String ATLAS_KAFKA_DATA = "data";
public static final String ATLAS_HOOK_TOPIC = "ATLAS_HOOK";
......@@ -92,9 +94,8 @@ public class KafkaNotification extends NotificationInterface {
}
}
@Override
public void initialize(Configuration applicationProperties) throws AtlasException {
super.initialize(applicationProperties);
public KafkaNotification(Configuration applicationProperties) throws AtlasException {
super(applicationProperties);
Configuration subsetConfiguration =
ApplicationProperties.getSubsetConfiguration(applicationProperties, PROPERTY_PREFIX);
properties = ConfigurationConverter.getProperties(subsetConfiguration);
......@@ -118,42 +119,42 @@ public class KafkaNotification extends NotificationInterface {
properties.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY, "roundrobin");
properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "smallest");
if (isEmbedded()) {
properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + ATLAS_KAFKA_PORT);
properties.setProperty("zookeeper.connect", "localhost:" + ATLAS_ZK_PORT);
}
//todo new APIs not available yet
// consumer = new KafkaConsumer(properties);
// consumer.subscribe(ATLAS_HOOK_TOPIC);
}
@Override
protected void _startService() throws IOException {
startZk();
startKafka();
private URL getURL(String url) throws MalformedURLException {
try {
return new URL(url);
} catch(MalformedURLException e) {
return new URL("http://" + url);
}
}
private String startZk() throws IOException {
//todo read zk endpoint from config
this.factory = NIOServerCnxnFactory.createFactory(new InetSocketAddress("0.0.0.0", ATLAS_ZK_PORT), 1024);
private String startZk() throws IOException, InterruptedException, URISyntaxException {
String zkValue = properties.getProperty("zookeeper.connect");
LOG.debug("Starting zookeeper at {}", zkValue);
URL zkAddress = getURL(zkValue);
this.factory = NIOServerCnxnFactory.createFactory(
new InetSocketAddress(zkAddress.getHost(), zkAddress.getPort()), 1024);
File snapshotDir = constructDir("zk/txn");
File logDir = constructDir("zk/snap");
try {
factory.startup(new ZooKeeperServer(snapshotDir, logDir, 500));
} catch (InterruptedException e) {
throw new IOException(e);
}
factory.startup(new ZooKeeperServer(snapshotDir, logDir, 500));
return factory.getLocalAddress().getAddress().toString();
}
private void startKafka() {
private void startKafka() throws IOException, URISyntaxException {
String kafkaValue = properties.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);
LOG.debug("Starting kafka at {}", kafkaValue);
URL kafkaAddress = getURL(kafkaValue);
Properties brokerConfig = properties;
brokerConfig.setProperty("broker.id", "1");
//todo read kafka endpoint from config
brokerConfig.setProperty("host.name", "0.0.0.0");
brokerConfig.setProperty("port", String.valueOf(ATLAS_KAFKA_PORT));
brokerConfig.setProperty("host.name", kafkaAddress.getHost());
brokerConfig.setProperty("port", String.valueOf(kafkaAddress.getPort()));
brokerConfig.setProperty("log.dirs", constructDir("kafka").getAbsolutePath());
brokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1));
......@@ -162,6 +163,29 @@ public class KafkaNotification extends NotificationInterface {
LOG.debug("Embedded kafka server started with broker config {}", brokerConfig);
}
@Override
public void start() throws AtlasException {
if (isEmbedded()) {
try {
startZk();
startKafka();
} catch(Exception e) {
throw new AtlasException("Failed to start embedded kafka", e);
}
}
}
@Override
public void stop() {
if (kafkaServer != null) {
kafkaServer.shutdown();
}
if (factory != null) {
factory.shutdown();
}
}
private static class SystemTime implements Time {
@Override
public long milliseconds() {
......@@ -192,29 +216,6 @@ public class KafkaNotification extends NotificationInterface {
}
@Override
public void _shutdown() {
if (producer != null) {
producer.close();
}
if (consumer != null) {
consumer.close();
}
for (ConsumerConnector consumerConnector : consumerConnectors) {
consumerConnector.shutdown();
}
if (kafkaServer != null) {
kafkaServer.shutdown();
}
if (factory != null) {
factory.shutdown();
}
}
@Override
public List<NotificationConsumer> createConsumers(NotificationType type, int numConsumers) {
String topic = topicMap.get(type);
......@@ -261,6 +262,24 @@ public class KafkaNotification extends NotificationInterface {
}
}
@Override
public void close() {
if (producer != null) {
producer.close();
producer = null;
}
if (consumer != null) {
consumer.close();
consumer = null;
}
for (ConsumerConnector consumerConnector : consumerConnectors) {
consumerConnector.shutdown();
}
consumerConnectors.clear();
}
//New API, not used now
private List<String> receive(long timeout) throws NotificationException {
Map<String, ConsumerRecords> recordsMap = consumer.poll(timeout);
......
notification/src/main/java/org/apache/atlas/kafka/KafkaNotificationProvider.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.kafka;
import com.google.inject.Provider;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.commons.configuration.Configuration;
public class KafkaNotificationProvider implements Provider<KafkaNotification> {
@Override
@Provides
@Singleton
public KafkaNotification get() {
try {
Configuration applicationProperties = ApplicationProperties.get();
KafkaNotification instance = new KafkaNotification(applicationProperties);
return instance;
} catch(AtlasException e) {
throw new RuntimeException(e);
}
}
}
notification/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
......@@ -18,37 +18,43 @@
package org.apache.atlas.notification;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.service.Service;
import org.apache.commons.configuration.Configuration;
import org.codehaus.jettison.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
public class NotificationHookConsumer {
/**
* Consumer of notifications from hooks e.g., hive hook etc
*/
@Singleton
public class NotificationHookConsumer implements Service {
private static final Logger LOG = LoggerFactory.getLogger(NotificationHookConsumer.class);
public static final String CONSUMER_THREADS_PROPERTY = "atlas.notification.hook.numthreads";
public static final String ATLAS_ENDPOINT_PROPERTY = "atlas.rest.address";
@Inject
private static NotificationInterface notificationInterface;
private static ExecutorService executors;
private static AtlasClient atlasClient;
private NotificationInterface notificationInterface;
private ExecutorService executors;
private AtlasClient atlasClient;
public static void start() throws AtlasException {
@Override
public void start() throws AtlasException {
Configuration applicationProperties = ApplicationProperties.get();
notificationInterface.initialize(applicationProperties);
String atlasEndpoint = applicationProperties.getString(ATLAS_ENDPOINT_PROPERTY, "http://localhost:21000");
atlasClient = new AtlasClient(atlasEndpoint);
int numThreads = applicationProperties.getInt(CONSUMER_THREADS_PROPERTY, 2);
int numThreads = applicationProperties.getInt(CONSUMER_THREADS_PROPERTY, 1);
List<NotificationConsumer> consumers =
notificationInterface.createConsumers(NotificationInterface.NotificationType.HOOK, numThreads);
executors = Executors.newFixedThreadPool(consumers.size());
......@@ -58,12 +64,20 @@ public class NotificationHookConsumer {
}
}
public static void stop() {
notificationInterface.shutdown();
executors.shutdown();
@Override
public void stop() {
//Allow for completion of outstanding work
notificationInterface.close();
try {
if (executors != null && !executors.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
LOG.error("Timed out waiting for consumer threads to shut down, exiting uncleanly");
}
} catch (InterruptedException e) {
LOG.error("Failure in shutting down consumers");
}
}
static class HookConsumer implements Runnable {
class HookConsumer implements Runnable {
private final NotificationConsumer consumer;
public HookConsumer(NotificationConsumer consumerInterface) {
......@@ -74,12 +88,13 @@ public class NotificationHookConsumer {
public void run() {
while(consumer.hasNext()) {
String entityJson = consumer.next();
LOG.debug("Processing message {}", entityJson);
LOG.info("Processing message {}", entityJson);
try {
atlasClient.createEntity(entityJson);
} catch (AtlasServiceException e) {
JSONArray guids = atlasClient.createEntity(new JSONArray(entityJson));
LOG.info("Create entities with guid {}", guids);
} catch (Exception e) {
//todo handle failures
LOG.warn("Error handling message {}", entityJson);
LOG.warn("Error handling message {}", entityJson, e);
}
}
}
......
notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java
......@@ -20,7 +20,6 @@ package org.apache.atlas.notification;
import org.apache.atlas.AtlasException;
import org.apache.commons.configuration.Configuration;
import java.io.IOException;
import java.util.List;
public abstract class NotificationInterface {
......@@ -33,26 +32,11 @@ public abstract class NotificationInterface {
HOOK, ENTITIES, TYPES
}
/**
* Initialise
* @param applicationProperties
* @throws AtlasException
*/
public void initialize(Configuration applicationProperties) throws AtlasException {
public NotificationInterface(Configuration applicationProperties) throws AtlasException {
this.embedded = applicationProperties.getBoolean(PROPERTY_EMBEDDED, false);
}
/**
* Start embedded notification service on atlast server
* @throws IOException
*/
public final void startService() throws IOException {
if (embedded) {
_startService();
}
}
/**
* Is the notification service embedded in atlas server
* @return
*/
......@@ -60,18 +44,9 @@ public abstract class NotificationInterface {
return embedded;
}
protected abstract void _startService() throws IOException;
/**
* Shutdown - close all the connections
*/
public final void shutdown() {
_shutdown();
}
protected abstract void _shutdown();
public abstract List<NotificationConsumer> createConsumers(NotificationType type, int numConsumers);
public abstract void send(NotificationType type, String... messages) throws NotificationException;
public abstract void close();
}
notification/src/main/java/org/apache/atlas/notification/NotificationModule.java
......@@ -18,11 +18,21 @@
package org.apache.atlas.notification;
import com.google.inject.AbstractModule;
import com.google.inject.Singleton;
import com.google.inject.multibindings.Multibinder;
import org.apache.atlas.kafka.KafkaNotification;
import org.apache.atlas.kafka.KafkaNotificationProvider;
import org.apache.atlas.service.Service;
public class NotificationModule extends AbstractModule {
@Override
protected void configure() {
bind(NotificationInterface.class).to(KafkaNotification.class).asEagerSingleton();
bind(NotificationInterface.class).to(KafkaNotification.class).in(Singleton.class);
bind(KafkaNotification.class).toProvider(KafkaNotificationProvider.class).in(Singleton.class);
Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder(), Service.class);
serviceBinder.addBinding().to(KafkaNotification.class);
serviceBinder.addBinding().to(NotificationHookConsumer.class);
}
}
notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java
......@@ -18,12 +18,10 @@
package org.apache.atlas.kafka;
import com.google.inject.Inject;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.notification.NotificationConsumer;
import org.apache.atlas.notification.NotificationInterface;
import org.apache.atlas.notification.NotificationModule;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
......@@ -35,18 +33,15 @@ import org.testng.annotations.Test;
public class KafkaNotificationTest {
@Inject
private NotificationInterface kafka;
private KafkaNotification kafka;
@BeforeClass
public void setUp() throws Exception {
Configuration conf = ApplicationProperties.get();
conf.setProperty(KafkaNotification.PROPERTY_PREFIX + ".data", "target/data/kafka" + random());
kafka.initialize(conf);
kafka.startService();
kafka.start();
}
@Test
public void testSendMessage() throws AtlasException {
public void testSendReceiveMessage() throws AtlasException {
String msg1 = "message" + random();
String msg2 = "message" + random();
kafka.send(NotificationInterface.NotificationType.HOOK, msg1, msg2);
......@@ -63,6 +58,6 @@ public class KafkaNotificationTest {
@AfterClass
public void teardown() throws Exception {
kafka.shutdown();
kafka.stop();
}
}
pom.xml
......@@ -392,6 +392,7 @@
</profile>
</profiles>
<modules>
<module>common</module>
<module>typesystem</module>
<module>notification</module>
<module>client</module>
......@@ -972,6 +973,12 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<version>${project.version}</version>
<type>war</type>
......@@ -1298,7 +1305,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
<version>3.2</version>
<version>3.3</version>
</plugin>
<plugin>
......@@ -1348,13 +1355,6 @@
<goal>testCompile</goal>
</goals>
</execution>
<execution>
<id>attach-scaladocs</id>
<phase>verify</phase>
<goals>
<goal>doc-jar</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
......@@ -1428,6 +1428,9 @@
-Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml
-Djava.net.preferIPv4Stack=true
</argLine>
<excludes>
<exclude>**/*Base*</exclude>
</excludes>
</configuration>
<dependencies>
<dependency>
......
release-log.txt
......@@ -4,6 +4,7 @@ Apache Atlas Release Notes
--trunk - unreleased
INCOMPATIBLE CHANGES:
ATLAS-58 Make hive hook reliable (shwethags)
ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
......
repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
......@@ -32,6 +32,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
@Inject
GraphProvider<TitanGraph> graphProvider;
@Override
public Object invoke(MethodInvocation invocation) throws Throwable {
if (titanGraph == null) {
titanGraph = graphProvider.get();
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -18,11 +18,11 @@
package org.apache.atlas;
import com.google.inject.Singleton;
import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.HiveLineageService;
......@@ -38,6 +38,7 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.types.TypeSystem;
/**
* Guice module for Repository module.
......@@ -55,6 +56,8 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the MetadataRepositoryService interface to an implementation
bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton();
bind(TypeSystem.class).in(Singleton.class);
// bind the ITypeStore interface to an implementation
bind(ITypeStore.class).to(GraphBackedTypeStore.class).asEagerSingleton();
......
repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
......@@ -21,6 +21,8 @@ package org.apache.atlas.listener;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import java.util.Collection;
/**
* Entity (a Typed instance) change notification listener.
*/
......@@ -29,10 +31,10 @@ public interface EntityChangeListener {
/**
* This is upon adding a new typed instance to the repository.
*
* @param typedInstance a typed instance
* @param typedInstances a typed instance
* @throws AtlasException
*/
void onEntityAdded(ITypedReferenceableInstance typedInstance) throws AtlasException;
void onEntityAdded(Collection<ITypedReferenceableInstance> typedInstances) throws AtlasException;
/**
* This is upon adding a new trait to a typed instance.
......
repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance;
public class EntityExistsException extends AtlasException {
public EntityExistsException(IReferenceableInstance typedInstance, Exception e) {
super("Model violation for type "+ typedInstance.getTypeName(), e);
}
public EntityExistsException(IReferenceableInstance typedInstance) {
super("Model violation for type "+ typedInstance.getTypeName());
}
}
repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
......@@ -19,7 +19,6 @@
package org.apache.atlas.repository;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.types.AttributeInfo;
......@@ -76,11 +75,12 @@ public interface MetadataRepository {
/**
* Creates an entity definition (instance) corresponding to a given type.
*
* @param entity entity (typed instance)
* @param entities entity (typed instance)
* @return a globally unique identifier
* @throws RepositoryException
* @throws EntityExistsException
*/
String createEntity(IReferenceableInstance entity) throws RepositoryException;
String[] createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, EntityExistsException;
/**
* Fetch the complete definition of an entity given its GUID.
......@@ -158,4 +158,13 @@ public interface MetadataRepository {
* @param value property value
*/
void updateEntity(String guid, String property, String value) throws RepositoryException;
/**
* Returns the entity for the given type and qualified name
* @param entityType
* @param attribute
* @param value
* @return entity instance
*/
ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value) throws AtlasException;
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -46,34 +46,35 @@ public final class GraphHelper {
}
public static Vertex createVertexWithIdentity(Graph graph, ITypedReferenceableInstance typedInstance,
Set<String> superTypeNames) {
final Vertex vertexWithIdentity =
createVertexWithoutIdentity(graph, typedInstance.getTypeName(), typedInstance.getId(), superTypeNames);
Set<String> superTypeNames) {
final Vertex vertexWithIdentity = createVertexWithoutIdentity(graph, typedInstance.getTypeName(),
typedInstance.getId(), superTypeNames);
// add identity
final String guid = UUID.randomUUID().toString();
vertexWithIdentity.setProperty(Constants.GUID_PROPERTY_KEY, guid);
setProperty(vertexWithIdentity, Constants.GUID_PROPERTY_KEY, guid);
return vertexWithIdentity;
}
public static Vertex createVertexWithoutIdentity(Graph graph, String typeName, Id typedInstanceId,
Set<String> superTypeNames) {
Set<String> superTypeNames) {
LOG.debug("Creating vertex for type {} id {}", typeName, typedInstanceId._getId());
final Vertex vertexWithoutIdentity = graph.addVertex(null);
// add type information
vertexWithoutIdentity.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
// add super types
for (String superTypeName : superTypeNames) {
((TitanVertex) vertexWithoutIdentity).addProperty(Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
addProperty(vertexWithoutIdentity, Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
// add version information
vertexWithoutIdentity.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
setProperty(vertexWithoutIdentity, Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
// add timestamp information
vertexWithoutIdentity.setProperty(Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
return vertexWithoutIdentity;
}
......@@ -84,12 +85,12 @@ public final class GraphHelper {
return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
}
public static Vertex findVertexByGUID(TitanGraph titanGraph, String value) {
LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
public static Vertex findVertex(TitanGraph titanGraph, String propertyKey, Object value) {
LOG.debug("Finding vertex for {}={}", propertyKey, value);
GraphQuery query = titanGraph.query().has(Constants.GUID_PROPERTY_KEY, value);
GraphQuery query = titanGraph.query().has(propertyKey, value);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
// returning one since entityType, qualifiedName should be unique
return results.hasNext() ? results.next() : null;
}
......@@ -107,6 +108,16 @@ public final class GraphHelper {
+ edge.getVertex(Direction.IN) + "]";
}
public static void setProperty(Vertex vertex, String propertyName, Object value) {
LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
vertex.setProperty(propertyName, value);
}
public static void addProperty(Vertex vertex, String propertyName, Object value) {
LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
((TitanVertex)vertex).addProperty(propertyName, value);
}
/*
public static void dumpToLog(final Graph graph) {
LOG.debug("*******************Graph Dump****************************");
......
repository/src/main/java/org/apache/atlas/services/MetadataService.java
......@@ -66,7 +66,7 @@ public interface MetadataService {
* @param entityDefinition definition
* @return guid
*/
String createEntity(String entityDefinition) throws AtlasException;
String createEntities(String entityDefinition) throws AtlasException;
/**
* Return the definition for the given guid.
......@@ -77,6 +77,16 @@ public interface MetadataService {
String getEntityDefinition(String guid) throws AtlasException;
/**
* Return the definition given type and attribute. The attribute has to be unique attribute for the type
* @param entityType - type name
* @param attribute - attribute name
* @param value - attribute value
* @return
* @throws AtlasException
*/
String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException;
/**
* Return the list of entity names for the given type in the repository.
*
* @param entityType type
......
repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
......@@ -19,11 +19,8 @@
package org.apache.atlas;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -85,7 +85,7 @@ public class GraphBackedDiscoveryServiceTest {
ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
repositoryService.createEntity(hrDept2);
repositoryService.createEntities(hrDept2);
}
private void setupSampleData() throws ScriptException {
......@@ -303,6 +303,6 @@ public class GraphBackedDiscoveryServiceTest {
ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, "D");
ITypedReferenceableInstance typedInstance = deptType.convert(instance, Multiplicity.REQUIRED);
repositoryService.createEntity(typedInstance);
repositoryService.createEntities(typedInstance);
}
}
\ No newline at end of file
repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
......@@ -54,9 +54,6 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
/**
......@@ -534,7 +531,10 @@ public class HiveLineageServiceTest {
String entityJSON = InstanceSerialization.toJson(referenceable, true);
System.out.println("Submitting new entity= " + entityJSON);
String guid = metadataService.createEntity(entityJSON);
JSONArray jsonArray = new JSONArray();
jsonArray.put(entityJSON);
String response = metadataService.createEntities(jsonArray.toString());
String guid = new JSONArray(response).getString(0);
System.out.println("created instance for type " + typeName + ", guid: " + guid);
// return the reference to created instance with guid
......
repository/src/test/java/org/apache/atlas/repository/BaseTest.java
......@@ -18,172 +18,13 @@
package org.apache.atlas.repository;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.memory.MemRepository;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.junit.Before;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
public abstract class BaseTest {
public static final String STRUCT_TYPE_1 = "t1";
public static final String STRUCT_TYPE_2 = "t2";
public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
public static final long TEST_DATE_IN_LONG = 1418265358440L;
protected IRepository repo;
public static Struct createStruct() throws AtlasException {
StructType structType = (StructType) TypeSystem.getInstance().getDataType(StructType.class, STRUCT_TYPE_1);
Struct s = new Struct(structType.getName());
s.set("a", 1);
s.set("b", true);
s.set("c", (byte) 1);
s.set("d", (short) 2);
s.set("e", 1);
s.set("f", 1);
s.set("g", 1L);
s.set("h", 1.0f);
s.set("i", 1.0);
s.set("j", BigInteger.valueOf(1L));
s.set("k", new BigDecimal(1));
s.set("l", new Date(1418265358440L));
s.set("m", Lists.<Integer>asList(Integer.valueOf(1), new Integer[]{Integer.valueOf(1)}));
s.set("n", Lists.<BigDecimal>asList(BigDecimal.valueOf(1.1), new BigDecimal[]{BigDecimal.valueOf(1.1)}));
Map<String, Double> hm = Maps.<String, Double>newHashMap();
hm.put("a", 1.0);
hm.put("b", 2.0);
s.set("o", hm);
return s;
}
protected final TypeSystem getTypeSystem() {
return TypeSystem.getInstance();
}
protected final IRepository getRepository() {
return repo;
}
@Before
public void setup() throws Exception {
TypeSystem ts = TypeSystem.getInstance();
ts.reset();
repo = new MemRepository(ts);
StructType structType =
ts.defineStructType(STRUCT_TYPE_1, true, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)), TypesUtil
.createOptionalAttrDef("o",
ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)));
StructType recursiveStructType =
ts.defineStructType(STRUCT_TYPE_2, true, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2));
}
protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition... tDefs) throws AtlasException {
return getTypeSystem().defineTraitTypes(tDefs);
}
/*
* Class Hierarchy is:
* Department(name : String, employees : Array[Person])
* Person(name : String, department : Department, manager : Manager)
* Manager(subordinates : Array[Person]) extends Person
*
* Persons can have SecurityClearance(level : Int) clearance.
*/
protected void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException {
HierarchicalTypeDefinition<ClassType> deptTypeDef = TypesUtil
.createClassTypeDef("Department", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"),
Multiplicity.COLLECTION, true, "department"));
HierarchicalTypeDefinition<ClassType> personTypeDef = TypesUtil
.createClassTypeDef("Person", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"));
HierarchicalTypeDefinition<ClassType> managerTypeDef = TypesUtil
.createClassTypeDef("Manager", ImmutableList.<String>of("Person"),
new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
Multiplicity.COLLECTION, false, "manager"));
HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = TypesUtil
.createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE));
ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef, managerTypeDef));
ImmutableList<HierarchicalType> types = ImmutableList
.of(ts.getDataType(HierarchicalType.class, "SecurityClearance"),
ts.getDataType(ClassType.class, "Department"), ts.getDataType(ClassType.class, "Person"),
ts.getDataType(ClassType.class, "Manager"));
repo.defineTypes(types);
}
protected Referenceable createDeptEg1(TypeSystem ts) throws AtlasException {
Referenceable hrDept = new Referenceable("Department");
Referenceable john = new Referenceable("Person");
Referenceable jane = new Referenceable("Manager", "SecurityClearance");
hrDept.set("name", "hr");
john.set("name", "John");
john.set("department", hrDept);
jane.set("name", "Jane");
jane.set("department", hrDept);
john.set("manager", jane);
hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
jane.set("subordinates", ImmutableList.<Referenceable>of(john));
jane.getTrait("SecurityClearance").set("level", 1);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
return hrDept;
}
}
repository/src/test/java/org/apache/atlas/repository/memory/ClassTest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.memory;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.BaseTest;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class ClassTest extends BaseTest {
@Before
public void setup() throws Exception {
super.setup();
}
@Test
public void test1() throws AtlasException {
TypeSystem ts = getTypeSystem();
defineDeptEmployeeTypes(ts);
Referenceable hrDept = createDeptEg1(ts);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
Assert.assertEquals(hrDept2.toString(), "{\n" +
"\tid : (type: Department, id: <unassigned>)\n" +
"\tname : \thr\n" +
"\temployees : \t[{\n" +
"\tid : (type: Person, id: <unassigned>)\n" +
"\tname : \tJohn\n" +
"\tdepartment : (type: Department, id: <unassigned>)\n" +
"\tmanager : (type: Manager, id: <unassigned>)\n" +
"}, {\n" +
"\tid : (type: Manager, id: <unassigned>)\n" +
"\tsubordinates : \t[{\n" +
"\tid : (type: Person, id: <unassigned>)\n" +
"\tname : \tJohn\n" +
"\tdepartment : (type: Department, id: <unassigned>)\n" +
"\tmanager : (type: Manager, id: <unassigned>)\n" +
"}]\n" +
"\tname : \tJane\n" +
"\tdepartment : (type: Department, id: <unassigned>)\n" +
"\tmanager : <null>\n" +
"\n" +
"\tSecurityClearance : \t{\n" +
"\t\tlevel : \t\t1\n" +
"\t}}]\n" +
"}");
}
}
repository/src/test/java/org/apache/atlas/repository/memory/InstanceE2ETest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.memory;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.BaseTest;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization$;
import org.apache.atlas.typesystem.json.Serialization$;
import org.apache.atlas.typesystem.json.TypesSerialization$;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
public class InstanceE2ETest extends BaseTest {
protected List<HierarchicalTypeDefinition> createHiveTypes(TypeSystem typeSystem) throws AtlasException {
ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = TypesUtil
.createClassTypeDef("hive_database", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
typeDefinitions.add(databaseTypeDefinition);
HierarchicalTypeDefinition<ClassType> tableTypeDefinition = TypesUtil
.createClassTypeDef("hive_table", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition("hive_database", "hive_database", Multiplicity.REQUIRED, false,
"hive_database"));
typeDefinitions.add(tableTypeDefinition);
HierarchicalTypeDefinition<TraitType> fetlTypeDefinition = TypesUtil
.createTraitTypeDef("hive_fetl", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE));
typeDefinitions.add(fetlTypeDefinition);
typeSystem.defineTypes(ImmutableList.<StructTypeDefinition>of(), ImmutableList.of(fetlTypeDefinition),
ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
return typeDefinitions;
}
protected Referenceable createHiveTableReferenceable() throws AtlasException {
Referenceable databaseInstance = new Referenceable("hive_database");
databaseInstance.set("name", "hive_database");
databaseInstance.set("description", "foo database");
Referenceable tableInstance = new Referenceable("hive_table", "hive_fetl");
tableInstance.set("name", "t1");
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
tableInstance.set("hive_database", databaseInstance);
Struct traitInstance = (Struct) tableInstance.getTrait("hive_fetl");
traitInstance.set("level", 1);
tableInstance.set("hive_fetl", traitInstance);
return tableInstance;
}
protected ITypedReferenceableInstance createHiveTableInstance(TypeSystem typeSystem) throws AtlasException {
ClassType tableType = typeSystem.getDataType(ClassType.class, "hive_table");
return tableType.convert(createHiveTableReferenceable(), Multiplicity.REQUIRED);
}
@Test
public void testType() throws AtlasException {
TypeSystem ts = getTypeSystem();
createHiveTypes(ts);
String jsonStr = TypesSerialization$.MODULE$.toJson(ts, ImmutableList.of("hive_database", "hive_table"));
System.out.println(jsonStr);
TypesDef typesDef1 = TypesSerialization$.MODULE$.fromJson(jsonStr);
System.out.println(typesDef1);
ts.reset();
ts.defineTypes(typesDef1);
jsonStr = TypesSerialization$.MODULE$.toJson(ts, ImmutableList.of("hive_database", "hive_table"));
System.out.println(jsonStr);
}
@Test
public void testInstance() throws AtlasException {
TypeSystem ts = getTypeSystem();
createHiveTypes(ts);
ITypedReferenceableInstance i = createHiveTableInstance(getTypeSystem());
String jsonStr = Serialization$.MODULE$.toJson(i);
System.out.println(jsonStr);
i = Serialization$.MODULE$.fromJson(jsonStr);
System.out.println(i);
}
@Test
public void testInstanceSerialization() throws AtlasException {
TypeSystem ts = getTypeSystem();
createHiveTypes(ts);
Referenceable r = createHiveTableReferenceable();
String jsonStr = InstanceSerialization$.MODULE$.toJson(r, true);
Referenceable r1 = InstanceSerialization$.MODULE$.fromJsonReferenceable(jsonStr, true);
ClassType tableType = ts.getDataType(ClassType.class, "hive_table");
ITypedReferenceableInstance i = tableType.convert(r1, Multiplicity.REQUIRED);
jsonStr = Serialization$.MODULE$.toJson(i);
System.out.println(jsonStr);
i = Serialization$.MODULE$.fromJson(jsonStr);
System.out.println(i);
}
}
repository/src/test/java/org/apache/atlas/repository/memory/StorageTest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.memory;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.BaseTest;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class StorageTest extends BaseTest {
@Before
public void setup() throws Exception {
super.setup();
}
@Test
public void test1() throws AtlasException {
TypeSystem ts = getTypeSystem();
defineDeptEmployeeTypes(ts);
Referenceable hrDept = createDeptEg1(ts);
ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
ITypedReferenceableInstance hrDept3 = getRepository().get(hrDept2.getId());
Assert.assertEquals(hrDept3.toString(), "{\n" +
"\tid : (type: Department, id: 1)\n" +
"\tname : \thr\n" +
"\temployees : \t[{\n" +
"\tid : (type: Person, id: 2)\n" +
"\tname : \tJohn\n" +
"\tdepartment : (type: Department, id: 1)\n" +
"\tmanager : (type: Manager, id: 3)\n" +
"}, {\n" +
"\tid : (type: Manager, id: 3)\n" +
"\tsubordinates : \t[(type: Person, id: 2)]\n" +
"\tname : \tJane\n" +
"\tdepartment : (type: Department, id: 1)\n" +
"\tmanager : <null>\n" +
"\n" +
"\tSecurityClearance : \t{\n" +
"\t\tlevel : \t\t1\n" +
"\t}}]\n" +
"}");
}
@Test
public void testGetPerson() throws AtlasException {
TypeSystem ts = getTypeSystem();
defineDeptEmployeeTypes(ts);
Referenceable hrDept = createDeptEg1(ts);
ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
Id e1Id = new Id(2, 0, "Person");
ITypedReferenceableInstance e1 = getRepository().get(e1Id);
Assert.assertEquals(e1.toString(), "{\n" +
"\tid : (type: Person, id: 2)\n" +
"\tname : \tJohn\n" +
"\tdepartment : (type: Department, id: 1)\n" +
"\tmanager : (type: Manager, id: 3)\n" +
"}");
}
@Test
public void testInvalidTypeName() throws AtlasException {
TypeSystem ts = getTypeSystem();
defineDeptEmployeeTypes(ts);
Referenceable hrDept = createDeptEg1(ts);
ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
Id e1Id = new Id(3, 0, "Person");
try {
ITypedReferenceableInstance e1 = getRepository().get(e1Id);
} catch (RepositoryException re) {
RepositoryException me = (RepositoryException) re.getCause();
Assert.assertEquals(me.getMessage(), "Invalid Id (unknown) : (type: Person, id: 3)");
}
}
@Test
public void testGetManager() throws AtlasException {
TypeSystem ts = getTypeSystem();
defineDeptEmployeeTypes(ts);
Referenceable hrDept = createDeptEg1(ts);
ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
Id m1Id = new Id(3, 0, "Manager");
ITypedReferenceableInstance m1 = getRepository().get(m1Id);
Assert.assertEquals(m1.toString(), "{\n" +
"\tid : (type: Manager, id: 3)\n" +
"\tsubordinates : \t[(type: Person, id: 2)]\n" +
"\tname : \tJane\n" +
"\tdepartment : (type: Department, id: 1)\n" +
"\tmanager : <null>\n" +
"\n" +
"\tSecurityClearance : \t{\n" +
"\t\tlevel : \t\t1\n" +
"\t}}");
}
}
\ No newline at end of file
repository/src/test/java/org/apache/atlas/repository/memory/StructTest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.memory;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.BaseTest;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.json.InstanceSerialization$;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class StructTest extends BaseTest {
StructType structType;
StructType recursiveStructType;
@Before
public void setup() throws Exception {
super.setup();
structType = (StructType) getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_1);
recursiveStructType = (StructType) getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_2);
}
@Test
public void test1() throws AtlasException {
Struct s = createStruct();
ITypedStruct ts = structType.convert(s, Multiplicity.REQUIRED);
Assert.assertEquals(ts.toString(), "{\n" +
"\ta : \t1\n" +
"\tb : \ttrue\n" +
"\tc : \t1\n" +
"\td : \t2\n" +
"\te : \t1\n" +
"\tf : \t1\n" +
"\tg : \t1\n" +
"\th : \t1.0\n" +
"\ti : \t1.0\n" +
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{a=1.0, b=2.0}\n" +
"}");
}
@Test
public void testRecursive() throws AtlasException {
Struct s1 = new Struct(recursiveStructType.getName());
s1.set("a", 1);
Struct s2 = new Struct(recursiveStructType.getName());
s2.set("a", 1);
s2.set("s", s1);
ITypedStruct ts = recursiveStructType.convert(s2, Multiplicity.REQUIRED);
Assert.assertEquals(ts.toString(), "{\n" +
"\ta : \t1\n" +
"\ts : \t{\n" +
"\t\ta : \t\t1\n" +
"\t\ts : <null>\n" +
"\n" +
"\t}\n" +
"}");
}
@Test
public void testSerialization() throws AtlasException {
Struct s = createStruct();
String jsonStr = InstanceSerialization$.MODULE$.toJson(s, true);
Struct s1 = InstanceSerialization$.MODULE$.fromJsonStruct(jsonStr, true);
ITypedStruct ts = structType.convert(s1, Multiplicity.REQUIRED);
Assert.assertEquals(ts.toString(), "{\n" +
"\ta : \t1\n" +
"\tb : \ttrue\n" +
"\tc : \t1\n" +
"\td : \t2\n" +
"\te : \t1\n" +
"\tf : \t1\n" +
"\tg : \t1\n" +
"\th : \t1.0\n" +
"\ti : \t1.0\n" +
"\tj : \t1\n" +
"\tk : \t1\n" +
"\tl : \t" + TEST_DATE + "\n" +
"\tm : \t[1, 1]\n" +
"\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
".100000000000000088817841970012523233890533447265625]\n" +
"\to : \t{a=1.0, b=2.0}\n" +
"}");
}
}
repository/src/test/java/org/apache/atlas/repository/memory/TraitTest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.memory;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.BaseTest;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TraitType;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
public class TraitTest extends BaseTest {
@Before
public void setup() throws Exception {
super.setup();
}
/*
* Type Hierarchy is:
* A(a,b,c,d)
* B(b) extends A
* C(c) extends A
* D(d) extends B,C
*
* - There are a total of 11 fields in an instance of D
* - an attribute that is hidden by a SubType can referenced by prefixing it with the
* complete Path.
* For e.g. the 'b' attribute in A (that is a superType for B) is hidden the 'b' attribute
* in B.
* So it is availabel by the name 'A.B.D.b'
*
* - Another way to set attributes is to cast. Casting a 'D' instance of 'B' makes the 'A.B.D
* .b' attribute
* available as 'A.B.b'. Casting one more time to an 'A' makes the 'A.B.b' attribute
* available as 'b'.
*/
@Test
public void test1() throws AtlasException {
HierarchicalTypeDefinition A = createTraitTypeDef("A", null, createRequiredAttrDef("a", DataTypes.INT_TYPE),
createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE), createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
HierarchicalTypeDefinition C =
createTraitTypeDef("C", ImmutableList.<String>of("A"), createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
defineTraits(A, B, C, D);
TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
Struct s1 = new Struct("D");
s1.set("d", 1);
s1.set("c", 1);
s1.set("b", true);
s1.set("a", 1);
s1.set("A.B.D.b", true);
s1.set("A.B.D.c", 2);
s1.set("A.B.D.d", 2);
s1.set("A.C.D.a", 3);
s1.set("A.C.D.b", false);
s1.set("A.C.D.c", 3);
s1.set("A.C.D.d", 3);
ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
Assert.assertEquals(ts.toString(), "{\n" +
"\td : \t1\n" +
"\tb : \ttrue\n" +
"\tc : \t1\n" +
"\ta : \t1\n" +
"\tA.B.D.b : \ttrue\n" +
"\tA.B.D.c : \t2\n" +
"\tA.B.D.d : \t2\n" +
"\tA.C.D.a : \t3\n" +
"\tA.C.D.b : \tfalse\n" +
"\tA.C.D.c : \t3\n" +
"\tA.C.D.d : \t3\n" +
"}");
/*
* cast to B and set the 'b' attribute on A.
*/
TraitType BType = (TraitType) getTypeSystem().getDataType(TraitType.class, "B");
IStruct s2 = DType.castAs(ts, "B");
s2.set("A.B.b", false);
Assert.assertEquals(ts.toString(), "{\n" +
"\td : \t1\n" +
"\tb : \ttrue\n" +
"\tc : \t1\n" +
"\ta : \t1\n" +
"\tA.B.D.b : \tfalse\n" +
"\tA.B.D.c : \t2\n" +
"\tA.B.D.d : \t2\n" +
"\tA.C.D.a : \t3\n" +
"\tA.C.D.b : \tfalse\n" +
"\tA.C.D.c : \t3\n" +
"\tA.C.D.d : \t3\n" +
"}");
/*
* cast again to A and set the 'b' attribute on A.
*/
TraitType AType = (TraitType) getTypeSystem().getDataType(TraitType.class, "A");
IStruct s3 = BType.castAs(s2, "A");
s3.set("b", true);
Assert.assertEquals(ts.toString(), "{\n" +
"\td : \t1\n" +
"\tb : \ttrue\n" +
"\tc : \t1\n" +
"\ta : \t1\n" +
"\tA.B.D.b : \ttrue\n" +
"\tA.B.D.c : \t2\n" +
"\tA.B.D.d : \t2\n" +
"\tA.C.D.a : \t3\n" +
"\tA.C.D.b : \tfalse\n" +
"\tA.C.D.c : \t3\n" +
"\tA.C.D.d : \t3\n" +
"}");
}
@Test
public void testRandomOrder() throws AtlasException {
HierarchicalTypeDefinition A = createTraitTypeDef("A", null, createRequiredAttrDef("a", DataTypes.INT_TYPE),
createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE), createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
HierarchicalTypeDefinition C =
createTraitTypeDef("C", ImmutableList.<String>of("A"), createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
defineTraits(B, D, A, C);
TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
Struct s1 = new Struct("D");
s1.set("d", 1);
s1.set("c", 1);
s1.set("b", true);
s1.set("a", 1);
s1.set("A.B.D.b", true);
s1.set("A.B.D.c", 2);
s1.set("A.B.D.d", 2);
s1.set("A.C.D.a", 3);
s1.set("A.C.D.b", false);
s1.set("A.C.D.c", 3);
s1.set("A.C.D.d", 3);
ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
Assert.assertEquals(ts.toString(), "{\n" +
"\td : \t1\n" +
"\tb : \ttrue\n" +
"\tc : \t1\n" +
"\ta : \t1\n" +
"\tA.B.D.b : \ttrue\n" +
"\tA.B.D.c : \t2\n" +
"\tA.B.D.d : \t2\n" +
"\tA.C.D.a : \t3\n" +
"\tA.C.D.b : \tfalse\n" +
"\tA.C.D.c : \t3\n" +
"\tA.C.D.d : \t3\n" +
"}");
}
}
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.service;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@Guice(modules = RepositoryMetadataModule.class)
public class DefaultMetadataServiceTest {
@Inject
private MetadataService metadataService;
@Inject
private GraphProvider<TitanGraph> graphProvider;
@BeforeClass
public void setUp() throws Exception {
TypesDef typesDef = TestUtils.defineHiveTypes();
try {
metadataService.getTypeDefinition(TestUtils.TABLE_TYPE);
} catch (TypeNotFoundException e) {
metadataService.createType(TypesSerialization.toJson(typesDef));
}
}
@AfterClass
public void shudown() {
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
}
private String createInstance(Referenceable entity) throws Exception {
String entityjson = InstanceSerialization.toJson(entity, true);
JSONArray entitiesJson = new JSONArray();
entitiesJson.put(entityjson);
String response = metadataService.createEntities(entitiesJson.toString());
return new JSONArray(response).getString(0);
}
private Referenceable createDBEntity() {
Referenceable entity = new Referenceable(TestUtils.DATABASE_TYPE);
String dbName = RandomStringUtils.randomAlphanumeric(10);
entity.set("name", dbName);
entity.set("description", "us db");
return entity;
}
@Test
public void testCreateEntityWithUniqueAttribute() throws Exception {
//name is the unique attribute
Referenceable entity = createDBEntity();
String id = createInstance(entity);
//using the same name should succeed, but not create another entity
String newId = createInstance(entity);
Assert.assertEquals(newId, id);
//Same entity, but different qualified name should succeed
entity.set("name", TestUtils.randomString());
newId = createInstance(entity);
Assert.assertNotEquals(newId, id);
}
@Test
public void testCreateEntityWithUniqueAttributeWithReference() throws Exception {
Referenceable db = createDBEntity();
String dbId = createInstance(db);
Referenceable table = new Referenceable(TestUtils.TABLE_TYPE);
table.set("name", TestUtils.randomString());
table.set("description", "random table");
table.set("type", "type");
table.set("tableType", "MANAGED");
table.set("database", db);
createInstance(table);
//table create should re-use the db instance created earlier
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
Referenceable actualDb = (Referenceable) tableDefinition.get("database");
Assert.assertEquals(actualDb.getId().id, dbId);
}
@Test
public void testGetEntityByUniqueAttribute() throws Exception {
Referenceable entity = createDBEntity();
createInstance(entity);
//get entity by valid qualified name
String entityJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "name",
(String) entity.get("name"));
Assert.assertNotNull(entityJson);
Referenceable referenceable = InstanceSerialization.fromJsonReferenceable(entityJson, true);
Assert.assertEquals(referenceable.get("name"), entity.get("name"));
//get entity by invalid qualified name
try {
metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "name", "random");
Assert.fail("Expected EntityNotFoundException");
} catch (EntityNotFoundException e) {
//expected
}
//get entity by non-unique attribute
try {
metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "description",
(String) entity.get("description"));
Assert.fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
//expected
}
}
}
repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala
......@@ -19,14 +19,13 @@
package org.apache.atlas.query
import java.io.File
import java.util.{Date, UUID}
import java.util.concurrent.atomic.AtomicInteger
import java.util.{Date, UUID}
import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
import com.thinkaurelius.titan.core.TitanGraph
import com.typesafe.config.ConfigFactory
import org.apache.atlas.repository.BaseTest
import org.apache.atlas.typesystem.types.TypeSystem
import org.apache.atlas.TestUtils
import org.apache.commons.io.FileUtils
import scala.collection.mutable.ArrayBuffer
......@@ -140,7 +139,7 @@ object HiveTitanSample {
created: Option[Date] = None) {
val createdDate : Date = created match {
case Some(x) => x
case None => new Date(BaseTest.TEST_DATE_IN_LONG)
case None => new Date(TestUtils.TEST_DATE_IN_LONG)
}
val sd = StorageDescriptor(inputFormat, outputFormat)
val colDefs = columns map { c =>
......
repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
......@@ -127,7 +127,8 @@ object QueryTestsUtils extends GraphUtils {
def jdbcTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "JdbcAccess", null,
Array[AttributeDefinition]())
TypeSystem.getInstance().defineTypes(ImmutableList.of[StructTypeDefinition],
TypeSystem.getInstance().defineTypes(ImmutableList.of[EnumTypeDefinition],
ImmutableList.of[StructTypeDefinition],
ImmutableList.of[HierarchicalTypeDefinition[TraitType]](dimTraitDef, piiTraitDef,
metricTraitDef, etlTraitDef, jdbcTraitDef),
ImmutableList.of[HierarchicalTypeDefinition[ClassType]](dbClsDef, storageDescClsDef, columnClsDef, tblClsDef,
......
typesystem/pom.xml
......@@ -126,6 +126,11 @@
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
......
typesystem/src/main/java/org/apache/atlas/TypeExistsException.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
public class TypeExistsException extends AtlasException {
public TypeExistsException(String message) {
super(message);
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
......@@ -115,7 +115,11 @@ public class StructInstance implements ITypedStruct {
clsType.validateId((Id) val);
cVal = val;
} else {
cVal = i.dataType().convert(val, i.multiplicity);
try {
cVal = i.dataType().convert(val, i.multiplicity);
} catch(ValueConversionException.NullConversionException e) {
throw new ValueConversionException.NullConversionException("For field '" + attrName + "'", e);
}
}
if (cVal == null) {
nullFlags[nullPos] = true;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
......@@ -25,6 +25,7 @@ public final class AttributeDefinition {
public final String name;
public final String dataTypeName;
public final Multiplicity multiplicity;
//A composite is the one whose lifecycle is dependent on the enclosing type and is not just a reference
public final boolean isComposite;
public final boolean isUnique;
public final boolean isIndexable;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
......@@ -27,6 +27,7 @@ import java.util.Map;
public class AttributeInfo {
public final String name;
public final Multiplicity multiplicity;
//A composite is the one whose lifecycle is dependent on the enclosing type and is not just a reference
public final boolean isComposite;
public final boolean isUnique;
public final boolean isIndexable;
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
......@@ -55,8 +55,8 @@ public class ObjectGraphWalker {
throws AtlasException {
this.typeSystem = typeSystem;
this.nodeProcessor = nodeProcessor;
queue = new LinkedList<IReferenceableInstance>();
processedIds = new HashSet<Id>();
queue = new LinkedList<>();
processedIds = new HashSet<>();
if (start != null) {
visitReferenceableInstance(start);
}
......@@ -194,7 +194,7 @@ public class ObjectGraphWalker {
}
}
public static interface NodeProcessor {
public interface NodeProcessor {
void processNode(Node nd) throws AtlasException;
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -22,6 +22,7 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TypeExistsException;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef;
......@@ -169,9 +170,9 @@ public class TypeSystem {
public StructType defineStructType(String name, boolean errorIfExists, AttributeDefinition... attrDefs)
throws AtlasException {
StructTypeDefinition structDef = new StructTypeDefinition(name, attrDefs);
defineTypes(ImmutableList.of(structDef), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(structDef),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
return getDataType(StructType.class, structDef.typeName);
}
......@@ -196,56 +197,50 @@ public class TypeSystem {
}
public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef) throws AtlasException {
defineTypes(ImmutableList.<StructTypeDefinition>of(), ImmutableList.of(traitDef),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(traitDef), ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
return getDataType(TraitType.class, traitDef.typeName);
}
public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef) throws AtlasException {
defineTypes(ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(classDef));
defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(classDef));
return getDataType(ClassType.class, classDef.typeName);
}
public Map<String, IDataType> defineTraitTypes(HierarchicalTypeDefinition<TraitType>... traitDefs)
throws AtlasException {
TransientTypeSystem transientTypes =
new TransientTypeSystem(ImmutableList.<StructTypeDefinition>of(), ImmutableList.copyOf(traitDefs),
new TransientTypeSystem(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(), ImmutableList.copyOf(traitDefs),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
return transientTypes.defineTypes();
}
public Map<String, IDataType> defineClassTypes(HierarchicalTypeDefinition<ClassType>... classDefs)
throws AtlasException {
TransientTypeSystem transientTypes = new TransientTypeSystem(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.copyOf(classDefs));
TransientTypeSystem transientTypes = new TransientTypeSystem(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.copyOf(classDefs));
return transientTypes.defineTypes();
}
public Map<String, IDataType> defineTypes(TypesDef typesDef) throws AtlasException {
Map<String, IDataType> typesAdded = new HashMap<>();
for (EnumTypeDefinition enumDef : typesDef.enumTypesAsJavaList()) {
typesAdded.put(enumDef.name, defineEnumType(enumDef));
}
ImmutableList<EnumTypeDefinition> enumDefs = ImmutableList.copyOf(typesDef.enumTypesAsJavaList());
ImmutableList<StructTypeDefinition> structDefs = ImmutableList.copyOf(typesDef.structTypesAsJavaList());
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs =
ImmutableList.copyOf(typesDef.traitTypesAsJavaList());
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs =
ImmutableList.copyOf(typesDef.classTypesAsJavaList());
typesAdded.putAll(defineTypes(structDefs, traitDefs, classDefs));
return typesAdded;
return defineTypes(enumDefs, structDefs, traitDefs, classDefs);
}
public Map<String, IDataType> defineTypes(ImmutableList<StructTypeDefinition> structDefs,
public Map<String, IDataType> defineTypes(ImmutableList<EnumTypeDefinition> enumDefs,
ImmutableList<StructTypeDefinition> structDefs,
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) throws AtlasException {
TransientTypeSystem transientTypes = new TransientTypeSystem(structDefs, traitDefs, classDefs);
TransientTypeSystem transientTypes = new TransientTypeSystem(enumDefs, structDefs, traitDefs, classDefs);
Map<String, IDataType> definedTypes = transientTypes.defineTypes();
// LOG.debug("Defined new types " + Arrays.toString(definedTypes.keySet().toArray(new
// String[definedTypes.size()])));
......@@ -307,6 +302,7 @@ public class TypeSystem {
final ImmutableList<StructTypeDefinition> structDefs;
final ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs;
final ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs;
private final ImmutableList<EnumTypeDefinition> enumDefs;
Map<String, StructTypeDefinition> structNameToDefMap = new HashMap<>();
Map<String, HierarchicalTypeDefinition<TraitType>> traitNameToDefMap = new HashMap<>();
Map<String, HierarchicalTypeDefinition<ClassType>> classNameToDefMap = new HashMap<>();
......@@ -318,10 +314,10 @@ public class TypeSystem {
List<DataTypes.MapType> recursiveMapTypes;
TransientTypeSystem(ImmutableList<StructTypeDefinition> structDefs,
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) {
TransientTypeSystem(ImmutableList<EnumTypeDefinition> enumDefs, ImmutableList<StructTypeDefinition> structDefs,
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) {
this.enumDefs = enumDefs;
this.structDefs = structDefs;
this.traitDefs = traitDefs;
this.classDefs = classDefs;
......@@ -345,10 +341,22 @@ public class TypeSystem {
* - setup shallow Type instances to facilitate recursive type graphs
*/
private void step1() throws AtlasException {
for (EnumTypeDefinition eDef : enumDefs) {
assert eDef.name != null;
if (types.containsKey(eDef.name)) {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
}
EnumType eT = new EnumType(this, eDef.name, eDef.enumValues);
TypeSystem.this.types.put(eDef.name, eT);
typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ENUM, eDef.name);
transientTypes.add(eDef.name);
}
for (StructTypeDefinition sDef : structDefs) {
assert sDef.typeName != null;
if (dataType(sDef.typeName) != null) {
throw new AtlasException(String.format("Cannot redefine type %s", sDef.typeName));
throw new TypeExistsException(String.format("Cannot redefine type %s", sDef.typeName));
}
TypeSystem.this.types
.put(sDef.typeName, new StructType(this, sDef.typeName, sDef.attributeDefinitions.length));
......@@ -359,7 +367,7 @@ public class TypeSystem {
for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) {
assert traitDef.typeName != null;
if (types.containsKey(traitDef.typeName)) {
throw new AtlasException(String.format("Cannot redefine type %s", traitDef.typeName));
throw new TypeExistsException(String.format("Cannot redefine type %s", traitDef.typeName));
}
TypeSystem.this.types.put(traitDef.typeName, new TraitType(this, traitDef.typeName, traitDef.superTypes,
......@@ -371,7 +379,7 @@ public class TypeSystem {
for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) {
assert classDef.typeName != null;
if (types.containsKey(classDef.typeName)) {
throw new AtlasException(String.format("Cannot redefine type %s", classDef.typeName));
throw new TypeExistsException(String.format("Cannot redefine type %s", classDef.typeName));
}
TypeSystem.this.types.put(classDef.typeName, new ClassType(this, classDef.typeName, classDef.superTypes,
......@@ -588,7 +596,8 @@ public class TypeSystem {
}
@Override
public Map<String, IDataType> defineTypes(ImmutableList<StructTypeDefinition> structDefs,
public Map<String, IDataType> defineTypes(ImmutableList<EnumTypeDefinition> enumDefs,
ImmutableList<StructTypeDefinition> structDefs,
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) throws AtlasException {
throw new AtlasException("Internal Error: define type called on TrasientTypeSystem");
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java
......@@ -43,10 +43,17 @@ public class ValueConversionException extends AtlasException {
super(msg);
}
protected ValueConversionException(String msg, Exception e) {
super(msg, e);
}
public static class NullConversionException extends ValueConversionException {
public NullConversionException(Multiplicity m) {
super(String.format("Null value not allowed for multiplicty %s", m));
}
public NullConversionException(String msg, Exception e) {
super(msg, e);
}
}
}
typesystem/src/main/resources/application.properties
......@@ -41,6 +41,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
#atlas.lineage.hive.table.type.name=DataSet
......@@ -53,8 +54,13 @@ atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, co
######### Notification Configs #########
atlas.notification.embedded=true
atlas.notification.implementation=org.apache.atlas.kafka.KafkaNotification
atlas.notification.kafka.data=target/data/kafka
atlas.kafka.zookeeper.connect=localhost:9026
atlas.kafka.bootstrap.servers=localhost:9027
atlas.kafka.data=target/data/kafka
atlas.kafka.zookeeper.session.timeout.ms=400
atlas.kafka.zookeeper.sync.time.ms=20
atlas.kafka.auto.commit.interval.ms=100
######### Security Properties #########
......
typesystem/src/main/resources/atlas-log4j.xml 0 → 100755
<?xml version="1.0" encoding="UTF-8" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<param name="Target" value="System.out"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
</layout>
</appender>
<appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${atlas.log.dir}/audit.log"/>
<param name="Append" value="true"/>
<param name="Threshold" value="debug"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %x %m%n"/>
</layout>
</appender>
<logger name="org.apache.atlas" additivity="false">
<level value="debug"/>
<appender-ref ref="console"/>
</logger>
<logger name="AUDIT">
<level value="info"/>
<appender-ref ref="console"/>
</logger>
<root>
<priority value="warn"/>
<appender-ref ref="console"/>
</root>
</log4j:configuration>
typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala
......@@ -236,7 +236,7 @@ trait TypeHelpers {
@throws(classOf[AtlasException])
def defineClassType(ts: TypeSystem, classDef: HierarchicalTypeDefinition[ClassType]): ClassType = {
ts.defineTypes(ImmutableList.of[StructTypeDefinition],
ts.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.of[StructTypeDefinition],
ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
ImmutableList.of[HierarchicalTypeDefinition[ClassType]](classDef))
return ts.getDataType(classOf[ClassType], classDef.typeName)
......
client/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java → typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java
......@@ -25,7 +25,7 @@ public class ApplicationPropertiesTest {
@Test
public void testVariables() throws Exception {
Configuration properties = ApplicationProperties.get();
Configuration properties = ApplicationProperties.get(ApplicationProperties.APPLICATION_PROPERTIES);
//plain property without variables
Assert.assertEquals(properties.getString("atlas.service"), "atlas");
......@@ -44,7 +44,7 @@ public class ApplicationPropertiesTest {
@Test
//variable substitutions should work with subset configuration as well
public void testSubset() throws Exception {
Configuration configuration = ApplicationProperties.get();
Configuration configuration = ApplicationProperties.get(ApplicationProperties.APPLICATION_PROPERTIES);
Configuration subConfiguration = configuration.subset("atlas");
Assert.assertEquals(subConfiguration.getString("service"), "atlas");
......
typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java
......@@ -28,6 +28,7 @@ import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.BaseTest;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
......@@ -79,9 +80,9 @@ public class SerializationJavaTest extends BaseTest {
createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE));
ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef, managerTypeDef));
ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(securityClearanceTypeDef),
ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef));
Referenceable hrDept = new Referenceable("Department");
Referenceable john = new Referenceable("Person");
......@@ -147,8 +148,8 @@ public class SerializationJavaTest extends BaseTest {
createTraitTypeDef("SecurityClearance2", ImmutableList.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE));
ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(securityClearanceTypeDef),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
......
typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
......@@ -136,7 +136,8 @@ public abstract class BaseTest {
.createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE));
ts.defineTypes(ImmutableList.<StructTypeDefinition>of(), ImmutableList.of(securityClearanceTypeDef),
ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(securityClearanceTypeDef),
ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef));
ImmutableList.of(ts.getDataType(HierarchicalType.class, "SecurityClearance"),
......
typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java
......@@ -80,9 +80,10 @@ public class TypeSystemTest extends BaseTest {
HierarchicalTypeDefinition<TraitType> financeTrait =
TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
getTypeSystem().defineTypes(ImmutableList.<StructTypeDefinition>of(), ImmutableList
.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait, soxTrait, secTrait,
financeTrait), ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
getTypeSystem().defineTypes(ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait, soxTrait, secTrait,
financeTrait), ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
final ImmutableList<String> traitsNames = getTypeSystem().getTypeNamesByCategory(DataTypes.TypeCategory.TRAIT);
Assert.assertEquals(traitsNames.size(), 7);
......@@ -102,7 +103,6 @@ public class TypeSystemTest extends BaseTest {
String enumType = random();
EnumTypeDefinition orgLevelEnum =
new EnumTypeDefinition(enumType, new EnumValue(random(), 1), new EnumValue(random(), 2));
ts.defineEnumType(orgLevelEnum);
String structName = random();
String attrType = random();
......@@ -117,7 +117,8 @@ public class TypeSystemTest extends BaseTest {
HierarchicalTypeDefinition<TraitType> traitType = createTraitTypeDef(traitName, ImmutableList.<String>of(),
createRequiredAttrDef(attrType, DataTypes.INT_TYPE));
ts.defineTypes(ImmutableList.of(structType), ImmutableList.of(traitType), ImmutableList.of(classType));
ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(structType),
ImmutableList.of(traitType), ImmutableList.of(classType));
}
@Test
......@@ -127,7 +128,7 @@ public class TypeSystemTest extends BaseTest {
HierarchicalTypeDefinition<ClassType> c = TypesUtil.createClassTypeDef("C", ImmutableList.of("B"));
TypeSystem ts = getTypeSystem();
ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(a, b, c));
ClassType ac = ts.getDataType(ClassType.class, "a");
......
typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
......@@ -146,7 +146,7 @@ class SerializationTest extends BaseTest {
TypesUtil.createTraitTypeDef("SecurityClearance", ImmutableList.of[String],
TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE))
ts.defineTypes(ImmutableList.of[StructTypeDefinition],
ts.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.of[StructTypeDefinition],
ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef)
)
......
typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
......@@ -45,7 +45,7 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
ts.defineTypes(ImmutableList.of[StructTypeDefinition](sDef),
ts.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.of[StructTypeDefinition](sDef),
ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
ImmutableList.of[HierarchicalTypeDefinition[ClassType]]
)
......@@ -123,7 +123,7 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance", List(),
requiredAttr("level", DataTypes.INT_TYPE)
)
ts.defineTypes(ImmutableList.of[StructTypeDefinition],
ts.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.of[StructTypeDefinition],
ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef))
......@@ -136,7 +136,7 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
typesDef1.enumTypes.foreach(ts1.defineEnumType(_))
ts1.defineTypes(ImmutableList.copyOf(typesDef1.structTypes.toArray),
ts1.defineTypes(ImmutableList.of[EnumTypeDefinition], ImmutableList.copyOf(typesDef1.structTypes.toArray),
ImmutableList.copyOf(typesDef1.traitTypes.toArray),
ImmutableList.copyOf(typesDef1.classTypes.toArray)
)
......
webapp/pom.xml
......@@ -188,6 +188,11 @@
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
......@@ -202,6 +207,11 @@
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-notification</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
......@@ -413,6 +423,45 @@
</configuration>
</plugin>
<!-- Running unit tests in pre-integration-test phase after war is built -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemProperties>
<user.dir>${project.basedir}</user.dir>
<projectBaseDir>${project.basedir}/..</projectBaseDir>
</systemProperties>
<!--<skipTests>true</skipTests>-->
<forkMode>always</forkMode>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
-Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name}
-Xmx1024m -XX:MaxPermSize=512m
</argLine>
<excludes>
<exclude>**/*Base*</exclude>
</excludes>
</configuration>
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-testng</artifactId>
<version>2.18.1</version>
</dependency>
</dependencies>
<executions>
<execution>
<id>default-test</id>
<phase>pre-integration-test</phase>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-maven-plugin</artifactId>
......@@ -435,6 +484,10 @@
<useTestScope>true</useTestScope>
<systemProperties>
<systemProperty>
<name>log4j.configuration</name>
<value>atlas-log4j.xml</value>
</systemProperty>
<systemProperty>
<name>atlas.log.dir</name>
<value>${project.build.directory}/logs</value>
</systemProperty>
......@@ -453,7 +506,7 @@
</systemProperty>
</systemProperties>
<stopKey>atlas-stop</stopKey>
<stopPort>41001</stopPort>
<stopPort>21001</stopPort>
<daemon>${debug.jetty.daemon}</daemon>
<testClassesDirectory>${project.build.directory}/../../webapp/target/test-classes/</testClassesDirectory>
<useTestClasspath>true</useTestClasspath>
......
webapp/src/main/java/org/apache/atlas/Main.java
......@@ -101,6 +101,7 @@ public final class Main {
configuration.setProperty("atlas.enableTLS", String.valueOf(enableTLS));
showStartupInfo(buildConfiguration, enableTLS, appPort);
server = EmbeddedServer.newServer(appPort, appPath, enableTLS);
server.start();
}
......
webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
......@@ -38,7 +38,6 @@ import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import java.util.List;
......@@ -241,12 +240,11 @@ public class QuickStart {
String entityJSON = InstanceSerialization.toJson(referenceable, true);
System.out.println("Submitting new entity= " + entityJSON);
JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
String guid = jsonObject.getString(AtlasClient.GUID);
System.out.println("created instance for type " + typeName + ", guid: " + guid);
JSONArray guids = metadataServiceClient.createEntity(entityJSON);
System.out.println("created instance for type " + typeName + ", guid: " + guids);
// return the Id for created instance with guid
return new Id(guid, referenceable.getId().getVersion(), referenceable.getTypeName());
return new Id(guids.getString(0), referenceable.getId().getVersion(), referenceable.getTypeName());
}
Id database(String name, String description, String owner, String locationUri, String... traitNames)
......@@ -387,11 +385,9 @@ public class QuickStart {
private void search() throws Exception {
for (String dslQuery : getDSLQueries()) {
JSONObject response = metadataServiceClient.searchEntity(dslQuery);
JSONObject results = response.getJSONObject(AtlasClient.RESULTS);
if (!results.isNull("rows")) {
JSONArray rows = results.getJSONArray("rows");
System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
JSONArray results = metadataServiceClient.search(dslQuery);
if (results != null) {
System.out.println("query [" + dslQuery + "] returned [" + results.length() + "] rows");
} else {
System.out.println("query [" + dslQuery + "] failed, results:" + results.toString());
}
......
webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java
......@@ -33,7 +33,9 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.notification.NotificationModule;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.service.Services;
import org.apache.atlas.web.filters.AtlasAuthenticationFilter;
import org.apache.atlas.web.filters.AuditFilter;
import org.apache.commons.configuration.Configuration;
......@@ -64,7 +66,8 @@ public class GuiceServletConfig extends GuiceServletContextListener {
* .html
*/
if (injector == null) {
injector = Guice.createInjector(new RepositoryMetadataModule(), new JerseyServletModule() {
injector = Guice.createInjector(new RepositoryMetadataModule(), new NotificationModule(),
new JerseyServletModule() {
@Override
protected void configureServlets() {
filter("/*").through(AuditFilter.class);
......@@ -110,6 +113,14 @@ public class GuiceServletConfig extends GuiceServletContextListener {
// perform login operations
LoginProcessor loginProcessor = new LoginProcessor();
loginProcessor.login();
startServices();
}
protected void startServices() {
LOG.debug("Starting services");
Services services = injector.getInstance(Services.class);
services.start();
}
/**
......@@ -132,6 +143,15 @@ public class GuiceServletConfig extends GuiceServletContextListener {
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
final Graph graph = graphProvider.get().get();
graph.shutdown();
//stop services
stopServices();
}
}
protected void stopServices() {
LOG.debug("Stopping services");
Services services = injector.getInstance(Services.class);
services.stop();
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/atlas/web/resources/EntitiesResource.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.EntityExistsException;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Singleton;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import javax.ws.rs.core.UriInfo;
import java.net.URI;
import java.util.List;
@Path("entities")
@Singleton
public class EntitiesResource {
private static final Logger LOG = LoggerFactory.getLogger(EntitiesResource.class);
@Inject
private MetadataService metadataService;
@Context
UriInfo uriInfo;
/**
* Submits the entity definitions (instances).
* The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any
* unique attribute for the give type.
*/
@POST
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response submit(@Context HttpServletRequest request) {
try {
final String entities = Servlets.getRequestPayload(request);
LOG.debug("submitting entities {} ", AtlasClient.toString(new JSONArray(entities)));
final String guids = metadataService.createEntities(entities);
UriBuilder ub = uriInfo.getAbsolutePathBuilder();
URI locationURI = ub.path(guids).build();
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
response.put(AtlasClient.GUID, new JSONArray(guids));
response.put(AtlasClient.DEFINITION, metadataService.getEntityDefinition(new JSONArray(guids).getString(0)));
return Response.created(locationURI).entity(response).build();
} catch(EntityExistsException e) {
LOG.error("Unique constraint violation", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (ValueConversionException ve) {
LOG.error("Unable to persist entity instance due to a desrialization error ", ve);
throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist entity instance", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to persist entity instance", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Gets the list of entities for a given entity type.
*
* @param entityType name of a type which is unique
*/
@GET
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response getEntityListByType(@QueryParam("type") String entityType) {
try {
Preconditions.checkNotNull(entityType, "Entity type cannot be null");
LOG.debug("Fetching entity list for type={} ", entityType);
final List<String> entityList = metadataService.getEntityList(entityType);
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
response.put(AtlasClient.TYPENAME, entityType);
response.put(AtlasClient.RESULTS, new JSONArray(entityList));
response.put(AtlasClient.COUNT, entityList.size());
return Response.ok(response).build();
} catch (NullPointerException e) {
LOG.error("Entity type cannot be null", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to get entity list for type {}", entityType, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to get entity list for type {}", entityType, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
}
webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
......@@ -25,7 +25,6 @@ import org.apache.atlas.ParamChecker;
import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
......@@ -59,7 +58,7 @@ import java.util.List;
* An entity is an "instance" of a Type. Entities conform to the definition
* of the Type they correspond with.
*/
@Path("entities")
@Path("entity")
@Singleton
public class EntityResource {
......@@ -82,40 +81,6 @@ public class EntityResource {
this.metadataService = metadataService;
}
/**
* Submits an entity definition (instance) corresponding to a given type.
*/
@POST
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response submit(@Context HttpServletRequest request) {
try {
final String entity = Servlets.getRequestPayload(request);
LOG.debug("submitting entity {} ", entity);
final String guid = metadataService.createEntity(entity);
UriBuilder ub = uriInfo.getAbsolutePathBuilder();
URI locationURI = ub.path(guid).build();
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
response.put(AtlasClient.GUID, guid);
response.put(AtlasClient.DEFINITION, metadataService.getEntityDefinition(guid));
return Response.created(locationURI).entity(response).build();
} catch (ValueConversionException ve) {
LOG.error("Unable to persist entity instance due to a desrialization error ", ve);
throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist entity instance", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to persist entity instance", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Fetch the complete definition of an entity given its GUID.
......@@ -159,34 +124,47 @@ public class EntityResource {
}
/**
* Gets the list of entities for a given entity type.
* Fetch the complete definition of an entity given its qualified name.
*
* @param entityType name of a type which is unique
* @param entityType
* @param attribute
* @param value
*/
@GET
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response getEntityListByType(@QueryParam("type") String entityType) {
public Response getEntityDefinitionByAttribute(@QueryParam("type") String entityType,
@QueryParam("property") String attribute,
@QueryParam("value") String value) {
try {
Preconditions.checkNotNull(entityType, "Entity type cannot be null");
LOG.debug("Fetching entity definition for type={}, qualified name={}", entityType, value);
ParamChecker.notEmpty(entityType, "type cannot be null");
ParamChecker.notEmpty(attribute, "attribute name cannot be null");
ParamChecker.notEmpty(value, "attribute value cannot be null");
LOG.debug("Fetching entity list for type={} ", entityType);
final List<String> entityList = metadataService.getEntityList(entityType);
final String entityDefinition = metadataService.getEntityDefinition(entityType, attribute, value);
JSONObject response = new JSONObject();
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
response.put(AtlasClient.TYPENAME, entityType);
response.put(AtlasClient.RESULTS, new JSONArray(entityList));
response.put(AtlasClient.COUNT, entityList.size());
return Response.ok(response).build();
} catch (NullPointerException e) {
LOG.error("Entity type cannot be null", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
Response.Status status = Response.Status.NOT_FOUND;
if (entityDefinition != null) {
response.put(AtlasClient.DEFINITION, entityDefinition);
status = Response.Status.OK;
} else {
response.put(AtlasClient.ERROR, Servlets.escapeJsonString(String.format("An entity with type={%s}, " +
"qualifiedName={%s} does not exist", entityType, value)));
}
return Response.status(status).entity(response).build();
} catch (EntityNotFoundException e) {
LOG.error("An entity with type={} and qualifiedName={} does not exist", entityType, value, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to get entity list for type {}", entityType, e);
LOG.error("Bad type={}, qualifiedName={}", entityType, value, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to get entity list for type {}", entityType, e);
LOG.error("Unable to get instance definition for type={}, qualifiedName={}", entityType, value, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
......
webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
......@@ -88,28 +88,16 @@ public class MetadataDiscoveryResource {
final String jsonResultStr = discoveryService.searchByDSL(query);
response = new DSLJSONResponseBuilder().results(jsonResultStr).query(query).build();
return Response.ok(response).build();
} catch (IllegalArgumentException e) {
LOG.error("Unable to get entity list for empty query", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable throwable) {
LOG.error("Unable to get entity list for query {} using dsl", query, throwable);
try { //fall back to full-text
final String jsonResultStr = discoveryService.searchByFullText(query);
response = new FullTextJSonResponseBuilder().results(jsonResultStr).query(query).build();
} catch (DiscoveryException | IllegalArgumentException e) {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
return searchUsingFullText(query);
}
return Response.ok(response).build();
}
/**
......@@ -267,7 +255,8 @@ public class MetadataDiscoveryResource {
count(rowsJsonArr.length());
queryType(QUERY_TYPE_DSL);
JSONObject response = super.build();
response.put(AtlasClient.RESULTS, dslResults);
response.put(AtlasClient.RESULTS, rowsJsonArr);
response.put(AtlasClient.DATATYPE, dslResults.get(AtlasClient.DATATYPE));
return response;
}
......
webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
......@@ -21,6 +21,7 @@ package org.apache.atlas.web.resources;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TypeExistsException;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.web.util.Servlets;
......@@ -96,6 +97,9 @@ public class TypesResource {
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
response.put(AtlasClient.TYPES, typesResponse);
return Response.status(ClientResponse.Status.CREATED).entity(response).build();
} catch (TypeExistsException e) {
LOG.error("Type already exists", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist types", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
......
webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
......@@ -20,8 +20,6 @@ package org.apache.atlas.web.service;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
......
webapp/src/main/resources/atlas-log4j.xml
......@@ -50,7 +50,6 @@
<appender-ref ref="FILE"/>
</logger>
<logger name="AUDIT">
<level value="info"/>
<appender-ref ref="AUDIT"/>
......
webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.notification;
import com.google.inject.Inject;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.web.resources.BaseResourceIT;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@Guice(modules = NotificationModule.class)
public class NotificationHookConsumerIT extends BaseResourceIT{
@Inject
private NotificationInterface kafka;
private String dbName;
@BeforeClass
public void setUp() throws Exception {
super.setUp();
createTypeDefinitions();
}
@AfterClass
public void teardown() throws Exception {
kafka.close();
}
private void sendHookMessage(Referenceable entity) throws NotificationException {
String entityJson = InstanceSerialization.toJson(entity, true);
JSONArray jsonArray = new JSONArray();
jsonArray.put(entityJson);
kafka.send(NotificationInterface.NotificationType.HOOK, jsonArray.toString());
}
@Test
public void testConsumeHookMessage() throws Exception {
Referenceable entity = new Referenceable(DATABASE_TYPE);
dbName = "db" + randomString();
entity.set("name", dbName);
entity.set("description", randomString());
sendHookMessage(entity);
waitFor(1000, new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results =
serviceClient.searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, dbName));
return results.length() == 1;
}
});
}
@Test (dependsOnMethods = "testConsumeHookMessage")
public void testEnityDeduping() throws Exception {
// Referenceable db = serviceClient.getEntity(DATABASE_TYPE, "name", dbName);
Referenceable db = new Referenceable(DATABASE_TYPE);
db.set("name", dbName);
db.set("description", randomString());
Referenceable table = new Referenceable(HIVE_TABLE_TYPE);
final String tableName = randomString();
table.set("name", tableName);
table.set("db", db);
sendHookMessage(table);
waitFor(1000, new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results =
serviceClient.searchByDSL(String.format("%s where name='%s'", HIVE_TABLE_TYPE, tableName));
return results.length() == 1;
}
});
JSONArray results =
serviceClient.searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, dbName));
Assert.assertEquals(results.length(), 1);
}
}
webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java
......@@ -21,8 +21,10 @@ import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -31,6 +33,7 @@ import javax.servlet.ServletContextEvent;
public class TestGuiceServletConfig extends GuiceServletConfig {
private static final Logger LOG = LoggerFactory.getLogger(TestGuiceServletConfig.class);
private boolean servicesEnabled;
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
......@@ -40,6 +43,8 @@ public class TestGuiceServletConfig extends GuiceServletConfig {
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
super.contextDestroyed(servletContextEvent);
if(injector != null) {
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
......@@ -47,11 +52,30 @@ public class TestGuiceServletConfig extends GuiceServletConfig {
LOG.info("Clearing graph store");
try {
graph.shutdown();
TitanCleanup.clear(graph);
} catch (Exception e) {
LOG.warn("Clearing graph store failed ", e);
}
}
}
@Override
protected void startServices() {
try {
Configuration conf = ApplicationProperties.get();
servicesEnabled = conf.getBoolean("atlas.services.enabled", true);
if (servicesEnabled) {
super.startServices();
}
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
@Override
protected void stopServices() {
if (servicesEnabled) {
super.stopServices();
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment