Skip to content

A
atlas
Commit 90a3a9e7 by Shwetha GS
Options

ATLAS-182 Add data model for Storm topology elements (svenkat,yhemanth via shwethags)

parent 4e1cc762
Showing with 204 additions and 16 deletions
addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataTypes.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.storm.model;
/**
* Storm Data Types for model and hook.
*/
public enum StormDataTypes {
// Topology Classes
STORM_TOPOLOGY, // represents the topology containing the DAG
STORM_NODE, // base abstraction for producer and processor
STORM_SPOUT, // data producer node having only outputs
STORM_BOLT, // data processing node having both inputs and outputs
// Data Sets
KAFKA_TOPIC, // kafka data set
JMS_TOPIC, // jms data set
HBASE_TABLE, // hbase table data set
HDFS_DATA_SET, // HDFS data set
;
public String getName() {
return name().toLowerCase();
}
}
addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.storm.model
import org.apache.atlas.typesystem.TypesDef
import org.apache.atlas.typesystem.builders.TypesBuilder
import org.apache.atlas.typesystem.json.TypesSerialization
/**
* This represents the data model for a storm topology.
*/
object StormDataModel extends App {
var typesDef : TypesDef = null
val typesBuilder = new TypesBuilder
import typesBuilder._
typesDef = types {
/**
* Model is represented as:
* Topology is a Process Super Type inheriting inputs/outputs
* Input DataSet(s) => Topology => Output DataSet(s)
* Also, Topology contains the Graph of Nodes
* Topology => Node(s) -> Spouts/Bolts
*/
_class(StormDataTypes.STORM_TOPOLOGY.getName, List("Process")) {
"id" ~ (string, required, indexed, unique)
"description" ~ (string, optional, indexed)
"owner" ~ (string, required, indexed)
"startTime" ~ long
"endTime" ~ long
"conf" ~ (map(string, string), optional)
"clusterName" ~ (string, optional, indexed)
// Nodes in the Graph
"nodes" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
}
// Base class for DataProducer aka Spouts and
// DataProcessor aka Bolts, also links from Topology
_class(StormDataTypes.STORM_NODE.getName) {
"name" ~ (string, required, indexed)
"description" ~ (string, optional, indexed)
// fully qualified driver java class name
"driverClass" ~ (string, required, indexed)
// spout or bolt configuration NVPs
"conf" ~ (map(string, string), optional)
}
// Data Producer and hence only outputs
_class(StormDataTypes.STORM_SPOUT.getName, List(StormDataTypes.STORM_NODE.getName)) {
// "outputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
"outputs" ~ (array(string), collection)
}
// Data Processor and hence both inputs and outputs (inherited from Spout)
_class(StormDataTypes.STORM_BOLT.getName, List(StormDataTypes.STORM_NODE.getName)) {
// "inputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
"inputs" ~ (array(string), collection)
"outputs" ~ (array(string), collection, optional)
}
// Kafka Data Set
_class(StormDataTypes.KAFKA_TOPIC.getName, List("DataSet")) {
"topic" ~ (string, required, unique, indexed)
"uri" ~ (string, required)
"owner" ~ (string, required, indexed)
}
// JMS Data Set
_class(StormDataTypes.JMS_TOPIC.getName, List("DataSet")) {
"topic" ~ (string, required, unique, indexed)
"uri" ~ (string, required)
"owner" ~ (string, required, indexed)
}
// HBase Data Set
_class(StormDataTypes.HBASE_TABLE.getName, List("DataSet")) {
"tableName" ~ (string, required, unique, indexed)
"uri" ~ (string, required)
"owner" ~ (string, required, indexed)
}
// HDFS Data Set
// todo: replace this with a generic data model for HDFS data sets
// todo: should only be used in light of storm
_class(StormDataTypes.HDFS_DATA_SET.getName, List("DataSet")) {
// fully qualified path to file or dir
"pathURI" ~ (string, required, unique, indexed)
"owner" ~ (string, required, indexed)
}
_trait("DataProcessor") {
}
_trait("DataProducer") {
}
// Hive table data set already exists in atlas.
}
// add the types to atlas
val typesAsJSON = TypesSerialization.toJson(typesDef)
println("Storm Data Model as JSON: ")
println(typesAsJSON)
}
pom.xml
......@@ -355,14 +355,16 @@
<fastutil.version>6.5.16</fastutil.version>
<guice.version>4.0</guice.version>
<!-- Needed for hooks -->
<aopalliance.version>1.0</aopalliance.version>
<commons-conf.version>1.10</commons-conf.version>
<commons-logging.version>1.1.3</commons-logging.version>
<javax-inject.version>1</javax-inject.version>
<jettison.version>1.3.7</jettison.version>
<paranamer.version>2.3</paranamer.version>
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
<SnapshotsId>apache.snapshots.repo</SnapshotsId>
<SnapshotsName>Apache Snapshot Repository</SnapshotsName>
<SnapshotsUrl>https://repository.apache.org/content/groups/snapshots</SnapshotsUrl>
<StagingId>apache-staging</StagingId>
<StagingName>Apache Release Distribution Repository</StagingName>
<StagingUrl>https://repository.apache.org/content/groups/staging</StagingUrl>
<!-- skips checkstyle and find bugs -->
<skipCheck>false</skipCheck>
......@@ -431,6 +433,7 @@
<module>addons/hive-bridge</module>
<module>addons/falcon-bridge</module>
<module>addons/sqoop-bridge</module>
<module>addons/storm-bridge</module>
<module>distro</module>
</modules>
......@@ -639,7 +642,7 @@
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
<version>${commons-conf.version}</version>
</dependency>
<dependency>
......@@ -757,7 +760,7 @@
<dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
<version>1.3.7</version>
<version>${jettison.version}</version>
</dependency>
<dependency>
......@@ -978,34 +981,34 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>hive-bridge</artifactId>
<artifactId>atlas-dashboard</artifactId>
<version>${project.version}</version>
<type>war</type>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>falcon-bridge</artifactId>
<artifactId>atlas-webapp</artifactId>
<version>${project.version}</version>
<type>war</type>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>sqoop-bridge</artifactId>
<artifactId>hive-bridge</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-dashboard</artifactId>
<artifactId>falcon-bridge</artifactId>
<version>${project.version}</version>
<type>war</type>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<artifactId>sqoop-bridge</artifactId>
<version>${project.version}</version>
<type>war</type>
</dependency>
<!--Scala dependencies-->
......@@ -1257,7 +1260,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<version>2.4</version>
</plugin>
<plugin>
......@@ -1403,6 +1406,20 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<phase>verify</phase>
<goals>
<goal>jar-no-fork</goal>
<goal>test-jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
......
release-log.txt
......@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-182 Add data model for Storm topology elements (svenkat,yhemanth via shwethags)
ATLAS-414 Doc: Increase MAVEN_OPTS limit to 512m in InstallationSteps.twiki (yhemanth via shwethags)
ATLAS-418 Update atlas website (shwethags)
ATLAS-392 Rename application.properties to atlas-application.properties (rishabhbhardwaj via shwethags)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment