Skip to content

A
atlas
Commit 22624786 by Shwetha GS
Options

removed un-used modules

parent fdd841dd
Showing with 0 additions and 1382 deletions
addons/metadata-bridge-parent/metadata-bridge-hive/pom.xml deleted 100755 → 0
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-bridge-parent</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>atlas-bridge-hive</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
<version>0.13.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.4.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>0.13.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
</plugin>
<!--
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-common</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
<outputDirectory>${project.build.directory}</outputDirectory>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
-->
</plugins>
</build>
</project>
\ No newline at end of file
addons/metadata-bridge-parent/metadata-bridge-hive/src/test/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/HiveLineageInfoTest.java deleted 100755 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.bridge.hivelineage.hook;
import com.google.gson.Gson;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
public class HiveLineageInfoTest {
public static String parseQuery(String query) throws SemanticException,
ParseException {
HiveLineageInfo lep = new HiveLineageInfo();
lep.getLineageInfo(query);
Gson gson = new Gson();
String jsonOut = gson.toJson(lep.getHLBean());
return jsonOut;
}
}
pom.xml
......@@ -134,7 +134,6 @@
<module>repository</module>
<module>webapp</module>
<module>docs</module>
<module>tools</module>
<module>addons/hive-bridge</module>
</modules>
......
tools/pom.xml deleted 100755 → 0
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>atlas-tools</artifactId>
<description>Apache Atlas Type System Tools Module</description>
<name>Apache Atlas Type System Tools</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>jline</artifactId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-actors</artifactId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scalap</artifactId>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalamacros</groupId>
<artifactId>quasiquotes_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-native_2.10</artifactId>
</dependency>
<dependency>
<groupId>com.github.nscala-time</groupId>
<artifactId>nscala-time_2.10</artifactId>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-actor_2.10</artifactId>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_2.10</artifactId>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-slf4j_2.10</artifactId>
</dependency>
<dependency>
<groupId>io.spray</groupId>
<artifactId>spray-routing</artifactId>
</dependency>
<dependency>
<groupId>io.spray</groupId>
<artifactId>spray-can</artifactId>
</dependency>
<dependency>
<groupId>io.spray</groupId>
<artifactId>spray-httpx</artifactId>
</dependency>
<dependency>
<groupId>io.spray</groupId>
<artifactId>spray-testkit</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>com.tinkerpop.blueprints</groupId>
<artifactId>blueprints-core</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-core</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-es</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
tools/src/main/resources/application.conf deleted 100755 → 0
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
akka {
loglevel = DEBUG
stdout-loglevel = DEBUG
event-handlers = ["akka.event.Logging$DefaultLogger"]
default-dispatcher {
fork-join-executor {
parallelism-min = 8
}
}
test {
timefactor = 1
}
}
spray {
can {
server {
server-header = "Metadata Service"
}
}
}
http {
host = "0.0.0.0"
host = ${?HOST}
port = 9140
port = ${?PORT}
}
graphRepo {
storage.backend = inmemory
# Graph Search Index
#index.search.backend=elasticsearch
#index.search.directory=target/data/es
#index.search.elasticsearch.client-only=false
#index.search.elasticsearch.local-mode=true
}
tools/src/main/scala/org/apache/atlas/tools/cli/Console.scala deleted 100755 → 0
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.tools.cli
import org.apache.atlas.repository.memory.MemRepository
import org.apache.atlas.typesystem.types.TypeSystem
import scala.tools.nsc.Settings
import scala.tools.nsc.interpreter.{ILoop, IMain}
object Console extends App {
val settings = new Settings
settings.usejavacp.value = true
settings.deprecation.value = true
settings.bootclasspath.value += """/Users/hbutani/.m2/repository/org/apache/atlas/1.0-SNAPSHOT/atlas-1.0-SNAPSHOT.jar:/Users/hbutani/.m2/repository/org/scala-lang/scala-compiler/2.10.4/scala-compiler-2.10.4.jar:/Users/hbutani/.m2/repository/org/scala-lang/scala-reflect/2.10.4/scala-reflect-2.10.4.jar:/Users/hbutani/.m2/repository/org/scala-lang/jline/2.10.4/jline-2.10.4.jar:/Users/hbutani/.m2/repository/org/fusesource/jansi/jansi/1.4/jansi-1.4.jar:/Users/hbutani/.m2/repository/org/scala-lang/scala-library/2.10.4/scala-library-2.10.4.jar:/Users/hbutani/.m2/repository/org/scala-lang/scala-actors/2.10.4/scala-actors-2.10.4.jar:/Users/hbutani/.m2/repository/org/scala-lang/scalap/2.10.4/scalap-2.10.4.jar:/Users/hbutani/.m2/repository/org/scalatest/scalatest_2.10/2.2.0/scalatest_2.10-2.2.0.jar:/Users/hbutani/.m2/repository/org/scalamacros/quasiquotes_2.10/2.0.1/quasiquotes_2.10-2.0.1.jar:/Users/hbutani/.m2/repository/org/json4s/json4s-native_2.10/3.2.11/json4s-native_2.10-3.2.11.jar:/Users/hbutani/.m2/repository/org/json4s/json4s-core_2.10/3.2.11/json4s-core_2.10-3.2.11.jar:/Users/hbutani/.m2/repository/org/json4s/json4s-ast_2.10/3.2.11/json4s-ast_2.10-3.2.11.jar:/Users/hbutani/.m2/repository/com/thoughtworks/paranamer/paranamer/2.6/paranamer-2.6.jar:/Users/hbutani/.m2/repository/com/github/nscala-time/nscala-time_2.10/1.6.0/nscala-time_2.10-1.6.0.jar:/Users/hbutani/.m2/repository/joda-time/joda-time/2.5/joda-time-2.5.jar:/Users/hbutani/.m2/repository/org/joda/joda-convert/1.2/joda-convert-1.2.jar:/Users/hbutani/.m2/repository/com/typesafe/config/1.2.1/config-1.2.1.jar:/Users/hbutani/.m2/repository/com/typesafe/akka/akka-actor_2.10/2.3.7/akka-actor_2.10-2.3.7.jar:/Users/hbutani/.m2/repository/com/typesafe/akka/akka-testkit_2.10/2.3.7/akka-testkit_2.10-2.3.7.jar:/Users/hbutani/.m2/repository/com/typesafe/akka/akka-slf4j_2.10/2.3.7/akka-slf4j_2.10-2.3.7.jar:/Users/hbutani/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/hbutani/.m2/repository/io/spray/spray-routing/1.3.1/spray-routing-1.3.1.jar:/Users/hbutani/.m2/repository/io/spray/spray-http/1.3.1/spray-http-1.3.1.jar:/Users/hbutani/.m2/repository/org/parboiled/parboiled-scala_2.10/1.1.6/parboiled-scala_2.10-1.1.6.jar:/Users/hbutani/.m2/repository/org/parboiled/parboiled-core/1.1.6/parboiled-core-1.1.6.jar:/Users/hbutani/.m2/repository/io/spray/spray-util/1.3.1/spray-util-1.3.1.jar:/Users/hbutani/.m2/repository/com/chuusai/shapeless_2.10/1.2.4/shapeless_2.10-1.2.4.jar:/Users/hbutani/.m2/repository/io/spray/spray-can/1.3.1/spray-can-1.3.1.jar:/Users/hbutani/.m2/repository/io/spray/spray-io/1.3.1/spray-io-1.3.1.jar:/Users/hbutani/.m2/repository/io/spray/spray-httpx/1.3.1/spray-httpx-1.3.1.jar:/Users/hbutani/.m2/repository/org/jvnet/mimepull/mimepull/1.9.4/mimepull-1.9.4.jar:/Users/hbutani/.m2/repository/io/spray/spray-testkit/1.3.1/spray-testkit-1.3.1.jar:/Users/hbutani/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/hbutani/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/hbutani/.m2/repository/junit/junit/4.10/junit-4.10.jar:/Users/hbutani/.m2/repository/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1.jar"""
val in = new IMain(settings) {
override protected def parentClassLoader = settings.getClass.getClassLoader()
}
new SampleILoop().process(settings)
}
class SampleILoop extends ILoop {
val ts: TypeSystem = TypeSystem.getInstance()
//intp = Console.in
val mr: MemRepository = new MemRepository(ts)
override def prompt = "==> "
addThunk {
intp.beQuietDuring {
intp.addImports("java.lang.Math._")
intp.addImports("org.json4s.native.Serialization.{read, write => swrite}")
intp.addImports("org.json4s._")
intp.addImports("org.json4s.native.JsonMethods._")
intp.addImports("org.apache.atlas.tools.dsl._")
//intp.bindValue("service", ms)
//intp.bindValue("cp", intp.compilerClasspath)
}
}
override def printWelcome() {
echo("\n" +
" \\,,,/\n" +
" (o o)\n" +
"-----oOOo-(_)-oOOo-----")
}
}
tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala deleted 100755 → 0
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.tools.dsl
import org.apache.atlas.typesystem.ITypedStruct
import org.apache.atlas.typesystem.types.{StructType, TypeSystem}
import scala.language.dynamics
class DynamicTypedStruct(val ts: ITypedStruct) extends Dynamic {
def selectDynamic(name: String) = ts.get(name)
def updateDynamic(name: String)(value: Any) {
var value1 = value
if (value != null && value.isInstanceOf[DynamicTypedStruct]) {
value1 = value.asInstanceOf[DynamicTypedStruct].ts
}
ts.set(name, value1)
}
def dataType = TypeSystem.getInstance().getDataType(classOf[StructType], ts.getTypeName)
}
tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala deleted 100755 → 0
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.tools
import org.apache.atlas.repository.memory.MemRepository
import org.apache.atlas.typesystem.json.{BigDecimalSerializer, BigIntegerSerializer, Serialization, TypedStructSerializer}
import org.apache.atlas.typesystem.persistence.StructInstance
import org.apache.atlas.typesystem.types._
import org.apache.atlas.typesystem.{IStruct, ITypedStruct}
import org.json4s._
import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.{write => swrite}
import scala.collection.JavaConversions._
import scala.language.implicitConversions
package object dsl {
val defFormat = new DefaultFormats {
override protected def dateFormatter = TypeSystem.getInstance().getDateFormat;
override val typeHints = NoTypeHints
}
implicit val formats = defFormat + new TypedStructSerializer +
new BigDecimalSerializer + new BigIntegerSerializer
val BOOLEAN_TYPE = DataTypes.BOOLEAN_TYPE
val BYTE_TYPE = DataTypes.BYTE_TYPE
val SHORT_TYPE = DataTypes.SHORT_TYPE
val INT_TYPE = DataTypes.INT_TYPE
val LONG_TYPE = DataTypes.LONG_TYPE
val FLOAT_TYPE = DataTypes.FLOAT_TYPE
val DOUBLE_TYPE = DataTypes.DOUBLE_TYPE
val BIGINT_TYPE = DataTypes.BIGINTEGER_TYPE
val BIGDECIMAL_TYPE = DataTypes.BIGDECIMAL_TYPE
val DATE_TYPE = DataTypes.DATE_TYPE
val STRING_TYPE = DataTypes.STRING_TYPE
val ATTR_OPTIONAL = Multiplicity.OPTIONAL
val ATTR_REQUIRED = Multiplicity.REQUIRED
def repo = new MemRepository(ts)
def arrayType(dT: IDataType[_]) = ts.defineArrayType(dT)
def mapType(kT: IDataType[_], vT: IDataType[_]) = ts.defineMapType(kT, vT)
def attrDef(name: String, dT: IDataType[_],
m: Multiplicity = Multiplicity.OPTIONAL,
isComposite: Boolean = false,
reverseAttributeName: String = null) = {
require(name != null)
require(dT != null)
new AttributeDefinition(name, dT.getName, m, isComposite, reverseAttributeName)
}
def listTypes = ts.getTypeNames.sorted.toList.mkString("[", ",", "]")
def ts = TypeSystem.getInstance
def defineStructType(name: String, attrDef: AttributeDefinition*) = {
require(name != null)
ts.defineStructType(name, false, attrDef: _*)
}
def createInstance(typeName: String, jsonStr: String)(implicit formats: Formats) = {
val j = parse(jsonStr)
assert(j.isInstanceOf[JObject])
var j1 = j.asInstanceOf[JObject]
j1 = JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(typeName)) :: j1.obj)
new DynamicTypedStruct(Extraction.extract[StructInstance](j1))
}
def createInstance(typeName: String) = {
new DynamicTypedStruct(
ts.getDataType(classOf[StructType], typeName).asInstanceOf[IConstructableType[IStruct, ITypedStruct]].createInstance())
}
implicit def dynTypedStructToTypedStruct(s: DynamicTypedStruct) = s.ts
implicit def dynTypedStructToJson(s: DynamicTypedStruct)(implicit formats: Formats) = {
Extraction.decompose(s.ts)(formats)
}
}
tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala deleted 100755 → 0
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.tools.simpleserver
import akka.actor.{ActorSystem, Props}
import akka.io.IO
import com.typesafe.config.ConfigFactory
import org.apache.atlas.repository.memory.MemRepository
import org.apache.atlas.typesystem.types.TypeSystem
import spray.can.Http
/**
* A Simple Spray based server to test the TypeSystem and MemRepository.
*
* @example {{{
* -- Using the [[ https://github.com/jakubroztocil/httpie Httpie tool]]
*
* http GET localhost:9140/listTypeNames
* pbpaste | http PUT localhost:9140/defineTypes
* http GET localhost:9140/typeDetails typeNames:='["Department", "Person", "Manager"]'
*
* pbpaste | http PUT localhost:9140/createInstance
* pbpaste | http GET localhost:9140/getInstance
* }}}
*
* - On the Mac, pbpaste makes available what is copied to clipboard. Copy contents of resources/sampleTypes.json
* - for createInstance resources/sampleInstance.json is an example
* - for getInstance send an Id back, you can copy the output from createInstance.
*
*/
object Main extends App {
val config = ConfigFactory.load()
val host = config.getString("http.host")
val port = config.getInt("http.port")
implicit val system = ActorSystem("atlasservice")
val typSys = TypeSystem.getInstance()
val memRepo = new MemRepository(typSys)
val api = system.actorOf(Props(new RestInterface(typSys, memRepo)), "httpInterface")
IO(Http) ! Http.Bind(listener = api, interface = host, port = port)
}
tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala deleted 100755 → 0
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.tools.simpleserver
import akka.actor._
import akka.util.Timeout
import com.google.common.collect.ImmutableList
import org.apache.atlas.repository.memory.MemRepository
import org.apache.atlas.typesystem.json._
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.types._
import org.apache.atlas.typesystem.{ITypedReferenceableInstance, TypesDef}
import org.json4s.{Formats, NoTypeHints}
import spray.httpx.Json4sSupport
import scala.concurrent.duration._
class MetadataActor(val typeSystem: TypeSystem, val memRepository: MemRepository) extends Actor with ActorLogging {
import org.apache.atlas.tools.simpleserver.MetadataProtocol._
import scala.collection.JavaConversions._
import scala.language.postfixOps
implicit val timeout = Timeout(5 seconds)
def receive = {
case ListTypeNames() =>
sender ! TypeNames(typeSystem.getTypeNames.toList)
case GetTypeDetails(typeNames) =>
val typesDef = TypesSerialization.convertToTypesDef(typeSystem, (d: IDataType[_]) => typeNames.contains(d.getName))
sender ! TypeDetails(typesDef)
case DefineTypes(typesDef: TypesDef) =>
typesDef.enumTypes.foreach(typeSystem.defineEnumType(_))
typeSystem.defineTypes(ImmutableList.copyOf(typesDef.structTypes.toArray),
ImmutableList.copyOf(typesDef.traitTypes.toArray),
ImmutableList.copyOf(typesDef.classTypes.toArray))
var newTypes: List[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]] = Nil
typesDef.traitTypes.foreach { tDef =>
val nm = tDef.typeName
newTypes = newTypes :+
typeSystem.getDataType(classOf[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]], nm)
}
typesDef.classTypes.foreach { tDef =>
val nm = tDef.typeName
newTypes = newTypes :+
typeSystem.getDataType(classOf[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]], nm)
}
memRepository.defineTypes(newTypes)
sender ! TypesCreated
case CreateInstance(i) =>
val r = memRepository.create(i)
sender ! InstanceCreated(r.getId)
case GetInstance(id) =>
val r = memRepository.get(id)
sender ! InstanceDetails(r)
}
}
object MetadataProtocol {
case class ListTypeNames()
case class TypeNames(typeNames: List[String])
case class GetTypeDetails(typeNames: List[String])
case class TypeDetails(types: TypesDef)
case class DefineTypes(types: TypesDef)
case class TypesCreated()
case class CreateInstance(i: ITypedReferenceableInstance)
case class InstanceCreated(id: Id)
case class GetInstance(id: Id)
case class InstanceDetails(i: ITypedReferenceableInstance)
}
trait Json4sProtocol extends Json4sSupport {
val typeSystem: TypeSystem
val memRepository: MemRepository
implicit def json4sFormats: Formats =
org.json4s.native.Serialization.formats(NoTypeHints) + new MultiplicitySerializer +
new TypedStructSerializer +
new TypedReferenceableInstanceSerializer +
new BigDecimalSerializer + new BigIntegerSerializer + new IdSerializer
}
tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala deleted 100755 → 0
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.tools.simpleserver
import akka.actor._
import akka.util.Timeout
import org.apache.atlas.repository.memory.MemRepository
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.types.TypeSystem
import org.apache.atlas.typesystem.{ITypedReferenceableInstance, TypesDef}
import spray.http.StatusCodes
import spray.routing._
import scala.concurrent.duration._
class Responder(val typeSystem: TypeSystem, val memRepository : MemRepository,
requestContext:RequestContext, mdSvc:ActorRef) extends Actor with Json4sProtocol with ActorLogging {
import org.apache.atlas.tools.simpleserver.MetadataProtocol._
def receive = {
case typNames:TypeNames =>
requestContext.complete(StatusCodes.OK, typNames)
self ! PoisonPill
case tD:TypeDetails =>
requestContext.complete(StatusCodes.OK, tD)
self ! PoisonPill
case TypesCreated =>
requestContext.complete(StatusCodes.OK)
self ! PoisonPill
case InstanceCreated(id) =>
requestContext.complete(StatusCodes.OK, id)
case InstanceDetails(i) =>
requestContext.complete(StatusCodes.OK, i)
}
}
class RestInterface(val typeSystem: TypeSystem, val memRepository : MemRepository) extends HttpServiceActor
with RestApi {
def receive = runRoute(routes)
}
trait RestApi extends HttpService with Json4sProtocol with ActorLogging { actor: Actor =>
import MetadataProtocol._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.language.postfixOps
val typeSystem : TypeSystem
val memRepository : MemRepository
implicit val timeout = Timeout(10 seconds)
import akka.pattern.{ask, pipe}
val mdSvc = context.actorOf(Props(new MetadataActor(typeSystem, memRepository)))
def routes: Route =
path("listTypeNames") {
get { requestContext =>
val responder: ActorRef = createResponder(requestContext)
pipe(mdSvc.ask(ListTypeNames))
mdSvc.ask(ListTypeNames()).pipeTo(responder)
}
} ~
path("typeDetails") {
get {
entity(as[GetTypeDetails]) { typeDetails => requestContext =>
val responder = createResponder(requestContext)
mdSvc.ask(typeDetails).pipeTo(responder)
}
}
} ~
path("defineTypes") {
put {
entity(as[TypesDef]) { typesDef => requestContext =>
val responder = createResponder(requestContext)
mdSvc.ask(DefineTypes(typesDef)).pipeTo(responder)
}
}
} ~
path("createInstance") {
put {
entity(as[ITypedReferenceableInstance]) { i => requestContext =>
val responder = createResponder(requestContext)
mdSvc.ask(CreateInstance(i)).pipeTo(responder)
}
}
} ~
path("getInstance") {
get {
entity(as[Id]) { id => requestContext =>
val responder = createResponder(requestContext)
mdSvc.ask(GetInstance(id)).pipeTo(responder)
}
}
}
def createResponder(requestContext:RequestContext) = {
context.actorOf(Props(new Responder(typeSystem, memRepository, requestContext, mdSvc)))
}
}
\ No newline at end of file
tools/src/test/scala/org/apache/atlas/tools/hive/HiveMockMetadataService.scala deleted 100755 → 0
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.tools.hive
object HiveMockMetadataService {
def getTable(dbName: String, table: String): Table = {
return Table(dbName, table,
StorageDescriptor(
List[FieldSchema](
FieldSchema("d_date_sk", "int", null),
FieldSchema("d_date_id", "string", null),
FieldSchema("d_date", "string", null),
FieldSchema("d_month_seq", "int", null),
FieldSchema("d_week_seq", "int", null),
FieldSchema("d_quarter_seq", "int", null),
FieldSchema("d_year", "int", null),
FieldSchema("d_dow", "int", null),
FieldSchema("d_moy", "int", null),
FieldSchema("d_dom", "int", null),
FieldSchema("d_qoy", "int", null),
FieldSchema("d_fy_year", "int", null),
FieldSchema("d_fy_quarter_seq", "int", null),
FieldSchema("d_fy_week_seq", "int", null),
FieldSchema("d_day_name", "string", null),
FieldSchema("d_quarter_name", "string", null),
FieldSchema("d_holiday", "string", null),
FieldSchema("d_weekend", "string", null),
FieldSchema("d_following_holiday", "string", null),
FieldSchema("d_first_dom", "int", null),
FieldSchema("d_last_dom", "int", null),
FieldSchema("d_same_day_ly", "int", null),
FieldSchema("d_same_day_lq", "int", null),
FieldSchema("d_current_day", "string", null),
FieldSchema("d_current_week", "string", null),
FieldSchema("d_current_month", "string", null),
FieldSchema("d_current_quarter", "string", null),
FieldSchema("d_current_year", "string", null)
),
"file:/tmp/warehouse/tpcds.db/date_dim",
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
false,
0, List[String](), List[String](),
Map[String, String](),
false
),
Map[String, String](),
"Table")
}
case class FieldSchema(name: String, typeName: String, comment: String)
case class SerDe(name: String, serializationLib: String, parameters: Map[String, String])
case class StorageDescriptor(fields: List[FieldSchema],
location: String, inputFormat: String,
outputFormat: String, compressed: Boolean,
numBuckets: Int, bucketColumns: List[String],
sortColumns: List[String],
parameters: Map[String, String],
storedAsSubDirs: Boolean
)
case class Table(dbName: String, tableName: String, storageDesc: StorageDescriptor,
parameters: Map[String, String],
tableType: String)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment