Skip to content

A
atlas
Commit 003b5a52 by Harish Butani
Options

simple rest server

parent 7f08d554
Showing with 890 additions and 12 deletions
typesystem/pom.xml
...@@ -160,7 +160,6 @@ ...@@ -160,7 +160,6 @@
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>
<version>${guava.version}</version> <version>${guava.version}</version>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
......
typesystem/src/main/resources/application.conf
akka { akka {
//loggers = [akka.event.slf4j.Slf4jLogger] loglevel = DEBUG
loglevel = debug stdout-loglevel = DEBUG
actor { event-handlers = ["akka.event.Logging$DefaultLogger"]
debug { default-dispatcher {
receive = on fork-join-executor {
lifecycle = on parallelism-min = 8
} }
} }
test {
timefactor = 1
}
}
spray {
can {
server {
server-header = "Metadata Service"
}
}
}
http {
host = "0.0.0.0"
host = ${?HOST}
port = 9140
port = ${?PORT}
} }
app {
interface="localhost"
port= 8080
}
\ No newline at end of file
typesystem/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/Main.scala 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.tools.simpleserver
import akka.actor.{Props, ActorSystem}
import akka.io.IO
import com.typesafe.config.ConfigFactory
import spray.can.Http
/**
* A Simple Spray based server to test the TypeSystem and MemRepository.
*
* @example {{{
* -- Using the [[ https://github.com/jakubroztocil/httpie Httpie tool]]
*
* http GET localhost:9140/listTypeNames
* pbpaste | http PUT localhost:9140/defineTypes
* http GET localhost:9140/typeDetails typeNames:='["Department", "Person", "Manager"]'
* }}}
*
* - On the Mac, pbpaste makes available what is copied to clipboard. Copy contents of resources/sampleTypes.json
*
*/
object Main extends App {
val config = ConfigFactory.load()
val host = config.getString("http.host")
val port = config.getInt("http.port")
implicit val system = ActorSystem("metadataservice")
val api = system.actorOf(Props(new RestInterface()), "httpInterface")
IO(Http) ! Http.Bind(listener = api, interface = host, port = port)
}
\ No newline at end of file
typesystem/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/MetadataService.scala 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.tools.simpleserver
import akka.actor._
import akka.util.Timeout
import com.google.common.collect.ImmutableList
import org.apache.hadoop.metadata.json._
import org.apache.hadoop.metadata.storage.memory.MemRepository
import org.apache.hadoop.metadata.types.{IDataType, TypeSystem}
import org.json4s.{Formats, NoTypeHints}
import spray.httpx.Json4sSupport
import scala.concurrent.duration._
class MetadataService extends Actor with ActorLogging {
import org.apache.hadoop.metadata.tools.simpleserver.MetadataProtocol._
import scala.collection.JavaConversions._
import scala.language.postfixOps
implicit val timeout = Timeout(5 seconds)
val typSys = new TypeSystem
val memRepo = new MemRepository(typSys)
def receive = {
case ListTypeNames() =>
sender ! TypeNames(typSys.getTypeNames.toList)
case GetTypeDetails(typeNames) =>
val typesDef = TypesSerialization.convertToTypesDef(typSys, (d : IDataType[_]) => typeNames.contains(d.getName))
sender ! TypeDetails(typesDef)
case DefineTypes(typesDef : TypesDef) =>
typesDef.enumTypes.foreach(typSys.defineEnumType(_))
typSys.defineTypes(ImmutableList.copyOf(typesDef.structTypes.toArray),
ImmutableList.copyOf(typesDef.traitTypes.toArray),
ImmutableList.copyOf(typesDef.classTypes.toArray))
sender ! TypesCreated
}
}
object MetadataProtocol {
case class ListTypeNames()
case class TypeNames(typeNames : List[String])
case class GetTypeDetails(typeNames : List[String])
case class TypeDetails(types : TypesDef)
case class DefineTypes(types : TypesDef)
case class TypesCreated()
}
object Json4sProtocol extends Json4sSupport {
implicit def json4sFormats: Formats =
org.json4s.native.Serialization.formats(NoTypeHints) + new MultiplicitySerializer + new TypedStructSerializer +
new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
}
typesystem/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/RestInterface.scala 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.tools.simpleserver
import akka.actor._
import akka.util.Timeout
import org.apache.hadoop.metadata.json.TypesDef
import spray.http.StatusCodes
import spray.routing._
import scala.concurrent.duration._
class Responder(requestContext:RequestContext, ticketMaster:ActorRef) extends Actor with ActorLogging {
import org.apache.hadoop.metadata.tools.simpleserver.Json4sProtocol._
import org.apache.hadoop.metadata.tools.simpleserver.MetadataProtocol._
def receive = {
case typNames:TypeNames =>
requestContext.complete(StatusCodes.OK, typNames)
self ! PoisonPill
case tD:TypeDetails =>
requestContext.complete(StatusCodes.OK, tD)
self ! PoisonPill
case TypesCreated =>
requestContext.complete(StatusCodes.OK)
self ! PoisonPill
}
}
class RestInterface extends HttpServiceActor
with RestApi {
def receive = runRoute(routes)
}
trait RestApi extends HttpService with ActorLogging { actor: Actor =>
import MetadataProtocol._
import scala.language.postfixOps
import scala.concurrent.ExecutionContext.Implicits.global
implicit val timeout = Timeout(10 seconds)
import akka.pattern.{ask, pipe}
val mdSvc = context.actorOf(Props[MetadataService])
import Json4sProtocol._
def routes: Route =
path("listTypeNames") {
get { requestContext =>
val responder : ActorRef = createResponder(requestContext)
pipe(mdSvc.ask(ListTypeNames))
mdSvc.ask(ListTypeNames()).pipeTo(responder)
}
} ~
path("typeDetails") {
get {
entity(as[GetTypeDetails]) { typeDetails => requestContext =>
val responder = createResponder(requestContext)
mdSvc.ask(typeDetails).pipeTo(responder)
}
}
} ~
path("defineTypes") {
put {
entity(as[TypesDef]) { typesDef => requestContext =>
val responder = createResponder(requestContext)
mdSvc.ask(DefineTypes(typesDef)).pipeTo(responder)
}
}
}
def createResponder(requestContext:RequestContext) = {
context.actorOf(Props(new Responder(requestContext, mdSvc)))
}
}
\ No newline at end of file
typesystem/src/test/resources/sampleTypes.json 0 → 100644
{
"enumTypes":[
{
"name":"HiveObjectType",
"enumValues":[
{
"value":"GLOBAL",
"ordinal":1
},
{
"value":"DATABASE",
"ordinal":2
},
{
"value":"TABLE",
"ordinal":3
},
{
"value":"PARTITION",
"ordinal":4
},
{
"value":"COLUMN",
"ordinal":5
}
]
},
{
"name":"LockLevel",
"enumValues":[
{
"value":"DB",
"ordinal":1
},
{
"value":"TABLE",
"ordinal":2
},
{
"value":"PARTITION",
"ordinal":3
}
]
},
{
"name":"TxnState",
"enumValues":[
{
"value":"COMMITTED",
"ordinal":1
},
{
"value":"ABORTED",
"ordinal":2
},
{
"value":"OPEN",
"ordinal":3
}
]
},
{
"name":"PrincipalType",
"enumValues":[
{
"value":"USER",
"ordinal":1
},
{
"value":"ROLE",
"ordinal":2
},
{
"value":"GROUP",
"ordinal":3
}
]
}
],
"structTypes":[
{
"typeName":"t2",
"attributeDefinitions":[
{
"name":"a",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"s",
"dataTypeName":"t2",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
}
]
},
{
"typeName":"t1",
"attributeDefinitions":[
{
"name":"a",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"b",
"dataTypeName":"boolean",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"c",
"dataTypeName":"byte",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"d",
"dataTypeName":"short",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"e",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"f",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"g",
"dataTypeName":"long",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"h",
"dataTypeName":"float",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"i",
"dataTypeName":"double",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"j",
"dataTypeName":"biginteger",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"k",
"dataTypeName":"bigdecimal",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"l",
"dataTypeName":"date",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"m",
"dataTypeName":"array<int>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"n",
"dataTypeName":"array<bigdecimal>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"o",
"dataTypeName":"map<string,double>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
}
]
},
{
"typeName":"ts1",
"attributeDefinitions":[
{
"name":"a",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"b",
"dataTypeName":"boolean",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"c",
"dataTypeName":"byte",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"d",
"dataTypeName":"short",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"e",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"f",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"g",
"dataTypeName":"long",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"h",
"dataTypeName":"float",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"i",
"dataTypeName":"double",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"j",
"dataTypeName":"biginteger",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"k",
"dataTypeName":"bigdecimal",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"l",
"dataTypeName":"date",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"m",
"dataTypeName":"array<int>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"n",
"dataTypeName":"array<bigdecimal>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"o",
"dataTypeName":"map<string,double>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
}
]
}
],
"traitTypes":[
{
"superTypes":[
"B",
"C"
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.TraitType",
"typeName":"D",
"attributeDefinitions":[
{
"name":"d",
"dataTypeName":"short",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.TraitType",
"typeName":"A",
"attributeDefinitions":[
{
"name":"a",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"b",
"dataTypeName":"boolean",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"c",
"dataTypeName":"byte",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"d",
"dataTypeName":"short",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
"A"
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.TraitType",
"typeName":"B",
"attributeDefinitions":[
{
"name":"b",
"dataTypeName":"boolean",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
"A"
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.TraitType",
"typeName":"C",
"attributeDefinitions":[
{
"name":"c",
"dataTypeName":"byte",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.TraitType",
"typeName":"SecurityClearance",
"attributeDefinitions":[
{
"name":"level",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":null
}
]
}
],
"classTypes":[
{
"superTypes":[
"Person"
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.ClassType",
"typeName":"Manager",
"attributeDefinitions":[
{
"name":"subordinates",
"dataTypeName":"array<Person>",
"multiplicity":"collection",
"isComposite":false,
"reverseAttributeName":"manager"
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.ClassType",
"typeName":"Department",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"employees",
"dataTypeName":"array<Person>",
"multiplicity":"collection",
"isComposite":true,
"reverseAttributeName":"department"
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.ClassType",
"typeName":"t4",
"attributeDefinitions":[
{
"name":"a",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"b",
"dataTypeName":"boolean",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"c",
"dataTypeName":"byte",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"d",
"dataTypeName":"short",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"enum1",
"dataTypeName":"HiveObjectType",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"e",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"f",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"g",
"dataTypeName":"long",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"enum2",
"dataTypeName":"PrincipalType",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"h",
"dataTypeName":"float",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"i",
"dataTypeName":"double",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"j",
"dataTypeName":"biginteger",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"k",
"dataTypeName":"bigdecimal",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"enum3",
"dataTypeName":"TxnState",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"l",
"dataTypeName":"date",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"m",
"dataTypeName":"array<int>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"n",
"dataTypeName":"array<bigdecimal>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"o",
"dataTypeName":"map<string,double>",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"enum4",
"dataTypeName":"LockLevel",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.types.ClassType",
"typeName":"Person",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":null
},
{
"name":"department",
"dataTypeName":"Department",
"multiplicity":"required",
"isComposite":false,
"reverseAttributeName":"employees"
},
{
"name":"manager",
"dataTypeName":"Manager",
"multiplicity":"optional",
"isComposite":false,
"reverseAttributeName":"subordinates"
}
]
}
]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment