Skip to content

A
atlas
Commit 61079cac by Venkatesh Seetharam
Options

ISSUE-43 Generate tar ball and add scripts to start/stop server. Contributed by Venkatesh Seetharam

parent 882c40fa
Showing with 370 additions and 147 deletions
InstallationSteps.txt
......@@ -34,7 +34,7 @@ Building metadata from the source repository
* git clone git@github.com:hortonworks/metadata.git metadata
* cd metadata
* export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean verify
* export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install
2. Deploying Metadata
......@@ -48,9 +48,47 @@ Tar can be found in {project dir}/target/apache-metadata-${project.version}-bin.
Tar is structured as follows
TBD
|- bin
|- metadata-start.sh
|- metadata-stop.sh
|- conf
|- application.properties
|- graph.properties
|- log4j.xml
|- docs
|- server
|- webapp
|- metadata.war
|- README
|- NOTICE.txt
|- LICENSE.txt
|- DISCLAIMER.txt
|- CHANGES.txt
3. Installing & running Metadata
--------------------------------
TBD
\ No newline at end of file
a. Installing Metadata
~~~~~~~~~~~~~~~~~~~~~~
* tar -xzvf apache-metadata-${project.version}-bin.tar.gz
* cd metadata-${project.version}
b. Starting Metadata Server
~~~~~~~~~~~~~~~~~~~~~~~~~
* bin/metadata-start.sh
c. Using Falcon
~~~~~~~~~~~~~~~
* curl -v http://localhost:21000/api/metadata/admin/version
{"Version":"v0.1"}
* curl -v http://localhost:21000/api/metadata/types/list
{"list":["biginteger","short","byte","int","string","bigdecimal","boolean","date","double","long","float"],"requestId":"902580786@qtp-1479771328-0"}
d. Stopping Falcon Server
~~~~~~~~~~~~~~~~~~~~~~~~~
* bin/metadata-stop.sh
hivehook/pom.xml deleted 100644 → 0
<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-governance</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-hivehook</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
<name>metadata-hivehook</name>
<packaging>jar</packaging>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<repositories>
<repository>
<id>central</id>
<!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution -->
<name>Maven Repository</name>
<url>https://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>apache-repo</id>
<name>Apache Repository</name>
<url>https://repository.apache.org/content/repositories/releases</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<fork>true</fork>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.3.2</version>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
<version>0.13.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>0.13.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>
hivetypes/pom.xml
......@@ -10,6 +10,29 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>metadata-hivetypes</artifactId>
<profiles>
<profile>
<id>hadoop-2</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
</dependency>
</dependencies>
</profile>
</profiles>
<dependencies>
<dependency>
<groupId>org.apache.hive</groupId>
......@@ -30,22 +53,18 @@
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId>
<version>0.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client</artifactId>
<version>2.6.0</version>
<artifactId>hadoop-mapreduce-client-common</artifactId>
</dependency>
<dependency>
<groupId>log4j</groupId>
......
pom.xml
......@@ -729,6 +729,17 @@
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>src/main/assemblies/standalone-package.xml</descriptor>
<descriptor>src/main/assemblies/src-package.xml</descriptor>
</descriptors>
<finalName>apache-metadata-governance-${project.version}</finalName>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.7.2</version>
......
src/bin/metadata-start.sh 0 → 100644
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
#
# resolve links - $0 may be a softlink
PRG="${0}"
while [ -h "${PRG}" ]; do
ls=`ls -ld "${PRG}"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "${PRG}"`/"$link"
fi
done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
if [ -z "$METADATA_CONF" ]; then
METADATA_CONF=${BASEDIR}/conf
fi
export METADATA_CONF
if [ -f "${METADATA_CONF}/metadata-env.sh" ]; then
. "${METADATA_CONF}/metadata-env.sh"
fi
if test -z ${JAVA_HOME}
then
JAVA_BIN=`which java`
JAR_BIN=`which jar`
else
JAVA_BIN=${JAVA_HOME}/bin/java
JAR_BIN=${JAVA_HOME}/bin/jar
fi
export JAVA_BIN
if [ ! -e $JAVA_BIN ] || [ ! -e $JAR_BIN ]; then
echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
exit 1
fi
# default the heap size to 1GB
DEFAULT_JAVA_HEAP_MAX=-Xmx1024m
METADATA_OPTS="$DEFAULT_JAVA_HEAP_MAX $METADATA_OPTS"
METADATACPPATH="$METADATA_CONF"
METADATA_EXPANDED_WEBAPP_DIR=${METADATA_EXPANDED_WEBAPP_DIR:-${BASEDIR}/server/webapp}
export METADATA_EXPANDED_WEBAPP_DIR
# set the server classpath
if [ ! -d ${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF ]; then
mkdir -p ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
cd ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
$JAR_BIN -xf ${BASEDIR}/server/webapp/metadata.war
cd -
fi
METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/classes"
METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/lib/*:${BASEDIR}/libext/*"
# log and pid dirs for applications
METADATA_LOG_DIR="${METADATA_LOG_DIR:-$BASEDIR/logs}"
export METADATA_LOG_DIR
METADATA_PID_DIR="${METADATA_PID_DIR:-$BASEDIR/logs}"
# create the pid dir if its not there
[ -w "$METADATA_PID_DIR" ] || mkdir -p "$METADATA_PID_DIR"
export METADATA_PID_DIR
METADATA_PID_FILE=${METADATA_PID_DIR}/metadata.pid
export METADATA_PID_FILE
METADATA_DATA_DIR=${METADATA_DATA_DIR:-${BASEDIR}/data}
METADATA_HOME_DIR="${METADATA_HOME_DIR:-$BASEDIR}"
export METADATA_HOME_DIR
# make sure the process is not running
if [ -f $METADATA_PID_FILE ]; then
if kill -0 `cat $METADATA_PID_FILE` > /dev/null 2>&1; then
echo metadata running as process `cat $METADATA_PID_FILE`. Stop it first.
exit 1
fi
fi
mkdir -p $METADATA_LOG_DIR
pushd ${BASEDIR} > /dev/null
JAVA_PROPERTIES="$METADATA_OPTS $METADATA_PROPERTIES -Dmetadata.log.dir=$METADATA_LOG_DIR -Dmetadata.home=${METADATA_HOME_DIR}"
shift
while [[ ${1} =~ ^\-D ]]; do
JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
shift
done
TIME=`date +%Y%m%d%H%M%s`
nohup ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${METADATACPPATH} org.apache.hadoop.metadata.Main -app ${BASEDIR}/server/webapp/metadata $* > "${METADATA_LOG_DIR}/metadata-server.$TIME.out" 2>&1 &
echo $! > $METADATA_PID_FILE
popd > /dev/null
echo Metadata Server started!!!
src/bin/metadata-stop.sh 0 → 100644
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
#
# resolve links - $0 may be a softlink
PRG="${0}"
while [ -h "${PRG}" ]; do
ls=`ls -ld "${PRG}"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "${PRG}"`/"$link"
fi
done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
METADATA_PID_DIR="${METADATA_PID_DIR:-$BASEDIR/logs}"
# create the pid dir if its not there
[ -w "$METADATA_PID_DIR" ] || mkdir -p "$METADATA_PID_DIR"
export METADATA_PID_DIR
METADATA_PID_FILE=${METADATA_PID_DIR}/metadata.pid
export METADATA_PID_FILE
if [ -f $METADATA_PID_FILE ]
then
kill -15 `cat $METADATA_PID_FILE`
echo Metadata Server stopped
else
echo "pid file $METADATA_PID_FILE not present"
fi
webapp/src/main/resources/application.properties → src/conf/application.properties
......@@ -20,19 +20,6 @@
# Graph implementation
metadata.graph.impl.class=org.apache.hadoop.metadata.repository.graph.TitanGraphService
# Graph Storage
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=target/data/berkeley
# Graph Search Index
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=target/data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
######### Graph Database Configs #########
######### Security Properties #########
# SSL config
......
webapp/src/main/resources/graph.properties → src/conf/graph.properties
storage.backend=inmemory
# Graph Storage
storage.backend=berkeleyje
storage.directory=./data/berkeley
# Graph Search Index
index.search.backend=elasticsearch
index.search.directory=target/data/es
index.search.directory=./data/es
index.search.elasticsearch.client-only=false
index.search.elasticsearch.local-mode=true
webapp/src/main/resources/log4j.xml → src/conf/log4j.xml
......@@ -28,7 +28,7 @@
</appender>
<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${user.dir}/target/logs/application.log"/>
<param name="File" value="${metadata.log.dir}/application.log"/>
<param name="Append" value="true"/>
<param name="Threshold" value="debug"/>
<layout class="org.apache.log4j.PatternLayout">
......@@ -37,7 +37,7 @@
</appender>
<appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${user.dir}/target/logs/audit.log"/>
<param name="File" value="${metadata.log.dir}/audit.log"/>
<param name="Append" value="true"/>
<param name="Threshold" value="debug"/>
<layout class="org.apache.log4j.PatternLayout">
......
src/main/assemblies/src-package.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<formats>
<format>tar.gz</format>
</formats>
<id>sources</id>
<baseDirectory>apache-metadata-sources-${project.version}</baseDirectory>
<fileSets>
<fileSet>
<directory>.</directory>
<excludes>
<exclude>**/target/**</exclude>
<exclude>**/logs/**</exclude>
<exclude>**/.git*/**</exclude>
<exclude>backlog</exclude>
<exclude>**/.idea/**</exclude>
<exclude>**/*.iml</exclude>
<exclude>**/*.log</exclude>
</excludes>
</fileSet>
</fileSets>
</assembly>
src/main/assemblies/standalone-package.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<formats>
<format>tar.gz</format>
</formats>
<id>bin</id>
<baseDirectory>metadata-${project.version}</baseDirectory>
<fileSets>
<fileSet>
<directory>src/conf/</directory>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>docs/target/site</directory>
<outputDirectory>docs</outputDirectory>
</fileSet>
<fileSet>
<directory>webapp/target/apidocs</directory>
<outputDirectory>apidocs</outputDirectory>
</fileSet>
<fileSet>
<directory>src/bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>logs</directory>
<outputDirectory>logs</outputDirectory>
<directoryMode>0777</directoryMode>
<excludes>
<exclude>*</exclude>
<exclude>**/**</exclude>
</excludes>
</fileSet>
<fileSet>
<directory>./</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>DISCLAIMER.txt</include>
<include>NOTICE.txt</include>
<include>LICENSE.txt</include>
<include>CHANGES.txt</include>
<include>README</include>
</includes>
</fileSet>
<fileSet>
<directory>src/main/examples</directory>
<outputDirectory>examples</outputDirectory>
</fileSet>
</fileSets>
<files>
<file>
<source>webapp/target/metadata-governance.war</source>
<outputDirectory>server/webapp</outputDirectory>
<destName>metadata.war</destName>
</file>
</files>
</assembly>
webapp/src/main/webapp/index.html
......@@ -25,7 +25,7 @@
<title>Apache Falcon - Data management and processing platform</title>
</head>
<body class="topBarEnabled">
<h1> Apache Metadata</h1>
<h1> Apache Metadata Governance</h1>
More information at: <a href="http://dgc.incubator.apache.org/index.html" title="About">Project Website</a>
</body>
</html>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment