ISSUE-5 Enable integration tests and add a few. Contributed by Venkatesh Seetharam

InstallationSteps.txt

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+A. Building & Installing Metadata
+=================================
+
+0. Prerequisites
+------------------
+
+You would need the following installed:
+
+* JDK 1.7
+* Maven 3.x
+
+
+1. Building Metadata
+--------------------
+
+Building metadata from the source repository
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* git clone git@github.com:hortonworks/metadata.git metadata
+* cd metadata
+* export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean verify
+
+
+2. Deploying Metadata
+---------------------
+
+Once the build successfully completes, artifacts can be packaged for deployment.
+
+* mvn clean assembly:assembly -DskipTests -DskipITs
+
+Tar can be found in {project dir}/target/apache-metadata-${project.version}-bin.tar.gz
+
+Tar is structured as follows
+
+TBD
+
+3. Installing & running Metadata
+--------------------------------
+
+TBD
\ No newline at end of file

common/src/main/java/org/apache/hadoop/metadata/service/ServiceInitializer.java

@@ -34,16 +34,16 @@ public class ServiceInitializer {
     private final Services services = Services.get();
 
     public void initialize() throws MetadataException {
-        String serviceClassNames;
+        String[] serviceClassNames;
         try {
             PropertiesConfiguration configuration =
                     new PropertiesConfiguration("application.properties");
-            serviceClassNames = configuration.getString("application.services");
+            serviceClassNames = configuration.getStringArray("application.services");
         } catch (ConfigurationException e) {
             throw new RuntimeException("unable to get server properties");
         }
 
-        for (String serviceClassName : serviceClassNames.split(",")) {
+        for (String serviceClassName : serviceClassNames) {
             serviceClassName = serviceClassName.trim();
             if (serviceClassName.isEmpty()) {
                 continue;

pom.xml

@@ -724,6 +724,33 @@
                 </configuration>
             </plugin>
 
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <version>2.16</version>
+                <configuration>
+                    <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                    <forkMode>always</forkMode>
+                    <argLine>-Djava.security.krb5.realm= -Djava.security.krb5.kdc=
+                        -Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name}</argLine>
+                    <excludedGroups>${excluded.test.groups}</excludedGroups>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>integration-test</id>
+                        <goals>
+                            <goal>integration-test</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>verify</id>
+                        <goals>
+                            <goal>verify</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
             <!-- Run the application using "mvn jetty:run" -->
             <plugin>
                 <groupId>org.mortbay.jetty</groupId>

repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryService.java

@@ -102,6 +102,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
     @Override
     public String submitEntity(String entity, String entityType) {
         LOG.info("adding entity={} type={}", entity, entityType);
+        @SuppressWarnings("unchecked")
         Map<String, String> properties = (Map<String, String>) JSONValue.parse(entity);
 
         final String entityName = properties.get("entityName");

webapp/src/main/java/org/apache/hadoop/metadata/Main.java

@@ -23,7 +23,6 @@ import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
-import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.metadata.web.service.EmbeddedServer;
@@ -61,7 +60,8 @@ public final class Main {
 
     public static void main(String[] args) throws Exception {
         CommandLine cmd = parseArgs(args);
-        String projectVersion = getProjectVersion();
+        // todo: enable version for webapp
+        // String projectVersion = getProjectVersion();
         // String appPath = "webapp/target/metadata-webapp-" + projectVersion;
         String appPath = "webapp/target/metadata-governance";
 
@@ -82,11 +82,13 @@ public final class Main {
         server.start();
     }
 
+/*
     private static String getProjectVersion() throws ConfigurationException {
         PropertiesConfiguration configuration =
                 new PropertiesConfiguration("metadata-buildinfo.properties");
         return configuration.getString("project.version");
     }
+*/
 
     private static int getApplicationPort(CommandLine cmd, String enableTLSFlag) {
         final int appPort;
@@ -95,7 +97,7 @@ public final class Main {
         } else {
             // default : metadata.enableTLS is true
             appPort = StringUtils.isEmpty(enableTLSFlag)
-                    || enableTLSFlag.equals("true") ? 15443 : 15000;
+                    || enableTLSFlag.equals("true") ? 21443 : 21000;
         }
 
         return appPort;

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/AdminResource.java

@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.web.resources;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.metadata.web.util.Servlets;
-import org.apache.hadoop.util.VersionInfo;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 
@@ -69,7 +68,8 @@ public class AdminResource {
             try {
                 JSONObject response = new JSONObject();
                 response.put("Version", "v0.1"); // todo: get version
-                response.put("Hadoop", VersionInfo.getVersion() + "-r" + VersionInfo.getRevision());
+                // todo: add hadoop version?
+                // response.put("Hadoop", VersionInfo.getVersion() + "-r" + VersionInfo.getRevision());
                 version = Response.ok(response).build();
             } catch (JSONException e) {
                 throw new WebApplicationException(
@@ -79,4 +79,4 @@ public class AdminResource {
 
         return version;
     }
-}
\ No newline at end of file
+}

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java

@@ -108,9 +108,10 @@ public class EntityResource {
     @Produces(MediaType.APPLICATION_JSON)
     public Response getEntityDefinition(@PathParam("entityType") String entityType,
                                         @PathParam("entityName") String entityName) {
-
         final String entityDefinition = repositoryService.getEntityDefinition(entityName, entityType);
-        return Response.ok(entityDefinition).build();
+        return (entityDefinition == null)
+                ? Response.status(Response.Status.NOT_FOUND).build()
+                : Response.ok(entityDefinition).build();
     }
 
     @POST

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/GraphResource.java → webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java

@@ -56,15 +56,15 @@ import java.util.Set;
  * https://github.com/tinkerpop/rexster/wiki/Basic-REST-API
  */
 @Path("graph")
-public class GraphResource {
-    private static final Logger LOG = LoggerFactory.getLogger(GraphResource.class);
+public class RexsterGraphResource {
+    private static final Logger LOG = LoggerFactory.getLogger(RexsterGraphResource.class);
 
     public static final String RESULTS = "results";
     public static final String TOTAL_SIZE = "totalSize";
 
     private GraphService graphService;
 
-    public GraphResource() {
+    public RexsterGraphResource() {
         graphService = Services.get().getService(TitanGraphService.NAME);
         if (graphService == null) {
             throw new WebApplicationException(Response

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java

@@ -32,7 +32,7 @@ public class TypesResource {
 
     @POST
     @Path("submit/{type}")
-    @Consumes(MediaType.TEXT_XML)
+    @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
     public Response submit(@Context HttpServletRequest request,
                            @PathParam("type") String type) {

webapp/src/main/resources/application.properties

@@ -20,6 +20,16 @@ application.services=org.apache.hadoop.metadata.services.TitanGraphService,\
                      org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
 
 
+#########  Implementation classes  #########
+## DO NOT MODIFY UNLESS SURE ABOUT CHANGE ##
+
+metadata.GraphService.impl=org.apache.hadoop.metadata.services.TitanGraphService
+metadata.MetadataRepositoryService.impl=org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
+
+#########  Implementation classes  #########
+
+
+#########  Graph Database Configs  #########
 # Graph implementation
 #metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
 
@@ -33,4 +43,12 @@ metadata.graph.index.search.directory=target/data/es
 metadata.graph.index.search.elasticsearch.client-only=false
 metadata.graph.index.search.elasticsearch.local-mode=true
 
+#########  Graph Database Configs  #########
+
+
+#########  Security Properties  #########
+
+# SSL config
 metadata.enableTLS=false
+
+#########  Security Properties  #########

webapp/src/test/java/org/apache/hadoop/metadata/GraphRepositoryServiceIntegrationTest.java → webapp/src/test/java/org/apache/hadoop/metadata/GraphRepositoryServiceIT.java

@@ -12,7 +12,7 @@ import org.testng.annotations.Test;
 import java.util.HashMap;
 import java.util.Map;
 
-public class GraphRepositoryServiceIntegrationTest {
+public class GraphRepositoryServiceIT {
 
     private static final String ENTITY_NAME = "clicks-table";
     private static final String ENTITY_TYPE = "hive-table";
@@ -48,6 +48,7 @@ public class GraphRepositoryServiceIntegrationTest {
         Assert.assertNotNull(guid);
 
         String entity = repositoryService.getEntityDefinition(ENTITY_NAME, ENTITY_TYPE);
+        @SuppressWarnings("unchecked")
         Map<String, String> entityProperties =
                 (Map<String, String>) JSONValue.parseWithException(entity);
         Assert.assertEquals(entityProperties.get("guid"), guid);

webapp/src/test/java/org/apache/hadoop/metadata/TitanGraphServiceIntegrationTest.java → webapp/src/test/java/org/apache/hadoop/metadata/TitanGraphServiceIT.java

@@ -13,7 +13,7 @@ import java.util.UUID;
 /**
  * End to end graph put/get test.
  */
-public class TitanGraphServiceIntegrationTest {
+public class TitanGraphServiceIT {
 
     @Test
     public void testTitanGraph() throws Exception {

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java

+package org.apache.hadoop.metadata.web.resources;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import org.testng.annotations.BeforeClass;
+
+import javax.ws.rs.core.UriBuilder;
+
+public class BaseResourceIT {
+
+    protected WebResource service;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        String baseUrl = "http://localhost:21000/";
+
+        DefaultClientConfig config = new DefaultClientConfig();
+        Client client = Client.create(config);
+        client.resource(UriBuilder.fromUri(baseUrl).build());
+
+        service = client.resource(UriBuilder.fromUri(baseUrl).build());
+    }
+}

webapp/src/test/java/org/apache/hadoop/metadata/MetadataJerseyIntegrationTest.java → webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java

@@ -16,85 +16,117 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.metadata;
+package org.apache.hadoop.metadata.web.resources;
 
-import com.sun.jersey.api.client.Client;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
 import org.json.simple.JSONValue;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.core.Response;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.UUID;
 
-public class MetadataJerseyIntegrationTest {
+/**
+ * Integration tests for Entity Jersey Resource.
+ */
+public class EntityJerseyResourceIT extends BaseResourceIT {
 
-    @Test (enabled = false)
-    public void testMetadataRESTAPI() throws Exception {
-        String baseUrl = "http://localhost:15000/";
+    private static final String ENTITY_NAME = "clicks-table";
+    private static final String ENTITY_TYPE = "hive-table";
+    private static final String DATABASE_NAME = "ads";
+    private static final String TABLE_NAME = "clicks-table";
 
-        DefaultClientConfig config = new DefaultClientConfig();
-        Client client = Client.create(config);
-        client.resource(UriBuilder.fromUri(baseUrl).build());
+    @Test
+    public void testSubmitEntity() {
+        String entityStream = getTestEntityJSON();
 
-        WebResource service = client.resource(UriBuilder.fromUri(baseUrl).build());
+        WebResource resource = service
+                .path("api/metadata/entities/submit")
+                .path(ENTITY_TYPE);
 
-/*
-        ClientResponse clientResponse = service.path("api/metadata/entities/list/blah")
+        ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
-                .method(HttpMethod.GET, ClientResponse.class);
+                .method(HttpMethod.POST, ClientResponse.class, entityStream);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
         String response = clientResponse.getEntity(String.class);
-        System.out.println("response = " + response);
-*/
-
+        Assert.assertNotNull(response);
 
-//        String filePath = "/tmp/metadata/sampleentity.json";
-//        InputStream entityStream = getServletInputStream(filePath);
-
-        final String entityName = "clicks-table";
-        final String entityType = "hive-table";
-
-        submitEntity(service, entityName, entityType);
+        try {
+            Assert.assertNotNull(UUID.fromString(response));
+        } catch (IllegalArgumentException e) {
+            Assert.fail("Response is not a guid, " + response);
+        }
+    }
 
+    @Test (dependsOnMethods = "testSubmitEntity")
+    public void testGetEntityDefinition() {
         WebResource resource = service
                 .path("api/metadata/entities/definition")
-                .path(entityType)
-                .path(entityName);
+                .path(ENTITY_TYPE)
+                .path(ENTITY_NAME);
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
         String response = clientResponse.getEntity(String.class);
         System.out.println("response = " + response);
     }
 
-    private static void submitEntity(WebResource service, String entityName, String entityType) {
-        String entityStream = getTestEntityJSON(entityName, entityType);
+    private static String getTestEntityJSON() {
+        Map<String, String> props = new HashMap<>();
+        props.put("entityName", ENTITY_NAME);
+        props.put("entityType", ENTITY_TYPE);
+        props.put("database", DATABASE_NAME);
+        props.put("table", TABLE_NAME);
+        return JSONValue.toJSONString(props);
+    }
 
+    @Test
+    public void testGetInvalidEntityDefinition() {
         WebResource resource = service
-                .path("api/metadata/entities/submit")
-                .path(entityType);
+                .path("api/metadata/entities/definition")
+                .path(ENTITY_TYPE)
+                .path("blah");
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
-                .method(HttpMethod.POST, ClientResponse.class, entityStream);
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
         String response = clientResponse.getEntity(String.class);
         System.out.println("response = " + response);
     }
 
-    private static String getTestEntityJSON(String entityName, String entityType) {
-        Map<String, String> props = new HashMap<>();
-        props.put("entityName", entityName);
-        props.put("entityType", entityType);
-        props.put("database", "foo");
-        props.put("blah", "blah");
-        return JSONValue.toJSONString(props);
+    @Test (dependsOnMethods = "testSubmitEntity")
+    public void testGetEntityList() {
+        ClientResponse clientResponse = service
+                .path("api/metadata/entities/list/")
+                .path(ENTITY_TYPE)
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+        String response = clientResponse.getEntity(String.class);
+        System.out.println("response = " + response);
+    }
+
+    @Test (enabled = false) // todo: enable this later
+    public void testGetEntityListForBadEntityType() {
+        ClientResponse clientResponse = service
+                .path("api/metadata/entities/list/blah")
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
+        String response = clientResponse.getEntity(String.class);
+        System.out.println("response = " + response);
     }
 }

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/RexsterGraphJerseyResourceIT.java

+package org.apache.hadoop.metadata.web.resources;
+
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+/**
+ * Integration tests for Rexster Graph Jersey Resource.
+ */
+public class RexsterGraphJerseyResourceIT extends BaseResourceIT {
+
+    @Test (enabled = false)
+    public void testGetVertex() throws Exception {
+        // todo: add a vertex before fetching it
+
+        WebResource resource = service
+                .path("api/metadata/graph/vertices")
+                .path("0");
+
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+        String response = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(response);
+    }
+
+    @Test
+    public void testGetVertexWithInvalidId() throws Exception {
+        WebResource resource = service
+                .path("api/metadata/graph/vertices/blah");
+
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
+    }
+
+    @Test
+    public void testGetVertexProperties() throws Exception {
+
+    }
+
+    @Test
+    public void testGetVertices() throws Exception {
+
+    }
+
+    @Test
+    public void testGetVertexEdges() throws Exception {
+
+    }
+
+    @Test
+    public void testGetEdge() throws Exception {
+
+    }
+}