ISSUE-4 Add unit tests for the repository services. Contributed by Venkatesh Seetharam

common/src/test/resources/log4j.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+    <appender name="console" class="org.apache.log4j.ConsoleAppender">
+        <param name="Target" value="System.out"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+        </layout>
+    </appender>
+
+    <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="${user.dir}/target/logs/audit.log"/>
+        <param name="Append" value="true"/>
+        <param name="Threshold" value="debug"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %x %m%n"/>
+        </layout>
+    </appender>
+
+    <appender name="METRIC" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="${user.dir}/target/logs/metric.log"/>
+        <param name="Append" value="true"/>
+        <param name="Threshold" value="debug"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %m%n"/>
+        </layout>
+    </appender>
+
+    <logger name="AUDIT" additivity="false">
+        <level value="debug"/>
+        <appender-ref ref="console"/>
+    </logger>
+
+    <logger name="METRIC" additivity="false">
+        <level value="debug"/>
+        <appender-ref ref="console"/>
+    </logger>
+
+    <logger name="org.apache.hadoop.metadata" additivity="false">
+        <level value="debug"/>
+        <appender-ref ref="console"/>
+    </logger>
+
+    <root>
+        <priority value="debug"/>
+        <appender-ref ref="console"/>
+    </root>
+
+</log4j:configuration>

repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryService.java

@@ -62,8 +62,9 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
      */
     @Override
     public void start() throws Exception {
-        graphService = Services.get().getService(TitanGraphService.NAME);
-        if (graphService == null) {
+        if (Services.get().isRegistered(TitanGraphService.NAME)) {
+            graphService = Services.get().getService(TitanGraphService.NAME);
+        } else {
             throw new RuntimeException("graph service is not initialized");
         }
     }

repository/src/main/resources/log4j.xml

@@ -54,13 +54,23 @@
         </layout>
     </appender>
 
+    <logger name="AUDIT" additivity="false">
+        <level value="debug"/>
+        <appender-ref ref="console"/>
+    </logger>
+
+    <logger name="METRIC" additivity="false">
+        <level value="debug"/>
+        <appender-ref ref="console"/>
+    </logger>
+
     <logger name="org.apache.hadoop.metadata" additivity="false">
         <level value="debug"/>
         <appender-ref ref="FILE"/>
     </logger>
 
     <root>
-        <priority value="info"/>
+        <priority value="debug"/>
         <appender-ref ref="console"/>
     </root>
 

repository/src/test/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryServiceTest.java

+package org.apache.hadoop.metadata.services;
+
+import org.apache.hadoop.metadata.service.Services;
+import org.json.simple.JSONValue;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class GraphBackedMetadataRepositoryServiceTest {
+
+    private static final String ENTITY_NAME = "clicks-table";
+    private static final String ENTITY_TYPE = "hive-table";
+    private static final String DATABASE_NAME = "ads";
+    private static final String TABLE_NAME = "clicks-table";
+
+    private TitanGraphService titanGraphService;
+    private GraphBackedMetadataRepositoryService repositoryService;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        titanGraphService = new TitanGraphService();
+        titanGraphService.start();
+        Services.get().register(titanGraphService);
+
+        repositoryService = new GraphBackedMetadataRepositoryService();
+        repositoryService.start();
+        Services.get().register(repositoryService);
+    }
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        Services.get().getService(GraphBackedMetadataRepositoryService.NAME).close();
+        Services.get().getService(TitanGraphService.NAME).close();
+        Services.get().reset();
+    }
+
+    @Test
+    public void testGetName() throws Exception {
+        Assert.assertEquals(GraphBackedMetadataRepositoryService.NAME,
+                GraphBackedMetadataRepositoryService.class.getSimpleName());
+        Assert.assertEquals(repositoryService.getName(), GraphBackedMetadataRepositoryService.NAME);
+    }
+
+    @Test
+    public void testSubmitEntity() throws Exception {
+        String entityStream = getTestEntityJSON();
+        String guid = repositoryService.submitEntity(entityStream, ENTITY_TYPE);
+        Assert.assertNotNull(guid);
+    }
+
+    private String getTestEntityJSON() {
+        Map<String, String> props = new HashMap<>();
+        props.put("entityName", ENTITY_NAME);
+        props.put("entityType", ENTITY_TYPE);
+        props.put("database", DATABASE_NAME);
+        props.put("table", TABLE_NAME);
+        return JSONValue.toJSONString(props);
+    }
+
+    @Test (dependsOnMethods = "testSubmitEntity")
+    public void testGetEntityDefinition() throws Exception {
+        String entity = repositoryService.getEntityDefinition(ENTITY_NAME, ENTITY_TYPE);
+        Map<String, String> entityProperties =
+                (Map<String, String>) JSONValue.parseWithException(entity);
+        Assert.assertNotNull(entityProperties.get("guid"));
+        Assert.assertEquals(entityProperties.get("entityName"), ENTITY_NAME);
+        Assert.assertEquals(entityProperties.get("entityType"), ENTITY_TYPE);
+        Assert.assertEquals(entityProperties.get("database"), DATABASE_NAME);
+        Assert.assertEquals(entityProperties.get("table"), TABLE_NAME);
+    }
+
+    @Test
+    public void testGetEntityDefinitionNonExistent() throws Exception {
+        String entity = repositoryService.getEntityDefinition("blah", "blah");
+        Assert.assertNull(entity);
+    }
+
+    @Test
+    public void testGetEntityList() throws Exception {
+        List<String> entityList = repositoryService.getEntityList(ENTITY_TYPE);
+        Assert.assertNotNull(entityList);
+        Assert.assertEquals(entityList.size(), 0); // as this is not implemented yet
+    }
+
+    @Test (expectedExceptions = RuntimeException.class)
+    public void testStartWithOutGraphServiceRegistration() throws Exception {
+        try {
+            Services.get().reset();
+            GraphBackedMetadataRepositoryService repositoryService = new
+                    GraphBackedMetadataRepositoryService();
+            repositoryService.start();
+            Assert.fail("This should have thrown an exception");
+        } finally {
+            Services.get().register(titanGraphService);
+            Services.get().register(repositoryService);
+        }
+    }
+}

repository/src/test/java/org/apache/hadoop/metadata/services/TitanGraphServiceTest.java

+package org.apache.hadoop.metadata.services;
+
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test for TitanGraphService.
+ */
+public class TitanGraphServiceTest {
+
+    private TitanGraphService titanGraphService;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        titanGraphService = new TitanGraphService();
+        titanGraphService.start();
+    }
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        titanGraphService.close();
+    }
+
+    @Test
+    public void testGetName() throws Exception {
+        Assert.assertEquals(TitanGraphService.NAME, TitanGraphService.class.getSimpleName());
+        Assert.assertEquals(titanGraphService.getName(), TitanGraphService.NAME);
+    }
+
+    @Test
+    public void testStart() throws Exception {
+        Assert.assertNotNull(titanGraphService.getBlueprintsGraph());
+    }
+
+    @Test
+    public void testGetBlueprintsGraph() throws Exception {
+        Assert.assertNotNull(titanGraphService.getBlueprintsGraph());
+    }
+
+    @Test
+    public void testGetIndexableGraph() throws Exception {
+        Assert.assertNotNull(titanGraphService.getIndexableGraph());
+    }
+
+    @Test
+    public void testGetTransactionalGraph() throws Exception {
+        Assert.assertNotNull(titanGraphService.getTransactionalGraph());
+    }
+
+    @Test
+    public void testGetTitanGraph() throws Exception {
+        Assert.assertNotNull(titanGraphService.getTitanGraph());
+    }
+
+    @Test
+    public void testGetVertexIndexedKeys() throws Exception {
+        Assert.assertNotNull(titanGraphService.getVertexIndexedKeys());
+        Assert.assertEquals(titanGraphService.getVertexIndexedKeys().size(), 0);
+    }
+
+    @Test
+    public void testGetEdgeIndexedKeys() throws Exception {
+        Assert.assertNotNull(titanGraphService.getEdgeIndexedKeys());
+        Assert.assertEquals(titanGraphService.getEdgeIndexedKeys().size(), 0);
+    }
+}
\ No newline at end of file

repository/src/test/resources/application.properties

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+application.services=org.apache.hadoop.metadata.services.TitanGraphService,\
+                     org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
+
+
+# Graph implementation
+#metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
+
+# Graph Storage
+metadata.graph.storage.backend=berkeleyje
+metadata.graph.storage.directory=target/data/berkeley
+
+# Graph Search Index
+metadata.graph.index.search.backend=elasticsearch
+metadata.graph.index.search.directory=target/data/es
+metadata.graph.index.search.elasticsearch.client-only=false
+metadata.graph.index.search.elasticsearch.local-mode=true
+
+metadata.enableTLS=false

webapp/src/test/java/org/apache/hadoop/metadata/GraphRepositoryServiceIntegrationTest.java

+package org.apache.hadoop.metadata;
+
+import org.apache.hadoop.metadata.service.Services;
+import org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService;
+import org.apache.hadoop.metadata.services.TitanGraphService;
+import org.json.simple.JSONValue;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class GraphRepositoryServiceIntegrationTest {
+
+    private static final String ENTITY_NAME = "clicks-table";
+    private static final String ENTITY_TYPE = "hive-table";
+    private static final String DATABASE_NAME = "ads";
+    private static final String TABLE_NAME = "clicks-table";
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        TitanGraphService titanGraphService = new TitanGraphService();
+        titanGraphService.start();
+        Services.get().register(titanGraphService);
+
+        GraphBackedMetadataRepositoryService repositoryService
+                = new GraphBackedMetadataRepositoryService();
+        repositoryService.start();
+        Services.get().register(repositoryService);
+    }
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        Services.get().getService(GraphBackedMetadataRepositoryService.NAME).close();
+        Services.get().getService(TitanGraphService.NAME).close();
+        Services.get().reset();
+    }
+
+    @Test
+    public void testRepository() throws Exception {
+        GraphBackedMetadataRepositoryService repositoryService =
+                Services.get().getService(GraphBackedMetadataRepositoryService.NAME);
+
+        String entityStream = getTestEntityJSON();
+        String guid = repositoryService.submitEntity(entityStream, ENTITY_TYPE);
+        Assert.assertNotNull(guid);
+
+        String entity = repositoryService.getEntityDefinition(ENTITY_NAME, ENTITY_TYPE);
+        Map<String, String> entityProperties =
+                (Map<String, String>) JSONValue.parseWithException(entity);
+        Assert.assertEquals(entityProperties.get("guid"), guid);
+        Assert.assertEquals(entityProperties.get("entityName"), ENTITY_NAME);
+        Assert.assertEquals(entityProperties.get("entityType"), ENTITY_TYPE);
+        Assert.assertEquals(entityProperties.get("database"), DATABASE_NAME);
+        Assert.assertEquals(entityProperties.get("table"), TABLE_NAME);
+    }
+
+    private String getTestEntityJSON() {
+        Map<String, String> props = new HashMap<>();
+        props.put("entityName", ENTITY_NAME);
+        props.put("entityType", ENTITY_TYPE);
+        props.put("database", DATABASE_NAME);
+        props.put("table", TABLE_NAME);
+        return JSONValue.toJSONString(props);
+    }
+}

webapp/src/test/java/org/apache/hadoop/metadata/TestDriver.java → webapp/src/test/java/org/apache/hadoop/metadata/MetadataJerseyIntegrationTest.java

@@ -23,6 +23,7 @@ import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
 import org.json.simple.JSONValue;
+import org.testng.annotations.Test;
 
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
@@ -30,9 +31,10 @@ import javax.ws.rs.core.UriBuilder;
 import java.util.HashMap;
 import java.util.Map;
 
-public class TestDriver {
+public class MetadataJerseyIntegrationTest {
 
-    public static void main(String[] args) throws Exception {
+    @Test (enabled = false)
+    public void testMetadataRESTAPI() throws Exception {
         String baseUrl = "http://localhost:15000/";
 
         DefaultClientConfig config = new DefaultClientConfig();
@@ -73,13 +75,7 @@ public class TestDriver {
     }
 
     private static void submitEntity(WebResource service, String entityName, String entityType) {
-        Map<String, String> props = new HashMap<>();
-        props.put("entityName", entityName);
-        props.put("entityType", entityType);
-        props.put("database", "foo");
-        props.put("blah", "blah");
-        String entityStream = JSONValue.toJSONString(props);
-
+        String entityStream = getTestEntityJSON(entityName, entityType);
 
         WebResource resource = service
                 .path("api/metadata/entities/submit")
@@ -92,4 +88,13 @@ public class TestDriver {
         String response = clientResponse.getEntity(String.class);
         System.out.println("response = " + response);
     }
+
+    private static String getTestEntityJSON(String entityName, String entityType) {
+        Map<String, String> props = new HashMap<>();
+        props.put("entityName", entityName);
+        props.put("entityType", entityType);
+        props.put("database", "foo");
+        props.put("blah", "blah");
+        return JSONValue.toJSONString(props);
+    }
 }

webapp/src/test/java/org/apache/hadoop/metadata/TitanGraphServiceIntegrationTest.java

+package org.apache.hadoop.metadata;
+
+import com.tinkerpop.blueprints.GraphQuery;
+import com.tinkerpop.blueprints.TransactionalGraph;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.hadoop.metadata.services.TitanGraphService;
+import org.apache.hadoop.metadata.util.GraphUtils;
+import org.testng.annotations.Test;
+
+import java.util.Iterator;
+import java.util.UUID;
+
+/**
+ * End to end graph put/get test.
+ */
+public class TitanGraphServiceIntegrationTest {
+
+    @Test
+    public void testTitanGraph() throws Exception {
+        TitanGraphService titanGraphService = new TitanGraphService();
+        titanGraphService.start();
+
+        try {
+            String guid = UUID.randomUUID().toString();
+
+            final TransactionalGraph graph = titanGraphService.getTransactionalGraph();
+            System.out.println("graph = " + graph);
+            System.out.println("graph.getVertices() = " + graph.getVertices());
+
+
+            Vertex entityVertex = null;
+            try {
+                graph.rollback();
+                entityVertex = graph.addVertex(null);
+                entityVertex.setProperty("guid", guid);
+                entityVertex.setProperty("entityName", "entityName");
+                entityVertex.setProperty("entityType", "entityType");
+            } catch (Exception e) {
+                graph.rollback();
+                e.printStackTrace();
+            } finally {
+                graph.commit();
+            }
+
+            System.out.println("vertex = " + GraphUtils.vertexString(entityVertex));
+
+            GraphQuery query = graph.query()
+                    .has("entityName", "entityName")
+                    .has("entityType", "entityType");
+
+            Iterator<Vertex> results = query.vertices().iterator();
+            if (results.hasNext()) {
+                Vertex vertexFromQuery = results.next();
+                System.out.println("vertex = " + GraphUtils.vertexString(vertexFromQuery));
+            }
+        } finally {
+            Thread.sleep(1000);
+            titanGraphService.stop();
+        }
+    }
+}