ISSUE-42 Add search REST API for gremlin query. Contributed by Venkatesh Seetharam

repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java

@@ -28,6 +28,8 @@ package org.apache.hadoop.metadata;
 import com.google.inject.Scopes;
 import com.google.inject.throwingproviders.ThrowingProviderBinder;
 import com.thinkaurelius.titan.core.TitanGraph;
+import org.apache.hadoop.metadata.discovery.DiscoveryService;
+import org.apache.hadoop.metadata.discovery.GraphBackedDiscoveryService;
 import org.apache.hadoop.metadata.services.DefaultMetadataService;
 import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
 import org.apache.hadoop.metadata.repository.graph.GraphProvider;
@@ -44,9 +46,11 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
 
 	// Graph Service implementation class
 	private Class<? extends GraphService> graphServiceClass;
+
 	// MetadataRepositoryService implementation class
 	private Class<? extends MetadataRepository> metadataRepoClass;
 	private Class<? extends MetadataService> metadataService;
+	private Class<? extends DiscoveryService> discoveryService;
 
 	public RepositoryMetadataModule() {
 		GraphServiceConfigurator gsp = new GraphServiceConfigurator();
@@ -55,6 +59,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
 		this.graphServiceClass = gsp.getImplClass();
 		this.metadataRepoClass = GraphBackedMetadataRepository.class;
 		this.metadataService = DefaultMetadataService.class;
+		this.discoveryService = GraphBackedDiscoveryService.class;
 	}
 
 	protected void configure() {
@@ -74,5 +79,8 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
 
 		// bind the MetadataService interface to an implementation
 		bind(MetadataService.class).to(metadataService);
+
+		// bind the DiscoveryService interface to an implementation
+		bind(DiscoveryService.class).to(discoveryService);
 	}
 }

repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryService.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metadata.discovery;
+
+import org.apache.hadoop.metadata.MetadataException;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Metadata discovery service.
+ */
+public interface DiscoveryService {
+
+    /**
+     * Assumes the User is familiar with the persistence structure of the Repository.
+     * The given query is run uninterpreted against the underlying Graph Store.
+     * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
+     *
+     * @param gremlinQuery query in gremlin dsl format
+     * @return List of Maps
+     * @throws org.apache.hadoop.metadata.MetadataException
+     */
+    List<Map<String,String>> searchByGremlin(String gremlinQuery) throws MetadataException;
+
+}

repository/src/main/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryService.java

+package org.apache.hadoop.metadata.discovery;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.metadata.MetadataException;
+import org.apache.hadoop.metadata.repository.MetadataRepository;
+
+import javax.inject.Inject;
+import java.util.List;
+import java.util.Map;
+
+public class GraphBackedDiscoveryService implements DiscoveryService {
+
+    private final MetadataRepository repository;
+
+    @Inject
+    GraphBackedDiscoveryService(MetadataRepository repository) throws MetadataException {
+        this.repository = repository;
+    }
+
+    /**
+     * Assumes the User is familiar with the persistence structure of the Repository.
+     * The given query is run uninterpreted against the underlying Graph Store.
+     * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
+     *
+     * @param gremlinQuery query in gremlin dsl format
+     * @return List of Maps
+     * @throws org.apache.hadoop.metadata.MetadataException
+     */
+    @Override
+    public List<Map<String, String>> searchByGremlin(String gremlinQuery) throws MetadataException {
+        Preconditions.checkNotNull(gremlinQuery, "gremlin query name cannot be null");
+        // simple pass-through
+        return repository.searchByGremlin(gremlinQuery);
+    }
+}

repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.metadata.repository;
 
 import org.apache.hadoop.metadata.IReferenceableInstance;
 import org.apache.hadoop.metadata.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.service.Service;
 import org.apache.hadoop.metadata.storage.RepositoryException;
 
@@ -39,12 +40,13 @@ public interface MetadataRepository extends Service {
     List<String> getEntityList(String entityType) throws RepositoryException;
 
     /**
-     * Assumes the User is familar with the persistence structure of the Repository.
+     * Assumes the User is familiar with the persistence structure of the Repository.
      * The given query is run uninterpreted against the underlying Graph Store.
      * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
-     * @param gremlinQuery
-     * @return
-     * @throws RepositoryException
+     *
+     * @param gremlinQuery query in gremlin dsl format
+     * @return List of Maps
+     * @throws org.apache.hadoop.metadata.MetadataException
      */
-    List<Map<String,String>> rawSearch(String gremlinQuery) throws RepositoryException;
+    List<Map<String,String>> searchByGremlin(String gremlinQuery) throws MetadataException;
 }

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java

@@ -26,7 +26,6 @@ import com.tinkerpop.blueprints.Graph;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.TransactionalGraph;
 import com.tinkerpop.blueprints.Vertex;
-import org.apache.commons.collections.map.HashedMap;
 import org.apache.hadoop.metadata.IReferenceableInstance;
 import org.apache.hadoop.metadata.ITypedInstance;
 import org.apache.hadoop.metadata.ITypedReferenceableInstance;
@@ -182,8 +181,17 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         return entityList;
     }
 
+    /**
+     * Assumes the User is familiar with the persistence structure of the Repository.
+     * The given query is run uninterpreted against the underlying Graph Store.
+     * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
+     *
+     * @param gremlinQuery query in gremlin dsl format
+     * @return List of Maps
+     * @throws org.apache.hadoop.metadata.MetadataException
+     */
     @Override
-    public List<Map<String,String>> rawSearch(String gremlinQuery) throws RepositoryException {
+    public List<Map<String, String>> searchByGremlin(String gremlinQuery) throws MetadataException {
         ScriptEngineManager manager = new ScriptEngineManager();
         ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
         Bindings bindings = engine.createBindings();
@@ -199,7 +207,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             List<Map<String,String>> result = new ArrayList<>();
             for(Object r : l) {
 
-                Map<String,String> oRow = new HashedMap();
+                Map<String,String> oRow = new HashMap<>();
                 if ( r instanceof  Map ) {
                     Map<Object,Object> iRow = (Map) r;
                     for(Map.Entry e : iRow.entrySet()) {
@@ -450,8 +458,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                     structInstance.fieldMapping().fields, idToVertexMap);
 
             // add an edge to the newly created vertex from the parent
-            String relationshipLabel = attributeInfo.dataType().getName() + "." + attributeInfo.name;
-            LOG.debug("Adding edge for {} -> label {} -> v{}",
+            String relationshipLabel = typedInstance.getTypeName() + "." + attributeInfo.name;
+            LOG.debug("Adding edge for {} -> struct label {} -> v{}",
                     parentInstanceVertex, relationshipLabel, structInstanceVertex);
             parentInstanceVertex.addEdge(relationshipLabel, structInstanceVertex);
         }
@@ -471,7 +479,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
 
             // add an edge to the newly created vertex from the parent
             String relationshipLabel = typedInstance.getTypeName() + "." + traitName;
-            LOG.debug("Adding edge for {} -> label {} -> v{}",
+            LOG.debug("Adding edge for {} -> trait label {} -> v{}",
                     parentInstanceVertex, relationshipLabel, traitInstanceVertex);
             parentInstanceVertex.addEdge(relationshipLabel, traitInstanceVertex);
         }
@@ -595,6 +603,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                         break;
 
                     case CLASS:
+                        // todo - use ObjectWalker here instead else it can be an infinite loop
+                        // for cross references
                         String relationshipLabel = typedInstance.getTypeName() + "." + attributeInfo.name;
                         LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
                         for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanBootstrap.java

@@ -20,8 +20,10 @@ package org.apache.hadoop.metadata.repository.graph;
 
 import com.thinkaurelius.titan.core.TitanFactory;
 import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.attribute.Cmp;
 import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
 
 public class TitanBootstrap {
@@ -78,6 +80,21 @@ public class TitanBootstrap {
                     System.out.println("e = " + edgeString(e));
                 }
             }
+
+            System.out.println("====================");
+            GraphQuery graphQuery = graph.query()
+                    .has("name", Cmp.EQUAL, "harish");
+            for (Vertex v : graphQuery.vertices()) {
+                System.out.println("v = " + vertexString(v));
+            }
+
+            graphQuery = graph.query()
+                    .has("name", Cmp.EQUAL, "venkatesh");
+            for (Edge e : graphQuery.edges()) {
+                System.out.println("e = " + edgeString(e));
+            }
+
+
         } finally {
             graph.shutdown();
         }

repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java

@@ -98,9 +98,9 @@ public class DefaultMetadataService implements MetadataService {
         Preconditions.checkNotNull(typeDefinition, "type definition cannot be null");
 
         // verify if the type already exists
-        String existingTypeDefinition = null;
+        IDataType existingTypeDefinition = null;
         try {
-            existingTypeDefinition = getTypeDefinition(typeName);
+            existingTypeDefinition = typeSystem.getDataType(IDataType.class, typeName);
         } catch (MetadataException ignore) {
             // do nothing
         }

repository/src/test/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryTest.java

@@ -113,15 +113,15 @@ public class GraphBackedMetadataRepositoryTest extends RepositoryModuleBaseTest
 
 
         // Query for all Vertices in Graph
-        Object r = repositoryService.rawSearch("g.V.toList()");
+        Object r = repositoryService.searchByGremlin("g.V.toList()");
         //System.out.println(r);
 
         // Query for all Vertices of a Type
-        r = repositoryService.rawSearch("g.V.filter{it.typeName == 'Department'}.toList()");
+        r = repositoryService.searchByGremlin("g.V.filter{it.typeName == 'Department'}.toList()");
         //System.out.println(r);
 
         // Property Query: list all Person names
-        r = repositoryService.rawSearch("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
+        r = repositoryService.searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
         //System.out.println(r);
     }
 

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java

@@ -18,8 +18,27 @@
 
 package org.apache.hadoop.metadata.web.resources;
 
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.metadata.MetadataException;
+import org.apache.hadoop.metadata.discovery.DiscoveryService;
+import org.apache.hadoop.metadata.web.util.Servlets;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
 import javax.inject.Singleton;
+import javax.ws.rs.GET;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Jersey Resource for metadata operations.
@@ -33,4 +52,49 @@ import javax.ws.rs.Path;
 @Path("discovery")
 @Singleton
 public class MetadataDiscoveryResource {
+
+    private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
+
+    private final DiscoveryService discoveryService;
+
+    /**
+     * Created by the Guice ServletModule and injected with the
+     * configured DiscoveryService.
+     *
+     * @param discoveryService metadata service handle
+     */
+    @Inject
+    public MetadataDiscoveryResource(DiscoveryService discoveryService) {
+        this.discoveryService = discoveryService;
+    }
+
+    @GET
+    @Path("search/gremlin/{gremlinQuery}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response searchUsingGremlinQuery(@PathParam("gremlinQuery") String gremlinQuery) {
+        Preconditions.checkNotNull(gremlinQuery, "gremlinQuery cannot be null");
+
+        try {
+            final List<Map<String,String>> results = discoveryService.searchByGremlin(gremlinQuery);
+
+            JSONObject response = new JSONObject();
+            response.put("requestId", Thread.currentThread().getName());
+
+            JSONArray list = new JSONArray();
+            for (Map<String, String> result : results) {
+                list.put(new JSONObject(result));
+            }
+            response.put("results", list);
+
+            return Response.ok(response).build();
+        } catch (MetadataException e) {
+            LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+        } catch (JSONException e) {
+            LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+        }
+    }
 }

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java

@@ -73,13 +73,6 @@ public class RexsterGraphResource {
     @Inject
     public RexsterGraphResource(GraphService graphService) {
     	this.graphService = graphService;
-        /*graphService = Services.get().getService(TitanGraphService.NAME);
-        if (graphService == null) {
-            throw new WebApplicationException(Response
-                    .status(Response.Status.INTERNAL_SERVER_ERROR)
-                    .tag("graph service is not initialized")
-                    .build());
-        }*/
     }
 
     protected Graph getGraph() {
@@ -148,8 +141,7 @@ public class RexsterGraphResource {
         try {
             Vertex vertex = findVertex(vertexId);
 
-            Map<String, String> vertexProperties =
-                    getVertexProperties(vertex, Boolean.valueOf(relationships));
+            Map<String, String> vertexProperties = getVertexProperties(vertex);
 
             JSONObject response = new JSONObject();
             response.put(RESULTS, new JSONObject(vertexProperties));
@@ -161,7 +153,7 @@ public class RexsterGraphResource {
         }
     }
 
-    private Map<String, String> getVertexProperties(Vertex vertex, boolean captureRelationships) {
+    private Map<String, String> getVertexProperties(Vertex vertex) {
         Map<String, String> vertexProperties = new HashMap<>();
         for (String key : vertex.getPropertyKeys()) {
             vertexProperties.put(key, vertex.<String>getProperty(key));