Merge branch 'master' of https://github.com/hortonworks/metadata into BUG-32830

docs/src/site/twiki/Configuration.twiki

+---+ Configuring Apache Atlas
+
+---++ Introduction
+
+All configuration in Atlas uses java properties style configuration.
+
+---++ Application Properties
+
+The main configuration file is application.properties which is in the *conf* dir at the deployed
+location. It consists of the following sections:
+
+---+++ Graph Database Configs
+
+---++++ Graph persistence engine
+
+This section sets up the graph db - titan - to use a persistence engine. Please refer to
+<a href="http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html">link</a> for more
+details. The example below uses BerkeleyDBJE.
+
+<verbatim>
+metadata.graph.storage.backend=berkeleyje
+metadata.graph.storage.directory=data/berkley
+</verbatim>
+
+---++++ Graph Search Index
+This section sets up the graph db - titan - to use an search indexing system. The example
+configuration below setsup to use an embedded Elastic search indexing system.
+
+<verbatim>
+metadata.graph.index.search.backend=elasticsearch
+metadata.graph.index.search.directory=data/es
+metadata.graph.index.search.elasticsearch.client-only=false
+metadata.graph.index.search.elasticsearch.local-mode=true
+metadata.graph.index.search.elasticsearch.create.sleep=2000
+</verbatim>
+
+---+++ Hive Lineage Configs
+The higher layer services like hive lineage, schema, etc. are driven by the type system and this
+section encodes the specific types for the hive data model.
+
+# This models follows the quick-start guide
+<verbatim>
+metadata.lineage.hive.table.type.name=hive_table
+metadata.lineage.hive.table.column.name=columns
+metadata.lineage.hive.process.type.name=hive_process
+metadata.lineage.hive.process.inputs.name=inputTables
+metadata.lineage.hive.process.outputs.name=outputTables
+#Currently unused
+#metadata.lineage.hive.column.type.name=Column
+</verbatim>
+
+---+++ Security Properties
+
+---++++ SSL config
+The following property is used to toggle the SSL feature.
+
+<verbatim>
+metadata.enableTLS=false
+</verbatim>
+

repository/src/main/java/org/apache/hadoop/metadata/discovery/HiveLineageService.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.metadata.discovery;
 
 import com.thinkaurelius.titan.core.TitanGraph;
-import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.PropertiesUtil;
@@ -116,6 +115,24 @@ public class HiveLineageService implements LineageService {
     }
 
     /**
+     * Return the lineage outputs graph for the given tableName.
+     *
+     * @param tableName tableName
+     * @return Outputs Graph as JSON
+     */
+    @Override
+    public String getOutputsGraph(String tableName) throws DiscoveryException {
+        LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
+
+        HiveWhereUsedQuery outputsQuery = new HiveWhereUsedQuery(
+                HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
+                HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
+                Option.empty(), SELECT_ATTRIBUTES, true,
+                graphPersistenceStrategy, titanGraph);
+        return outputsQuery.graph().toInstanceJson();
+    }
+
+    /**
      * Return the lineage inputs for the given tableName.
      *
      * @param tableName tableName
@@ -141,6 +158,24 @@ public class HiveLineageService implements LineageService {
     }
 
     /**
+     * Return the lineage inputs graph for the given tableName.
+     *
+     * @param tableName tableName
+     * @return Inputs Graph as JSON
+     */
+    @Override
+    public String getInputsGraph(String tableName) throws DiscoveryException {
+        LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
+
+        HiveLineageQuery inputsQuery = new HiveLineageQuery(
+                HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
+                HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
+                Option.empty(), SELECT_ATTRIBUTES, true,
+                graphPersistenceStrategy, titanGraph);
+        return inputsQuery.graph().toInstanceJson();
+    }
+
+    /**
      * Return the schema for the given tableName.
      *
      * @param tableName tableName

repository/src/main/java/org/apache/hadoop/metadata/discovery/LineageService.java

@@ -32,6 +32,14 @@ public interface LineageService {
     String getOutputs(String tableName) throws DiscoveryException;
 
     /**
+     * Return the lineage outputs graph for the given tableName.
+     *
+     * @param tableName tableName
+     * @return Outputs Graph as JSON
+     */
+    String getOutputsGraph(String tableName) throws DiscoveryException;
+
+    /**
      * Return the lineage inputs for the given tableName.
      *
      * @param tableName tableName
@@ -40,6 +48,14 @@ public interface LineageService {
     String getInputs(String tableName) throws DiscoveryException;
 
     /**
+     * Return the lineage inputs graph for the given tableName.
+     *
+     * @param tableName tableName
+     * @return Inputs Graph as JSON
+     */
+    String getInputsGraph(String tableName) throws DiscoveryException;
+
+    /**
      * Return the schema for the given tableName.
      *
      * @param tableName tableName

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java

@@ -25,6 +25,8 @@ import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.PropertiesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.inject.Singleton;
 import java.util.Iterator;
@@ -34,6 +36,8 @@ import java.util.Iterator;
  */
 public class TitanGraphProvider implements GraphProvider<TitanGraph> {
 
+    private static final Logger LOG = LoggerFactory.getLogger(TitanGraphProvider.class);
+
     /**
      * Constant for the configuration property that indicates the prefix.
      */
@@ -51,6 +55,7 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
                 String value = (String) configProperties.getProperty(key);
                 key = key.substring(METADATA_PREFIX.length());
                 graphConfig.setProperty(key, value);
+                LOG.info("Using graph property {}={}", key, value);
             }
         }
 

repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java

@@ -207,46 +207,53 @@ public class GraphBackedTypeStore implements ITypeStore {
 
     @Override
     public TypesDef restore() throws MetadataException {
-        //Get all vertices for type system
-        Iterator vertices = titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
-
-        ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
-        ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
-        ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
-        ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
-
-        while (vertices.hasNext()) {
-            Vertex vertex = (Vertex) vertices.next();
-            DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
-            String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
-            LOG.info("Restoring type {}.{}", typeCategory, typeName);
-            switch(typeCategory) {
-            case ENUM:
-                enums.add(getEnumType(vertex));
-                break;
-
-            case STRUCT:
-                AttributeDefinition[] attributes = getAttributes(vertex);
-                structs.add(new StructTypeDefinition(typeName, attributes));
-                break;
-
-            case CLASS:
-                ImmutableList<String> superTypes = getSuperTypes(vertex);
-                attributes = getAttributes(vertex);
-                classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, superTypes, attributes));
-                break;
-
-            case TRAIT:
-                superTypes = getSuperTypes(vertex);
-                attributes = getAttributes(vertex);
-                traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, superTypes, attributes));
-                break;
-
-            default:
-                throw new IllegalArgumentException("Unhandled type category " + typeCategory);
+        try {
+            titanGraph.rollback();  //Cleanup previous state
+            //Get all vertices for type system
+            Iterator vertices =
+                    titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
+
+            ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
+            ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
+            ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
+            ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
+
+            while (vertices.hasNext()) {
+                Vertex vertex = (Vertex) vertices.next();
+                DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
+                String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
+                LOG.info("Restoring type {}.{}", typeCategory, typeName);
+                switch (typeCategory) {
+                case ENUM:
+                    enums.add(getEnumType(vertex));
+                    break;
+
+                case STRUCT:
+                    AttributeDefinition[] attributes = getAttributes(vertex);
+                    structs.add(new StructTypeDefinition(typeName, attributes));
+                    break;
+
+                case CLASS:
+                    ImmutableList<String> superTypes = getSuperTypes(vertex);
+                    attributes = getAttributes(vertex);
+                    classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, superTypes, attributes));
+                    break;
+
+                case TRAIT:
+                    superTypes = getSuperTypes(vertex);
+                    attributes = getAttributes(vertex);
+                    traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, superTypes, attributes));
+                    break;
+
+                default:
+                    throw new IllegalArgumentException("Unhandled type category " + typeCategory);
+                }
             }
+            titanGraph.commit();
+            return TypeUtils.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
+        } finally {
+            titanGraph.rollback();
         }
-        return TypeUtils.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
     }
 
     private EnumTypeDefinition getEnumType(Vertex vertex) {

repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.metadata.services;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.inject.Injector;
 import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.discovery.SearchIndexer;
 import org.apache.hadoop.metadata.listener.EntityChangeListener;
@@ -73,9 +74,22 @@ public class DefaultMetadataService implements MetadataService {
         this.typeSystem = TypeSystem.getInstance();
         this.repository = repository;
 
+        restoreTypeSystem();
         registerListener(searchIndexer);
     }
 
+    private void restoreTypeSystem() {
+        LOG.info("Restoring type system from the store");
+        try {
+            TypesDef typesDef = typeStore.restore();
+            typeSystem.defineTypes(typesDef);
+        } catch (MetadataException e) {
+            throw new RuntimeException(e);
+        }
+        LOG.info("Restored type system from the store");
+    }
+
+
     /**
      * Creates a new type based on the type system to enable adding
      * entities (instances for types).

webapp/src/main/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationFilter.java

@@ -44,7 +44,7 @@ import java.util.Properties;
 public class MetadataAuthenticationFilter extends AuthenticationFilter {
     private static final Logger LOG = LoggerFactory.getLogger(MetadataAuthenticationFilter.class);
     static final String PREFIX = "metadata.http.authentication.";
-    static final String BIND_ADDRESS = "bind.address";
+    static final String BIND_ADDRESS = "metadata.server.bind.address";
 
     @Override
     protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException {

webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java

@@ -20,7 +20,11 @@ package org.apache.hadoop.metadata.web.listeners;
 
 import com.google.inject.Guice;
 import com.google.inject.Injector;
+import com.google.inject.TypeLiteral;
+import com.google.inject.matcher.Matchers;
 import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.spi.TypeEncounter;
+import com.google.inject.spi.TypeListener;
 import com.sun.jersey.api.core.PackagesResourceConfig;
 import com.sun.jersey.guice.JerseyServletModule;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
@@ -41,6 +45,8 @@ import javax.servlet.ServletContextEvent;
 import java.util.HashMap;
 import java.util.Map;
 
+import static com.google.inject.matcher.Matchers.*;
+
 public class GuiceServletConfig extends GuiceServletContextListener {
 
     private static final Logger LOG = LoggerFactory.getLogger(GuiceServletConfig.class);
@@ -105,22 +111,6 @@ public class GuiceServletConfig extends GuiceServletContextListener {
         // perform login operations
         LoginProcessor loginProcessor = new LoginProcessor();
         loginProcessor.login();
-
-        restoreTypeSystem();
-    }
-
-    private void restoreTypeSystem() {
-        LOG.info("Restoring type system from the store");
-        Injector injector = getInjector();
-        ITypeStore typeStore = injector.getInstance(ITypeStore.class);
-        try {
-            TypesDef typesDef = typeStore.restore();
-            TypeSystem typeSystem = injector.getInstance(TypeSystem.class);
-            typeSystem.defineTypes(typesDef);
-        } catch (MetadataException e) {
-            throw new RuntimeException(e);
-        }
-        LOG.info("Restored type system from the store");
     }
 
     @Override

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java

@@ -93,6 +93,40 @@ public class HiveLineageResource {
     }
 
     /**
+     * Returns the inputs graph for a given entity.
+     *
+     * @param tableName table name
+     */
+    @GET
+    @Path("table/{tableName}/inputs/graph")
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response inputsGraph(@Context HttpServletRequest request,
+                                @PathParam("tableName") String tableName) {
+        Preconditions.checkNotNull(tableName, "table name cannot be null");
+        LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
+
+        try {
+            final String jsonResult = lineageService.getInputsGraph(tableName);
+
+            JSONObject response = new JSONObject();
+            response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+            response.put("tableName", tableName);
+            response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
+
+            return Response.ok(response).build();
+        } catch (DiscoveryException e) {
+            LOG.error("Unable to get lineage inputs graph for table {}", tableName, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+        } catch (JSONException e) {
+            LOG.error("Unable to get lineage inputs graph for table {}", tableName, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+        }
+    }
+
+    /**
      * Returns the outputs for a given entity.
      *
      * @param tableName table name
@@ -117,11 +151,45 @@ public class HiveLineageResource {
 
             return Response.ok(response).build();
         } catch (DiscoveryException e) {
-            LOG.error("Unable to get lineage inputs for table {}", tableName, e);
+            LOG.error("Unable to get lineage outputs for table {}", tableName, e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
         } catch (JSONException e) {
-            LOG.error("Unable to get lineage inputs for table {}", tableName, e);
+            LOG.error("Unable to get lineage outputs for table {}", tableName, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+        }
+    }
+
+    /**
+     * Returns the outputs graph for a given entity.
+     *
+     * @param tableName table name
+     */
+    @GET
+    @Path("table/{tableName}/outputs/graph")
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response outputsGraph(@Context HttpServletRequest request,
+                                 @PathParam("tableName") String tableName) {
+        Preconditions.checkNotNull(tableName, "table name cannot be null");
+        LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
+
+        try {
+            final String jsonResult = lineageService.getOutputs(tableName);
+
+            JSONObject response = new JSONObject();
+            response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+            response.put("tableName", tableName);
+            response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
+
+            return Response.ok(response).build();
+        } catch (DiscoveryException e) {
+            LOG.error("Unable to get lineage outputs graph for table {}", tableName, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+        } catch (JSONException e) {
+            LOG.error("Unable to get lineage outputs graph for table {}", tableName, e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
         }