Skip to content

A
atlas
Commit 93fed80a by Suma Shivaprasad
Options

Merge branch 'master' of https://github.com/hortonworks/metadata into BUG-32830

parents 9d50a245 5becd8a5
Showing with 253 additions and 58 deletions
docs/src/site/twiki/Configuration.twiki 0 → 100644
---+ Configuring Apache Atlas
---++ Introduction
All configuration in Atlas uses java properties style configuration.
---++ Application Properties
The main configuration file is application.properties which is in the *conf* dir at the deployed
location. It consists of the following sections:
---+++ Graph Database Configs
---++++ Graph persistence engine
This section sets up the graph db - titan - to use a persistence engine. Please refer to
<a href="http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html">link</a> for more
details. The example below uses BerkeleyDBJE.
<verbatim>
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=data/berkley
</verbatim>
---++++ Graph Search Index
This section sets up the graph db - titan - to use an search indexing system. The example
configuration below setsup to use an embedded Elastic search indexing system.
<verbatim>
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
metadata.graph.index.search.elasticsearch.create.sleep=2000
</verbatim>
---+++ Hive Lineage Configs
The higher layer services like hive lineage, schema, etc. are driven by the type system and this
section encodes the specific types for the hive data model.
# This models follows the quick-start guide
<verbatim>
metadata.lineage.hive.table.type.name=hive_table
metadata.lineage.hive.table.column.name=columns
metadata.lineage.hive.process.type.name=hive_process
metadata.lineage.hive.process.inputs.name=inputTables
metadata.lineage.hive.process.outputs.name=outputTables
#Currently unused
#metadata.lineage.hive.column.type.name=Column
</verbatim>
---+++ Security Properties
---++++ SSL config
The following property is used to toggle the SSL feature.
<verbatim>
metadata.enableTLS=false
</verbatim>
repository/src/main/java/org/apache/hadoop/metadata/discovery/HiveLineageService.java
......@@ -19,7 +19,6 @@
package org.apache.hadoop.metadata.discovery;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.PropertiesUtil;
......@@ -116,6 +115,24 @@ public class HiveLineageService implements LineageService {
}
/**
* Return the lineage outputs graph for the given tableName.
*
* @param tableName tableName
* @return Outputs Graph as JSON
*/
@Override
public String getOutputsGraph(String tableName) throws DiscoveryException {
LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
HiveWhereUsedQuery outputsQuery = new HiveWhereUsedQuery(
HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
Option.empty(), SELECT_ATTRIBUTES, true,
graphPersistenceStrategy, titanGraph);
return outputsQuery.graph().toInstanceJson();
}
/**
* Return the lineage inputs for the given tableName.
*
* @param tableName tableName
......@@ -141,6 +158,24 @@ public class HiveLineageService implements LineageService {
}
/**
* Return the lineage inputs graph for the given tableName.
*
* @param tableName tableName
* @return Inputs Graph as JSON
*/
@Override
public String getInputsGraph(String tableName) throws DiscoveryException {
LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
HiveLineageQuery inputsQuery = new HiveLineageQuery(
HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
Option.empty(), SELECT_ATTRIBUTES, true,
graphPersistenceStrategy, titanGraph);
return inputsQuery.graph().toInstanceJson();
}
/**
* Return the schema for the given tableName.
*
* @param tableName tableName
......
repository/src/main/java/org/apache/hadoop/metadata/discovery/LineageService.java
......@@ -32,6 +32,14 @@ public interface LineageService {
String getOutputs(String tableName) throws DiscoveryException;
/**
* Return the lineage outputs graph for the given tableName.
*
* @param tableName tableName
* @return Outputs Graph as JSON
*/
String getOutputsGraph(String tableName) throws DiscoveryException;
/**
* Return the lineage inputs for the given tableName.
*
* @param tableName tableName
......@@ -40,6 +48,14 @@ public interface LineageService {
String getInputs(String tableName) throws DiscoveryException;
/**
* Return the lineage inputs graph for the given tableName.
*
* @param tableName tableName
* @return Inputs Graph as JSON
*/
String getInputsGraph(String tableName) throws DiscoveryException;
/**
* Return the schema for the given tableName.
*
* @param tableName tableName
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java
......@@ -25,6 +25,8 @@ import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.PropertiesUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Singleton;
import java.util.Iterator;
......@@ -34,6 +36,8 @@ import java.util.Iterator;
*/
public class TitanGraphProvider implements GraphProvider<TitanGraph> {
private static final Logger LOG = LoggerFactory.getLogger(TitanGraphProvider.class);
/**
* Constant for the configuration property that indicates the prefix.
*/
......@@ -51,6 +55,7 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
String value = (String) configProperties.getProperty(key);
key = key.substring(METADATA_PREFIX.length());
graphConfig.setProperty(key, value);
LOG.info("Using graph property {}={}", key, value);
}
}
......
repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java
......@@ -207,46 +207,53 @@ public class GraphBackedTypeStore implements ITypeStore {
@Override
public TypesDef restore() throws MetadataException {
//Get all vertices for type system
Iterator vertices = titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
while (vertices.hasNext()) {
Vertex vertex = (Vertex) vertices.next();
DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
LOG.info("Restoring type {}.{}", typeCategory, typeName);
switch(typeCategory) {
case ENUM:
enums.add(getEnumType(vertex));
break;
case STRUCT:
AttributeDefinition[] attributes = getAttributes(vertex);
structs.add(new StructTypeDefinition(typeName, attributes));
break;
case CLASS:
ImmutableList<String> superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex);
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, superTypes, attributes));
break;
case TRAIT:
superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex);
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, superTypes, attributes));
break;
default:
throw new IllegalArgumentException("Unhandled type category " + typeCategory);
try {
titanGraph.rollback(); //Cleanup previous state
//Get all vertices for type system
Iterator vertices =
titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
while (vertices.hasNext()) {
Vertex vertex = (Vertex) vertices.next();
DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
LOG.info("Restoring type {}.{}", typeCategory, typeName);
switch (typeCategory) {
case ENUM:
enums.add(getEnumType(vertex));
break;
case STRUCT:
AttributeDefinition[] attributes = getAttributes(vertex);
structs.add(new StructTypeDefinition(typeName, attributes));
break;
case CLASS:
ImmutableList<String> superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex);
classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, superTypes, attributes));
break;
case TRAIT:
superTypes = getSuperTypes(vertex);
attributes = getAttributes(vertex);
traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, superTypes, attributes));
break;
default:
throw new IllegalArgumentException("Unhandled type category " + typeCategory);
}
}
titanGraph.commit();
return TypeUtils.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
} finally {
titanGraph.rollback();
}
return TypeUtils.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
}
private EnumTypeDefinition getEnumType(Vertex vertex) {
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
......@@ -20,6 +20,7 @@ package org.apache.hadoop.metadata.services;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.inject.Injector;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.discovery.SearchIndexer;
import org.apache.hadoop.metadata.listener.EntityChangeListener;
......@@ -73,9 +74,22 @@ public class DefaultMetadataService implements MetadataService {
this.typeSystem = TypeSystem.getInstance();
this.repository = repository;
restoreTypeSystem();
registerListener(searchIndexer);
}
private void restoreTypeSystem() {
LOG.info("Restoring type system from the store");
try {
TypesDef typesDef = typeStore.restore();
typeSystem.defineTypes(typesDef);
} catch (MetadataException e) {
throw new RuntimeException(e);
}
LOG.info("Restored type system from the store");
}
/**
* Creates a new type based on the type system to enable adding
* entities (instances for types).
......
webapp/src/main/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationFilter.java
......@@ -44,7 +44,7 @@ import java.util.Properties;
public class MetadataAuthenticationFilter extends AuthenticationFilter {
private static final Logger LOG = LoggerFactory.getLogger(MetadataAuthenticationFilter.class);
static final String PREFIX = "metadata.http.authentication.";
static final String BIND_ADDRESS = "bind.address";
static final String BIND_ADDRESS = "metadata.server.bind.address";
@Override
protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException {
......
webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java
......@@ -20,7 +20,11 @@ package org.apache.hadoop.metadata.web.listeners;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.TypeLiteral;
import com.google.inject.matcher.Matchers;
import com.google.inject.servlet.GuiceServletContextListener;
import com.google.inject.spi.TypeEncounter;
import com.google.inject.spi.TypeListener;
import com.sun.jersey.api.core.PackagesResourceConfig;
import com.sun.jersey.guice.JerseyServletModule;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
......@@ -41,6 +45,8 @@ import javax.servlet.ServletContextEvent;
import java.util.HashMap;
import java.util.Map;
import static com.google.inject.matcher.Matchers.*;
public class GuiceServletConfig extends GuiceServletContextListener {
private static final Logger LOG = LoggerFactory.getLogger(GuiceServletConfig.class);
......@@ -105,22 +111,6 @@ public class GuiceServletConfig extends GuiceServletContextListener {
// perform login operations
LoginProcessor loginProcessor = new LoginProcessor();
loginProcessor.login();
restoreTypeSystem();
}
private void restoreTypeSystem() {
LOG.info("Restoring type system from the store");
Injector injector = getInjector();
ITypeStore typeStore = injector.getInstance(ITypeStore.class);
try {
TypesDef typesDef = typeStore.restore();
TypeSystem typeSystem = injector.getInstance(TypeSystem.class);
typeSystem.defineTypes(typesDef);
} catch (MetadataException e) {
throw new RuntimeException(e);
}
LOG.info("Restored type system from the store");
}
@Override
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java
......@@ -93,6 +93,40 @@ public class HiveLineageResource {
}
/**
* Returns the inputs graph for a given entity.
*
* @param tableName table name
*/
@GET
@Path("table/{tableName}/inputs/graph")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response inputsGraph(@Context HttpServletRequest request,
@PathParam("tableName") String tableName) {
Preconditions.checkNotNull(tableName, "table name cannot be null");
LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
try {
final String jsonResult = lineageService.getInputsGraph(tableName);
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put("tableName", tableName);
response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
return Response.ok(response).build();
} catch (DiscoveryException e) {
LOG.error("Unable to get lineage inputs graph for table {}", tableName, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (JSONException e) {
LOG.error("Unable to get lineage inputs graph for table {}", tableName, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Returns the outputs for a given entity.
*
* @param tableName table name
......@@ -117,11 +151,45 @@ public class HiveLineageResource {
return Response.ok(response).build();
} catch (DiscoveryException e) {
LOG.error("Unable to get lineage inputs for table {}", tableName, e);
LOG.error("Unable to get lineage outputs for table {}", tableName, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (JSONException e) {
LOG.error("Unable to get lineage inputs for table {}", tableName, e);
LOG.error("Unable to get lineage outputs for table {}", tableName, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
/**
* Returns the outputs graph for a given entity.
*
* @param tableName table name
*/
@GET
@Path("table/{tableName}/outputs/graph")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response outputsGraph(@Context HttpServletRequest request,
@PathParam("tableName") String tableName) {
Preconditions.checkNotNull(tableName, "table name cannot be null");
LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
try {
final String jsonResult = lineageService.getOutputs(tableName);
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put("tableName", tableName);
response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
return Response.ok(response).build();
} catch (DiscoveryException e) {
LOG.error("Unable to get lineage outputs graph for table {}", tableName, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (JSONException e) {
LOG.error("Unable to get lineage outputs graph for table {}", tableName, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment