Skip to content

A
atlas
Commit 43a1e25b by Shwetha GS
Options

ATLAS-1081 Atlas jetty server configuration (shwethags)

parent d9577516
Showing with 67 additions and 47 deletions
common/src/main/java/org/apache/atlas/AtlasProperties.java → common/src/main/java/org/apache/atlas/AtlasConfiguration.java
......@@ -21,12 +21,21 @@ package org.apache.atlas;
import org.apache.commons.configuration.Configuration;
/**
* Utility for reading properties in atlas-application.properties.
* Enum that encapsulated each property name and its default value.
*/
public final class AtlasProperties {
private static final Configuration APPLICATION_PROPERTIES;
public enum AtlasConfiguration {
//web server configuration
WEBSERVER_MIN_THREADS("atlas.webserver.minthreads", 10),
WEBSERVER_MAX_THREADS("atlas.webserver.maxthreads", 100),
WEBSERVER_KEEPALIVE_SECONDS("atlas.webserver.keepalivetimesecs", 60),
WEBSERVER_QUEUE_SIZE("atlas.webserver.queuesize", 100),
WEBSERVER_REQUEST_BUFFER_SIZE("atlas.jetty.request.buffer.size", 16192),
private AtlasProperties() { }
//search configuration
SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000),
SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100);
private static final Configuration APPLICATION_PROPERTIES;
static {
try {
......@@ -36,29 +45,28 @@ public final class AtlasProperties {
}
}
/**
* Enum that encapsulated each property name and its default value.
*/
public enum AtlasProperty {
SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000),
SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100);
private final String propertyName;
private final Object defaultValue;
AtlasProperty(String propertyName, Object defaultValue) {
AtlasConfiguration(String propertyName, Object defaultValue) {
this.propertyName = propertyName;
this.defaultValue = defaultValue;
}
public int getInt() {
return APPLICATION_PROPERTIES.getInt(propertyName, Integer.valueOf(defaultValue.toString()).intValue());
}
public static <T> T getProperty(AtlasProperty property) {
Object value = APPLICATION_PROPERTIES.getProperty(property.propertyName);
if (value == null) {
return (T) property.defaultValue;
} else {
return (T) value;
public long getLong() {
return APPLICATION_PROPERTIES.getLong(propertyName, Long.valueOf(defaultValue.toString()).longValue());
}
public String getString() {
return APPLICATION_PROPERTIES.getString(propertyName, defaultValue.toString());
}
public Object get() {
Object value = APPLICATION_PROPERTIES.getProperty(propertyName);
return value == null ? defaultValue : value;
}
}
distro/src/bin/atlas_start.py
......@@ -25,7 +25,7 @@ ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=%s.log"
ATLAS_COMMAND_OPTS="-Datlas.home=%s"
ATLAS_CONFIG_OPTS="-Datlas.conf=%s"
DEFAULT_JVM_HEAP_OPTS="-Xmx1024m -XX:MaxPermSize=512m"
DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true"
DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true -server"
def main():
......
docs/src/site/twiki/Configuration.twiki
......@@ -271,4 +271,16 @@ atlas.graph.storage.lock.retries=10
# Milliseconds to wait before evicting a cached entry. This should be > atlas.graph.storage.lock.wait-time x atlas.graph.storage.lock.retries
# If this is set to a low value (default is 10000), warnings on transactions taking too long will occur in the Atlas application log.
atlas.graph.storage.cache.db-cache-time=120000
# Minimum number of threads in the atlas web server
atlas.webserver.minthreads=10
# Maximum number of threads in the atlas web server
atlas.webserver.maxthreads=100
# Keepalive time in secs for the thread pool of the atlas web server
atlas.webserver.keepalivetimesecs=60
# Queue size for the requests(when max threads are busy) for the atlas web server
atlas.webserver.queuesize=100
</verbatim>
\ No newline at end of file
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1081 Atlas jetty server configuration (shwethags)
ATLAS-1257 Map Entity REST APIs to ATLAS v1 backend (sumasai)
ATLAS-1279 Remove QueryPlan attribute from Hive Process entity (svimal2106)
ATLAS-1234 Lineage REST API - v2 (sarath.kum4r@gmail.com via mneethiraj)
......
repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
......@@ -20,8 +20,8 @@ package org.apache.atlas.discovery;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.AtlasException;
import org.apache.atlas.AtlasProperties;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
......@@ -178,7 +178,7 @@ public class DataSetLineageService implements LineageService {
if (propertiesConf.getString(configName) != null) {
final String schemaQuery =
String.format(propertiesConf.getString(configName), guid);
int limit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_MAX_LIMIT);
int limit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt();
return discoveryService.searchByDSL(schemaQuery, new QueryParams(limit, 0));
}
throw new SchemaNotFoundException("Schema is not configured for type " + typeName + ". Configure " + configName);
......
webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
......@@ -20,7 +20,7 @@ package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasProperties;
import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.discovery.DiscoveryException;
import org.apache.atlas.discovery.DiscoveryService;
......@@ -157,8 +157,8 @@ public class MetadataDiscoveryResource {
}
private QueryParams validateQueryParams(int limitParam, int offsetParam) {
int maxLimit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_MAX_LIMIT);
int defaultLimit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_DEFAULT_LIMIT);
int maxLimit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt();
int defaultLimit = AtlasConfiguration.SEARCH_DEFAULT_LIMIT.getInt();
int limit = defaultLimit;
boolean limitSet = (limitParam != Integer.valueOf(LIMIT_OFFSET_DEFAULT));
......
webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
......@@ -18,18 +18,20 @@
package org.apache.atlas.web.service;
import org.apache.atlas.ApplicationProperties;
import org.apache.commons.configuration.Configuration;
import org.apache.atlas.AtlasConfiguration;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
import org.eclipse.jetty.webapp.WebAppContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
/**
* This class embeds a Jetty server and a connector.
......@@ -37,11 +39,19 @@ import java.io.IOException;
public class EmbeddedServer {
public static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class);
private static final int DEFAULT_BUFFER_SIZE = 16192;
protected final Server server = new Server();
protected final Server server;
public EmbeddedServer(int port, String path) throws IOException {
int queueSize = AtlasConfiguration.WEBSERVER_QUEUE_SIZE.getInt();
LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<>(queueSize);
int minThreads = AtlasConfiguration.WEBSERVER_MIN_THREADS.getInt();
int maxThreads = AtlasConfiguration.WEBSERVER_MAX_THREADS.getInt();
long keepAliveTime = AtlasConfiguration.WEBSERVER_KEEPALIVE_SECONDS.getLong();
ExecutorThreadPool pool =
new ExecutorThreadPool(minThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, queue);
server = new Server(pool);
Connector connector = getConnector(port);
server.addConnector(connector);
......@@ -66,31 +76,18 @@ public class EmbeddedServer {
}
protected Connector getConnector(int port) throws IOException {
HttpConfiguration http_config = new HttpConfiguration();
// this is to enable large header sizes when Kerberos is enabled with AD
final int bufferSize = getBufferSize();
final int bufferSize = AtlasConfiguration.WEBSERVER_REQUEST_BUFFER_SIZE.getInt();;
http_config.setResponseHeaderSize(bufferSize);
http_config.setRequestHeaderSize(bufferSize);
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(http_config));
connector.setPort(port);
connector.setHost("0.0.0.0");
server.addConnector(connector);
return connector;
}
protected Integer getBufferSize() {
try {
Configuration configuration = ApplicationProperties.get();
return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);
} catch (Exception e) {
// do nothing
}
return DEFAULT_BUFFER_SIZE;
}
public void start() throws Exception {
server.start();
server.join();
......
webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
......@@ -19,10 +19,12 @@
package org.apache.atlas.web.service;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.AtlasException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
......@@ -30,24 +32,23 @@ import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.http.HttpVersion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import static org.apache.atlas.security.SecurityProperties.ATLAS_SSL_EXCLUDE_CIPHER_SUITES;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.CLIENT_AUTH_KEY;
import static org.apache.atlas.security.SecurityProperties.DEFATULT_TRUSTORE_FILE_LOCATION;
import static org.apache.atlas.security.SecurityProperties.DEFAULT_CIPHER_SUITES;
import static org.apache.atlas.security.SecurityProperties.DEFAULT_KEYSTORE_FILE_LOCATION;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.ATLAS_SSL_EXCLUDE_CIPHER_SUITES;
import static org.apache.atlas.security.SecurityProperties.DEFAULT_CIPHER_SUITES;
/**
* This is a jetty server which requires client auth via certificates.
......@@ -81,7 +82,7 @@ public class SecureEmbeddedServer extends EmbeddedServer {
// HTTP Configuration
HttpConfiguration http_config = new HttpConfiguration();
http_config.setSecureScheme("https");
final int bufferSize = getBufferSize();
final int bufferSize = AtlasConfiguration.WEBSERVER_REQUEST_BUFFER_SIZE.getInt();
http_config.setSecurePort(port);
http_config.setRequestHeaderSize(bufferSize);
http_config.setResponseHeaderSize(bufferSize);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment