Skip to content

A
atlas
Commit 9d303743 by Shwetha GS
Options

ATLAS-80 Support for variables in application properties (shwethags)

parent 266d7cc0
Showing with 282 additions and 213 deletions
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java
......@@ -18,7 +18,6 @@
package org.apache.atlas.hive.hook;
import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.security.SecurityProperties;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
......@@ -100,11 +99,12 @@ public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
String confLocation = System.getProperty("atlas.conf");
URL url;
if (confLocation == null) {
url = PropertiesUtil.class.getResource("/application.properties");
url = NegativeSSLAndKerberosHiveHookIT.class.getResource("/application.properties");
} else {
url = new File(confLocation, "application.properties").toURI().toURL();
}
configuration.load(url);
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty("atlas.http.authentication.enabled", "true");
configuration.setProperty("atlas.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java
......@@ -20,7 +20,6 @@ package org.apache.atlas.hive.hook;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.security.SecurityProperties;
import org.apache.commons.configuration.PropertiesConfiguration;
......@@ -116,7 +115,7 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
String confLocation = System.getProperty("atlas.conf");
URL url;
if (confLocation == null) {
url = PropertiesUtil.class.getResource("/application.properties");
url = SSLAndKerberosHiveHookIT.class.getResource("/application.properties");
} else {
url = new File(confLocation, "application.properties").toURI().toURL();
}
......
client/src/main/java/org/apache/atlas/ApplicationProperties.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ApplicationProperties extends PropertiesConfiguration {
private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class);
public static final String APPLICATION_PROPERTIES = "application.properties";
public static final String CLIENT_PROPERTIES = "client.properties";
private static Configuration INSTANCE = null;
private ApplicationProperties(URL url) throws ConfigurationException {
super(url);
}
public static Configuration get() throws AtlasException {
if (INSTANCE == null) {
synchronized (ApplicationProperties.class) {
if (INSTANCE == null) {
INSTANCE = get(APPLICATION_PROPERTIES);
}
}
}
return INSTANCE;
}
public static Configuration get(String fileName) throws AtlasException {
String confLocation = System.getProperty("atlas.conf");
try {
URL url = confLocation == null ? ApplicationProperties.class.getResource("/" + fileName)
: new File(confLocation, fileName).toURI().toURL();
LOG.info("Loading {} from {}", fileName, url);
ApplicationProperties configuration = new ApplicationProperties(url);
Iterator<String> keys = configuration.getKeys();
LOG.debug("Configuration loaded:");
while(keys.hasNext()) {
String key = keys.next();
LOG.debug("{} = {}", key, configuration.getProperty(key));
}
return configuration;
} catch (Exception e) {
throw new AtlasException("Failed to load application properties", e);
}
}
public static final Configuration getSubsetConfiguration(Configuration inConf, String prefix) {
return inConf.subset(prefix);
}
@Override
public Object getProperty(String key) {
Object value = super.getProperty(key);
if (value instanceof String) {
value = substituteVars((String) value);
}
return value;
}
private static final Pattern VAR_PATTERN = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
private static final int MAX_SUBST = 20;
private String substituteVars(String expr) {
if (expr == null) {
return null;
}
Matcher match = VAR_PATTERN.matcher("");
String eval = expr;
for(int s = 0; s < MAX_SUBST; s++) {
match.reset(eval);
if (!match.find()) {
return eval;
}
String var = match.group();
var = var.substring(2, var.length() - 1); // remove ${ .. }
String val = null;
try {
val = System.getProperty(var);
} catch(SecurityException se) {
LOG.warn("Unexpected SecurityException in Configuration", se);
}
if (val == null) {
val = getString(var);
}
if (val == null) {
return eval; // return literal ${var}: var is unbound
}
// substitute
eval = eval.substring(0, match.start()) + val + eval.substring(match.end());
}
throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr);
}
}
client/src/main/java/org/apache/atlas/AtlasClient.java
......@@ -26,6 +26,7 @@ import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.atlas.security.SecureClientUtils;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.codehaus.jettison.json.JSONArray;
......@@ -87,7 +88,7 @@ public class AtlasClient {
public AtlasClient(String baseUrl, UserGroupInformation ugi, String doAsUser) {
DefaultClientConfig config = new DefaultClientConfig();
PropertiesConfiguration clientConfig = null;
Configuration clientConfig = null;
try {
clientConfig = getClientProperties();
if (clientConfig.getBoolean(TLS_ENABLED, false)) {
......@@ -109,8 +110,8 @@ public class AtlasClient {
service = client.resource(UriBuilder.fromUri(baseUrl).build());
}
protected PropertiesConfiguration getClientProperties() throws AtlasException {
return PropertiesUtil.getClientProperties();
protected Configuration getClientProperties() throws AtlasException {
return ApplicationProperties.get(ApplicationProperties.CLIENT_PROPERTIES);
}
enum API {
......
client/src/main/java/org/apache/atlas/PropertiesUtil.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
public class PropertiesUtil {
private static final Logger LOG = LoggerFactory.getLogger(PropertiesUtil.class);
private static final String APPLICATION_PROPERTIES = "application.properties";
public static final String CLIENT_PROPERTIES = "client.properties";
public static PropertiesConfiguration getApplicationProperties() throws AtlasException {
return getPropertiesConfiguration(APPLICATION_PROPERTIES);
}
public static PropertiesConfiguration getClientProperties() throws AtlasException {
return getPropertiesConfiguration(CLIENT_PROPERTIES);
}
private static PropertiesConfiguration getPropertiesConfiguration(String name) throws AtlasException {
String confLocation = System.getProperty("atlas.conf");
URL url;
try {
if (confLocation == null) {
url = PropertiesUtil.class.getResource("/" + name);
} else {
url = new File(confLocation, name).toURI().toURL();
}
LOG.info("Loading {} from {}", name, url);
return new PropertiesConfiguration(url);
} catch (Exception e) {
throw new AtlasException("Failed to load application properties", e);
}
}
}
client/src/main/java/org/apache/atlas/security/SecureClientUtils.java
......@@ -20,8 +20,6 @@ import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.atlas.AtlasException;
import org.apache.atlas.PropertiesUtil;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
......@@ -62,7 +60,8 @@ public class SecureClientUtils {
public static URLConnectionClientHandler getClientConnectionHandler(DefaultClientConfig config,
PropertiesConfiguration clientConfig, final String doAsUser, final UserGroupInformation ugi) {
org.apache.commons.configuration.Configuration clientConfig, final String doAsUser,
final UserGroupInformation ugi) {
config.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
Configuration conf = new Configuration();
conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, "ssl-client.xml"));
......@@ -177,7 +176,7 @@ public class SecureClientUtils {
try {
if (confLocation == null) {
String persistDir = null;
URL resource = PropertiesUtil.class.getResource("/");
URL resource = SecureClientUtils.class.getResource("/");
if (resource != null) {
persistDir = resource.toURI().getPath();
}
......@@ -193,7 +192,7 @@ public class SecureClientUtils {
return new File(sslDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
}
public static void persistSSLClientConfiguration(PropertiesConfiguration clientConfig)
public static void persistSSLClientConfiguration(org.apache.commons.configuration.Configuration clientConfig)
throws AtlasException, IOException {
//trust settings
Configuration configuration = new Configuration(false);
......
client/src/main/java/org/apache/atlas/security/SecurityProperties.java
......@@ -21,16 +21,16 @@ package org.apache.atlas.security;
*
*/
public interface SecurityProperties {
public static final String TLS_ENABLED = "atlas.enableTLS";
public static final String KEYSTORE_FILE_KEY = "keystore.file";
public static final String DEFAULT_KEYSTORE_FILE_LOCATION = "target/atlas.keystore";
public static final String KEYSTORE_PASSWORD_KEY = "keystore.password";
public static final String TRUSTSTORE_FILE_KEY = "truststore.file";
public static final String DEFATULT_TRUSTORE_FILE_LOCATION = "target/atlas.keystore";
public static final String TRUSTSTORE_PASSWORD_KEY = "truststore.password";
public static final String SERVER_CERT_PASSWORD_KEY = "password";
public static final String CLIENT_AUTH_KEY = "client.auth.enabled";
public static final String CERT_STORES_CREDENTIAL_PROVIDER_PATH = "cert.stores.credential.provider.path";
public static final String SSL_CLIENT_PROPERTIES = "ssl-client.xml";
public static final String BIND_ADDRESS = "atlas.server.bind.address";
String TLS_ENABLED = "atlas.enableTLS";
String KEYSTORE_FILE_KEY = "keystore.file";
String DEFAULT_KEYSTORE_FILE_LOCATION = "target/atlas.keystore";
String KEYSTORE_PASSWORD_KEY = "keystore.password";
String TRUSTSTORE_FILE_KEY = "truststore.file";
String DEFATULT_TRUSTORE_FILE_LOCATION = "target/atlas.keystore";
String TRUSTSTORE_PASSWORD_KEY = "truststore.password";
String SERVER_CERT_PASSWORD_KEY = "password";
String CLIENT_AUTH_KEY = "client.auth.enabled";
String CERT_STORES_CREDENTIAL_PROVIDER_PATH = "cert.stores.credential.provider.path";
String SSL_CLIENT_PROPERTIES = "ssl-client.xml";
String BIND_ADDRESS = "atlas.server.bind.address";
}
client/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import org.apache.commons.configuration.Configuration;
import org.testng.Assert;
import org.testng.annotations.Test;
public class ApplicationPropertiesTest {
@Test
public void testVariables() throws Exception {
Configuration properties = ApplicationProperties.get();
//plain property without variables
Assert.assertEquals(properties.getString("atlas.service"), "atlas");
//property containing system property
String data = "/var/data/" + System.getProperty("user.name") + "/atlas";
Assert.assertEquals(properties.getString("atlas.data"), data);
//property referencing other property
Assert.assertEquals(properties.getString("atlas.graph.data"), data + "/graph");
//invalid system property - not substituted
Assert.assertEquals(properties.getString("atlas.db"), "${atlasdb}");
}
@Test
//variable substitutions should work with subset configuration as well
public void testSubset() throws Exception {
Configuration configuration = ApplicationProperties.get();
Configuration subConfiguration = configuration.subset("atlas");
Assert.assertEquals(subConfiguration.getString("service"), "atlas");
String data = "/var/data/" + System.getProperty("user.name") + "/atlas";
Assert.assertEquals(subConfiguration.getString("data"), data);
Assert.assertEquals(subConfiguration.getString("graph.data"), data + "/graph");
}
}
client/src/test/resources/application.properties 0 → 100644
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#system property
atlas.data=/var/data/${user.name}/atlas
#re-use existing property
atlas.graph.data=${atlas.data}/graph
#plain property
atlas.service=atlas
#invalid system property
atlas.db=${atlasdb}
\ No newline at end of file
docs/pom.xml
......@@ -32,39 +32,9 @@
<properties>
<skipTests>true</skipTests>
<skipDocs>false</skipDocs>
</properties>
<!--
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.maven.doxia</groupId>
<artifactId>doxia-module-twiki</artifactId>
<version>1.3</version>
</dependency>
</dependencies>
<executions>
<execution>
<goals>
<goal>site</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
<configuration>
<generateProjectInfo>false</generateProjectInfo>
<generateReports>false</generateReports>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
-->
<build>
<plugins>
<plugin>
......@@ -92,7 +62,7 @@
</execution>
</executions>
<configuration>
<!--<outputDirectory>../../site</outputDirectory>-->
<skip>${skipDocs}</skip>
<reportPlugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
......@@ -131,25 +101,6 @@
</reportPlugins>
</configuration>
</plugin>
<!--
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
<version>2.3</version>
<configuration>
<webResources>
<resource>
<directory>src/site/resources</directory>
<targetPath>pages</targetPath>
</resource>
<resource>
<directory>target/site</directory>
<targetPath>pages</targetPath>
</resource>
</webResources>
</configuration>
</plugin>
-->
</plugins>
</build>
</project>
pom.xml
......@@ -1304,6 +1304,7 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<inherited>false</inherited>
<configuration>
<descriptors>
<descriptor>src/main/assemblies/standalone-package.xml</descriptor>
......@@ -1318,6 +1319,9 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.18.1</version>
<configuration>
<systemProperties>
<user.dir>${project.basedir}</user.dir>
</systemProperties>
<!--<skipTests>true</skipTests>-->
<forkMode>always</forkMode>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
......@@ -1536,6 +1540,7 @@
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<configuration>
......
release-log.txt
......@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES:
ATLAS-80 Support for variables in application properties (shwethags)
ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags)
ATLAS-45 Entity submit fails (suma.shivaprasad via shwethags)
......
repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
......@@ -19,10 +19,10 @@
package org.apache.atlas.discovery;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.ParamChecker;
import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.Expressions;
......@@ -33,7 +33,7 @@ import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Option;
......@@ -63,12 +63,12 @@ public class HiveLineageService implements LineageService {
private static final String HIVE_TABLE_EXISTS_QUERY;
private static final PropertiesConfiguration propertiesConf;
private static final Configuration propertiesConf;
static {
// todo - externalize this using type system - dog food
try {
propertiesConf = PropertiesUtil.getApplicationProperties();
propertiesConf = ApplicationProperties.get();
HIVE_TABLE_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.table.type.name", "DataSet");
HIVE_PROCESS_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.process.type.name", "Process");
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.inputs.name", "inputs");
......
repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
......@@ -21,16 +21,13 @@ package org.apache.atlas.repository.graph;
import com.google.inject.Provides;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.PropertiesUtil;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Singleton;
import java.util.Iterator;
import java.util.Properties;
/**
* Default implementation for Graph Provider that doles out Titan Graph.
......@@ -42,31 +39,13 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
/**
* Constant for the configuration property that indicates the prefix.
*/
private static final String ATLAS_PREFIX = "atlas.graph.";
private static final String GRAPH_PREFIX = "atlas.graph";
private static TitanGraph graphInstance;
private static Configuration getConfiguration() throws AtlasException {
PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties();
Configuration graphConfig = new PropertiesConfiguration();
Properties sysProperties = System.getProperties();
LOG.info("System properties: ");
LOG.info(sysProperties.toString());
final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) {
String key = iterator.next();
if (key.startsWith(ATLAS_PREFIX)) {
String value = (String) configProperties.getProperty(key);
key = key.substring(ATLAS_PREFIX.length());
graphConfig.setProperty(key, value);
LOG.info("Using graph property {}={}", key, value);
}
}
return graphConfig;
Configuration configProperties = ApplicationProperties.get();
return ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
}
@Override
......
src/bin/atlas_start.py
......@@ -24,7 +24,6 @@ import atlas_config as mc
METADATA_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=application.log"
METADATA_COMMAND_OPTS="-Datlas.home=%s"
METADATA_CONFIG_OPTS="-Datlas.conf=%s"
ATLAS_USER_DIR="-Duser.dir=%s"
DEFAULT_JVM_OPTS="-Xmx1024m"
def main():
......@@ -40,9 +39,6 @@ def main():
cmd_opts = (METADATA_COMMAND_OPTS % metadata_home)
jvm_opts_list.extend(cmd_opts.split())
user_dir = (ATLAS_USER_DIR % metadata_home)
jvm_opts_list.extend(user_dir.split())
config_opts = (METADATA_CONFIG_OPTS % confdir)
jvm_opts_list.extend(config_opts.split())
......
src/conf/application.properties
......@@ -19,7 +19,7 @@
######### Graph Database Configs #########
# Graph Storage
atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.directory=data/berkley
atlas.graph.storage.directory=${atlas.home}/data/berkley
#Hbase as stoarge backend
#hbase
......@@ -40,7 +40,7 @@ atlas.graph.storage.directory=data/berkley
# Graph Search Index
atlas.graph.index.search.backend=elasticsearch
atlas.graph.index.search.directory=data/es
atlas.graph.index.search.directory=${atlas.home}/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
......
src/test/python/scripts/TestMetadata.py
......@@ -51,13 +51,13 @@ class TestMetadata(unittest.TestCase):
'org.apache.atlas.Main',
['-app', 'metadata_home/server/webapp/atlas'],
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib\\*:metadata_home/libext\\*',
['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Duser.dir=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
else:
java_mock.assert_called_with(
'org.apache.atlas.Main',
['-app', 'metadata_home/server/webapp/atlas'],
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib/*:metadata_home/libext/*',
['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Duser.dir=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
pass
def test_jar_java_lookups_fail(self):
......
webapp/src/main/java/org/apache/atlas/Main.java
......@@ -24,6 +24,7 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
......@@ -76,7 +77,7 @@ public final class Main {
}
setApplicationHome();
PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties();
Configuration configuration = ApplicationProperties.get();
final String enableTLSFlag = configuration.getString("atlas.enableTLS");
final int appPort = getApplicationPort(cmd, enableTLSFlag, configuration);
final boolean enableTLS = isTLSEnabled(enableTLSFlag, appPort);
......@@ -100,9 +101,7 @@ public final class Main {
return buildConfiguration.getString("project.version");
}
static int getApplicationPort(CommandLine cmd,
String enableTLSFlag,
PropertiesConfiguration configuration) {
static int getApplicationPort(CommandLine cmd, String enableTLSFlag, Configuration configuration) {
final int appPort;
if (cmd.hasOption(APP_PORT)) {
appPort = Integer.valueOf(cmd.getOptionValue(APP_PORT));
......@@ -114,7 +113,7 @@ public final class Main {
return appPort;
}
private static int getPortValue(PropertiesConfiguration configuration, String enableTLSFlag) {
private static int getPortValue(Configuration configuration, String enableTLSFlag) {
int appPort;
assert configuration != null;
......
webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java
......@@ -19,9 +19,10 @@
package org.apache.atlas.web.filters;
import com.google.inject.Singleton;
import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.security.SecurityProperties;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationConverter;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
......@@ -44,18 +45,20 @@ import java.util.Properties;
@Singleton
public class AtlasAuthenticationFilter extends AuthenticationFilter {
private static final Logger LOG = LoggerFactory.getLogger(AtlasAuthenticationFilter.class);
static final String PREFIX = "atlas.http.authentication.";
static final String PREFIX = "atlas.http.authentication";
@Override
protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException {
PropertiesConfiguration configuration;
Configuration configuration;
try {
configuration = PropertiesUtil.getApplicationProperties();
configuration = ApplicationProperties.get();
} catch (Exception e) {
throw new ServletException(e);
}
Properties config = new Properties();
// transfer application.properties config items starting with defined prefix
Configuration subConfiguration = ApplicationProperties.getSubsetConfiguration(configuration, PREFIX);
Properties config = ConfigurationConverter.getProperties(subConfiguration);
config.put(AuthenticationFilter.COOKIE_PATH, "/");
......@@ -65,16 +68,6 @@ public class AtlasAuthenticationFilter extends AuthenticationFilter {
String name = enumeration.nextElement();
config.put(name, filterConfig.getInitParameter(name));
}
// transfer application.properties config items starting with defined prefix
Iterator<String> itor = configuration.getKeys();
while (itor.hasNext()) {
String name = itor.next();
if (name.startsWith(PREFIX)) {
String value = configuration.getString(name);
name = name.substring(PREFIX.length());
config.put(name, value);
}
}
//Resolve _HOST into bind address
String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS);
......
webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java
......@@ -29,19 +29,18 @@ import com.sun.jersey.guice.JerseyServletModule;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.web.filters.AtlasAuthenticationFilter;
import org.apache.atlas.web.filters.AuditFilter;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.servlet.ServletContextEvent;
import java.util.HashMap;
import java.util.Map;
......@@ -85,7 +84,7 @@ public class GuiceServletConfig extends GuiceServletContextListener {
private void configureAuthenticationFilter() throws ConfigurationException {
try {
PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties();
Configuration configuration = ApplicationProperties.get();
if (Boolean.valueOf(configuration.getString(HTTP_AUTHENTICATION_ENABLED))) {
filter("/*").through(AtlasAuthenticationFilter.class);
}
......
webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java
......@@ -16,11 +16,10 @@
*/
package org.apache.atlas.web.listeners;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.security.SecurityProperties;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
......@@ -51,12 +50,7 @@ public class LoginProcessor {
// first, let's see if we're running in a hadoop cluster and have the env configured
boolean isHadoopCluster = isHadoopCluster();
Configuration hadoopConfig = isHadoopCluster ? getHadoopConfiguration() : new Configuration(false);
PropertiesConfiguration configuration = null;
try {
configuration = getPropertiesConfiguration();
} catch (ConfigurationException e) {
LOG.warn("Error reading application configuration", e);
}
org.apache.commons.configuration.Configuration configuration = getApplicationConfiguration();
if (!isHadoopCluster) {
// need to read the configured authentication choice and create the UGI configuration
setupHadoopConfiguration(hadoopConfig, configuration);
......@@ -64,7 +58,8 @@ public class LoginProcessor {
doServiceLogin(hadoopConfig, configuration);
}
protected void doServiceLogin(Configuration hadoopConfig, PropertiesConfiguration configuration) {
protected void doServiceLogin(Configuration hadoopConfig,
org.apache.commons.configuration.Configuration configuration) {
UserGroupInformation.setConfiguration(hadoopConfig);
UserGroupInformation ugi = null;
......@@ -85,7 +80,7 @@ public class LoginProcessor {
}
}
private String getHostname(PropertiesConfiguration configuration) {
private String getHostname(org.apache.commons.configuration.Configuration configuration) {
String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS);
if (bindAddress == null) {
LOG.info("No host name configured. Defaulting to local host name.");
......@@ -98,7 +93,8 @@ public class LoginProcessor {
return bindAddress;
}
protected void setupHadoopConfiguration(Configuration hadoopConfig, PropertiesConfiguration configuration) {
protected void setupHadoopConfiguration(Configuration hadoopConfig, org.apache.commons.configuration.Configuration
configuration) {
String authMethod;
authMethod = configuration != null ? configuration.getString(AUTHENTICATION_METHOD) : null;
// getString may return null, and would like to log the nature of the default setting
......@@ -135,12 +131,13 @@ public class LoginProcessor {
* @return the metadata configuration.
* @throws ConfigurationException
*/
protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
protected org.apache.commons.configuration.Configuration getApplicationConfiguration() {
try {
return PropertiesUtil.getApplicationProperties();
return ApplicationProperties.get();
} catch (AtlasException e) {
throw new ConfigurationException(e);
LOG.warn("Error reading application configuration", e);
}
return null;
}
/**
......
webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
......@@ -23,9 +23,8 @@ import org.apache.commons.configuration.PropertiesConfiguration;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppClassLoader;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.webapp.WebAppContext;
import java.io.IOException;
......
webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
......@@ -38,7 +38,7 @@ public class LoginProcessorIT extends BaseSecurityTest {
public void testDefaultSimpleLogin() throws Exception {
LoginProcessor processor = new LoginProcessor() {
@Override
protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
protected org.apache.commons.configuration.Configuration getApplicationConfiguration() {
return new PropertiesConfiguration();
}
};
......@@ -55,7 +55,7 @@ public class LoginProcessorIT extends BaseSecurityTest {
LoginProcessor processor = new LoginProcessor() {
@Override
protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
protected org.apache.commons.configuration.Configuration getApplicationConfiguration() {
PropertiesConfiguration config = new PropertiesConfiguration();
config.setProperty("atlas.authentication.method", "kerberos");
config.setProperty("atlas.authentication.principal", "dgi@EXAMPLE.COM");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment