Skip to content

A
atlas
Commit 997abdb0 by Dave Kantor
Options

ATLAS-1539 Integration tests in projects which use the typesystem test jar (e.g.…

ATLAS-1539 Integration tests in projects which use the typesystem test jar (e.g. webapp) can now be run successfully when invoked in the project directory (dkantor)
parent 9db4d261
master
No related merge requests found
Showing with 212 additions and 45 deletions
authorization/src/main/java/org/apache/atlas/authorize/simple/FileReaderUtil.java
......@@ -19,13 +19,13 @@
package org.apache.atlas.authorize.simple;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -33,12 +33,12 @@ public class FileReaderUtil {
private static Logger LOG = LoggerFactory.getLogger(FileReaderUtil.class);
private static boolean isDebugEnabled = LOG.isDebugEnabled();
public static List<String> readFile(String path) throws IOException {
public static List<String> readFile(InputStream policyStoreStream) throws IOException {
if (isDebugEnabled) {
LOG.debug("==> FileReaderUtil readFile({})", path);
LOG.debug("==> FileReaderUtil readFile()");
}
List<String> list = new ArrayList<>();
List<String> fileLines = Files.readAllLines(Paths.get(path), Charset.forName("UTF-8"));
List<String> fileLines = IOUtils.readLines(policyStoreStream, StandardCharsets.UTF_8);
if (fileLines != null) {
for (String line : fileLines) {
if ((!line.startsWith("#")) && Pattern.matches(".+;;.*;;.*;;.+", line))
......@@ -47,7 +47,7 @@ public class FileReaderUtil {
}
if (isDebugEnabled) {
LOG.debug("<== FileReaderUtil readFile({})", path);
LOG.debug("<== FileReaderUtil readFile()");
LOG.debug("Policies read :: " + list);
}
......
authorization/src/main/java/org/apache/atlas/authorize/simple/PolicyUtil.java
......@@ -33,7 +33,7 @@ public class PolicyUtil {
private static boolean isDebugEnabled = LOG.isDebugEnabled();
public Map<String, Map<AtlasResourceTypes, List<String>>> createPermissionMap(List<PolicyDef> policyDefList,
public static Map<String, Map<AtlasResourceTypes, List<String>>> createPermissionMap(List<PolicyDef> policyDefList,
AtlasActionTypes permissionType, SimpleAtlasAuthorizer.AtlasAccessorTypes principalType) {
if (isDebugEnabled) {
LOG.debug("==> PolicyUtil createPermissionMap\nCreating Permission Map for :: {} & {}", permissionType, principalType);
......@@ -104,5 +104,4 @@ public class PolicyUtil {
return userReadMap;
}
}
authorization/src/main/java/org/apache/atlas/authorize/simple/SimpleAtlasAuthorizer.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.authorize.simple;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
......@@ -33,7 +34,6 @@ import org.apache.atlas.authorize.AtlasAuthorizer;
import org.apache.atlas.authorize.AtlasResourceTypes;
import org.apache.atlas.utils.PropertiesUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.lang.StringUtils;
......@@ -44,7 +44,7 @@ import com.google.common.annotations.VisibleForTesting;
public final class SimpleAtlasAuthorizer implements AtlasAuthorizer {
public enum AtlasAccessorTypes {
public enum AtlasAccessorTypes {
USER, GROUP
}
......@@ -74,7 +74,6 @@ public final class SimpleAtlasAuthorizer implements AtlasAuthorizer {
}
try {
PolicyUtil util = new PolicyUtil();
PolicyParser parser = new PolicyParser();
optIgnoreCase = Boolean.valueOf(PropertiesUtil.getProperty("optIgnoreCase", "false"));
......@@ -82,25 +81,25 @@ public final class SimpleAtlasAuthorizer implements AtlasAuthorizer {
LOG.debug("Read from PropertiesUtil --> optIgnoreCase :: {}", optIgnoreCase);
}
Configuration configuration = ApplicationProperties.get();
String policyStorePath = configuration.getString("atlas.auth.policy.file", System.getProperty("atlas.conf")+"/policy-store.txt");
if (isDebugEnabled) {
LOG.debug("Loading Apache Atlas policies from : {}", policyStorePath);
InputStream policyStoreStream = ApplicationProperties.getFileAsInputStream(ApplicationProperties.get(), "atlas.auth.policy.file", "policy-store.txt");
List<String> policies = null;
try {
policies = FileReaderUtil.readFile(policyStoreStream);
}
finally {
policyStoreStream.close();
}
List<String> policies = FileReaderUtil.readFile(policyStorePath);
List<PolicyDef> policyDef = parser.parsePolicies(policies);
userReadMap = util.createPermissionMap(policyDef, AtlasActionTypes.READ, AtlasAccessorTypes.USER);
userWriteMap = util.createPermissionMap(policyDef, AtlasActionTypes.CREATE, AtlasAccessorTypes.USER);
userUpdateMap = util.createPermissionMap(policyDef, AtlasActionTypes.UPDATE, AtlasAccessorTypes.USER);
userDeleteMap = util.createPermissionMap(policyDef, AtlasActionTypes.DELETE, AtlasAccessorTypes.USER);
userReadMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.READ, AtlasAccessorTypes.USER);
userWriteMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.CREATE, AtlasAccessorTypes.USER);
userUpdateMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.UPDATE, AtlasAccessorTypes.USER);
userDeleteMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.DELETE, AtlasAccessorTypes.USER);
groupReadMap = util.createPermissionMap(policyDef, AtlasActionTypes.READ, AtlasAccessorTypes.GROUP);
groupWriteMap = util.createPermissionMap(policyDef, AtlasActionTypes.CREATE, AtlasAccessorTypes.GROUP);
groupUpdateMap = util.createPermissionMap(policyDef, AtlasActionTypes.UPDATE, AtlasAccessorTypes.GROUP);
groupDeleteMap = util.createPermissionMap(policyDef, AtlasActionTypes.DELETE, AtlasAccessorTypes.GROUP);
groupReadMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.READ, AtlasAccessorTypes.GROUP);
groupWriteMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.CREATE, AtlasAccessorTypes.GROUP);
groupUpdateMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.UPDATE, AtlasAccessorTypes.GROUP);
groupDeleteMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.DELETE, AtlasAccessorTypes.GROUP);
if (isDebugEnabled) {
LOG.debug("\n\nUserReadMap :: {}\nGroupReadMap :: {}", userReadMap, groupReadMap);
......
common/src/main/java/org/apache/atlas/ApplicationProperties.java
......@@ -25,6 +25,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.net.URL;
import java.util.Iterator;
......@@ -32,6 +35,8 @@ import java.util.Iterator;
* Application properties used by Atlas.
*/
public final class ApplicationProperties extends PropertiesConfiguration {
public static final String ATLAS_CONFIGURATION_DIRECTORY_PROPERTY = "atlas.conf";
private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class);
public static final String APPLICATION_PROPERTIES = "atlas-application.properties";
......@@ -65,7 +70,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
}
public static Configuration get(String fileName) throws AtlasException {
String confLocation = System.getProperty("atlas.conf");
String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY);
try {
URL url = null;
......@@ -125,4 +130,74 @@ public final class ApplicationProperties extends PropertiesConfiguration {
throw new AtlasException(e);
}
}
/**
* Get the specified property as an {@link InputStream}.
* If the property is not set, then the specified default filename
* is searched for in the following locations, in order of precedence:
* 1. Atlas configuration directory specified by the {@link #ATLAS_CONFIGURATION_DIRECTORY_PROPERTY} system property
* 2. relative to the working directory if {@link #ATLAS_CONFIGURATION_DIRECTORY_PROPERTY} is not set
* 3. as a classloader resource
*
* @param configuration
* @param propertyName
* @param defaultFileName name of file to use by default if specified property is not set in the configuration- if null,
* an {@link AtlasException} is thrown if the property is not set
* @return an {@link InputStream}
* @throws AtlasException if no file was found or if there was an error loading the file
*/
public static InputStream getFileAsInputStream(Configuration configuration, String propertyName, String defaultFileName) throws AtlasException {
File fileToLoad = null;
String fileName = configuration.getString(propertyName);
if (fileName == null) {
if (defaultFileName == null) {
throw new AtlasException(propertyName + " property not set and no default value specified");
}
fileName = defaultFileName;
String atlasConfDir = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY);
if (atlasConfDir != null) {
// Look for default filename in Atlas config directory
fileToLoad = new File(atlasConfDir, fileName);
}
else {
// Look for default filename under the working directory
fileToLoad = new File(fileName);
}
if (LOG.isDebugEnabled()) {
LOG.debug("{} property not set - defaulting to {}", propertyName, fileToLoad.getPath());
}
}
else {
// Look for configured filename
fileToLoad = new File(fileName);
if (LOG.isDebugEnabled()) {
LOG.debug("Using {} property setting: {}", propertyName, fileToLoad.getPath());
}
}
InputStream inStr = null;
if (fileToLoad.exists()) {
try {
inStr = new FileInputStream(fileToLoad);
} catch (FileNotFoundException e) {
throw new AtlasException("Error loading file " + fileName, e);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Loaded file from : {}", fileToLoad.getPath());
}
}
else {
// Look for file as class loader resource
inStr = Thread.currentThread().getContextClassLoader().getResourceAsStream(fileName);
if (inStr == null) {
String msg = fileName + " not found in file system or as class loader resource";
LOG.error(msg);
throw new AtlasException(msg);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Loaded {} as resource from : {}", fileName, Thread.currentThread().getContextClassLoader().getResource(fileName).toString());
}
}
return inStr;
}
}
common/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import java.io.InputStream;
import org.apache.commons.configuration.Configuration;
import org.testng.annotations.Test;
import static org.testng.Assert.*;
/**
* Unit test for {@link ApplicationProperties}
*
*/
public class ApplicationPropertiesTest {
@Test
public void testGetFileAsInputStream() throws Exception {
Configuration props = ApplicationProperties.get("test.properties");
// configured file as class loader resource
InputStream inStr = ApplicationProperties.getFileAsInputStream(props, "jaas.properties.file", null);
assertNotNull(inStr);
// configured file from file system path
props.setProperty("jaas.properties.file", "src/test/resources/atlas-jaas.properties");
inStr = ApplicationProperties.getFileAsInputStream(props, "jaas.properties.file", null);
assertNotNull(inStr);
// default file as class loader resource
inStr = ApplicationProperties.getFileAsInputStream(props, "property.not.specified.in.config", "atlas-jaas.properties");
assertNotNull(inStr);
// default file relative to working directory
inStr = ApplicationProperties.getFileAsInputStream(props, "property.not.specified.in.config", "src/test/resources/atlas-jaas.properties");
assertNotNull(inStr);
// default file relative to atlas configuration directory
String originalConfDirSetting = System.setProperty(ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY, "src/test/resources");
try {
inStr = ApplicationProperties.getFileAsInputStream(props, "property.not.specified.in.config", "atlas-jaas.properties");
assertNotNull(inStr);
}
finally {
if (originalConfDirSetting != null) {
System.setProperty(ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY, originalConfDirSetting);
}
else {
System.clearProperty(ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY);
}
}
// non-existent property and no default file
try {
ApplicationProperties.getFileAsInputStream(props, "property.not.specified.in.config", null);
fail("Expected " + AtlasException.class.getSimpleName() + " but none thrown");
}
catch (AtlasException e) {
// good
}
// configured file not found in file system or classpath
props.setProperty("jaas.properties.file", "does_not_exist.txt");
try {
ApplicationProperties.getFileAsInputStream(props, "jaas.properties.file", null);
fail("Expected " + AtlasException.class.getSimpleName() + " but none thrown");
}
catch (AtlasException e) {
// good
}
}
}
common/src/test/resources/test.properties 0 → 100644
jaas.properties.file=atlas-jaas.properties
\ No newline at end of file
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1539 Integration tests in projects which use the typesystem test jar (e.g. webapp) can now be run successfully when invoked in the project directory (dkantor)
ATLAS-1542 Atlas server fails to start if duplicate types are found during Typesystem bootstrap (svimal2106)
ATLAS-1535 Some webapp tests are failing due to a stale Titan transaction (jnhagelberg)
ATLAS-1401 Document in detail how to set up Eclipse for Atlas dev environment
......
typesystem/src/test/resources/atlas-application.properties
......@@ -127,9 +127,9 @@ atlas.server.ha.enabled=false
#atlas.server.address.id1=localhost:21000
#########POLICY FILE PATH #########
atlas.auth.policy.file=${sys:user.dir}/distro/src/conf/policy-store.txt
# atlas.auth.policy.file=policy-store.txt
atlas.authentication.method.file=true
atlas.authentication.method.ldap.type=none
atlas.authentication.method.file.filename=${sys:user.dir}/distro/src/conf/users-credentials.properties
# atlas.authentication.method.file.filename=users-credentials.properties
atlas.authentication.method.kerberos=false
typesystem/src/test/resources/policy-store.txt 0 → 100644
##Policy Format
##r-READ, w-WRITE, u-UPDATE, d-DELETE
##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name
##
adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:*,taxonomy:*,term:*
dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:*,taxonomy:*,term:*
dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:*,taxonomy:*,term:*
hadoopPolicy;;;;hadoop:rwud;;type:*,entity:*,operation:*,taxonomy:*,term:*
rangerTagSyncPolicy;;;;RANGER_TAG_SYNC:r;;type:*,entity:*
typesystem/src/test/resources/users-credentials.properties 0 → 100644
#username=group::sha256-password
admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
rangertagsync=RANGER_TAG_SYNC::e3f67240f5117d1753c940dae9eea772d36ed5fe9bd9c94a300e40413f1afb9d
webapp/src/main/java/org/apache/atlas/web/dao/UserDao.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.web.dao;
import com.google.common.annotations.VisibleForTesting;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.IOException;
import java.util.ArrayList;
......@@ -44,6 +44,8 @@ import org.springframework.util.StringUtils;
@Repository
public class UserDao {
private static final String DEFAULT_USER_CREDENTIALS_PROPERTIES = "users-credentials.properties";
private static final Logger LOG = LoggerFactory.getLogger(UserDao.class);
private Properties userLogins;
......@@ -54,24 +56,14 @@ public class UserDao {
}
void loadFileLoginsDetails() {
String PROPERTY_FILE_PATH = null;
InputStream inStr = null;
try {
Configuration configuration = ApplicationProperties.get();
PROPERTY_FILE_PATH = configuration
.getString("atlas.authentication.method.file.filename");
if (PROPERTY_FILE_PATH != null && !"".equals(PROPERTY_FILE_PATH)) {
userLogins = new Properties();
inStr = new FileInputStream(PROPERTY_FILE_PATH);
userLogins.load(inStr);
}else {
LOG.error("Error while reading user.properties file, filepath={}", PROPERTY_FILE_PATH);
}
inStr = ApplicationProperties.getFileAsInputStream(configuration, "atlas.authentication.method.file.filename", DEFAULT_USER_CREDENTIALS_PROPERTIES);
userLogins = new Properties();
userLogins.load(inStr);
} catch (IOException | AtlasException e) {
LOG.error("Error while reading user.properties file, filepath={}", PROPERTY_FILE_PATH, e);
LOG.error("Error while reading user.properties file", e);
throw new RuntimeException(e);
} finally {
if(inStr != null) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment