ATLAS-1539 Integration tests in projects which use the typesystem test jar (e.g.…

authorization/src/main/java/org/apache/atlas/authorize/simple/FileReaderUtil.java

@@ -19,13 +19,13 @@
 package org.apache.atlas.authorize.simple;
 
 import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Paths;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Pattern;
 
+import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,12 +33,12 @@ public class FileReaderUtil {
     private static Logger LOG = LoggerFactory.getLogger(FileReaderUtil.class);
     private static boolean isDebugEnabled = LOG.isDebugEnabled();
 
-    public static List<String> readFile(String path) throws IOException {
+    public static List<String> readFile(InputStream policyStoreStream) throws IOException {
         if (isDebugEnabled) {
-            LOG.debug("==> FileReaderUtil readFile({})", path);
+            LOG.debug("==> FileReaderUtil readFile()");
         }
         List<String> list = new ArrayList<>();
-        List<String> fileLines = Files.readAllLines(Paths.get(path), Charset.forName("UTF-8"));
+        List<String> fileLines = IOUtils.readLines(policyStoreStream, StandardCharsets.UTF_8);
         if (fileLines != null) {
             for (String line : fileLines) {
                 if ((!line.startsWith("#")) && Pattern.matches(".+;;.*;;.*;;.+", line))
@@ -47,7 +47,7 @@ public class FileReaderUtil {
         }
 
         if (isDebugEnabled) {
-            LOG.debug("<== FileReaderUtil readFile({})", path);
+            LOG.debug("<== FileReaderUtil readFile()");
             LOG.debug("Policies read :: " + list);
         }
 

authorization/src/main/java/org/apache/atlas/authorize/simple/PolicyUtil.java

@@ -33,7 +33,7 @@ public class PolicyUtil {
     private static boolean isDebugEnabled = LOG.isDebugEnabled();
 
 
-    public Map<String, Map<AtlasResourceTypes, List<String>>> createPermissionMap(List<PolicyDef> policyDefList,
+    public static Map<String, Map<AtlasResourceTypes, List<String>>> createPermissionMap(List<PolicyDef> policyDefList,
         AtlasActionTypes permissionType, SimpleAtlasAuthorizer.AtlasAccessorTypes principalType) {
         if (isDebugEnabled) {
             LOG.debug("==> PolicyUtil createPermissionMap\nCreating Permission Map for :: {} & {}", permissionType, principalType);
@@ -104,5 +104,4 @@ public class PolicyUtil {
         return userReadMap;
 
     }
-
 }

authorization/src/main/java/org/apache/atlas/authorize/simple/SimpleAtlasAuthorizer.java

@@ -19,6 +19,7 @@
 package org.apache.atlas.authorize.simple;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -33,7 +34,6 @@ import org.apache.atlas.authorize.AtlasAuthorizer;
 import org.apache.atlas.authorize.AtlasResourceTypes;
 import org.apache.atlas.utils.PropertiesUtil;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.configuration.Configuration;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOCase;
 import org.apache.commons.lang.StringUtils;
@@ -44,7 +44,7 @@ import com.google.common.annotations.VisibleForTesting;
 
 public final class SimpleAtlasAuthorizer implements AtlasAuthorizer {
 
-	public enum AtlasAccessorTypes {
+    public enum AtlasAccessorTypes {
         USER, GROUP
     }
 
@@ -74,7 +74,6 @@ public final class SimpleAtlasAuthorizer implements AtlasAuthorizer {
         }
         try {
 
-            PolicyUtil util = new PolicyUtil();
             PolicyParser parser = new PolicyParser();
             optIgnoreCase = Boolean.valueOf(PropertiesUtil.getProperty("optIgnoreCase", "false"));
 
@@ -82,25 +81,25 @@ public final class SimpleAtlasAuthorizer implements AtlasAuthorizer {
                 LOG.debug("Read from PropertiesUtil --> optIgnoreCase :: {}", optIgnoreCase);
             }
 
-            Configuration configuration = ApplicationProperties.get();
-            String policyStorePath = configuration.getString("atlas.auth.policy.file", System.getProperty("atlas.conf")+"/policy-store.txt");
-
-            if (isDebugEnabled) {
-                LOG.debug("Loading Apache Atlas policies from : {}", policyStorePath);
+            InputStream policyStoreStream = ApplicationProperties.getFileAsInputStream(ApplicationProperties.get(), "atlas.auth.policy.file", "policy-store.txt");
+            List<String> policies = null;
+            try {
+                policies = FileReaderUtil.readFile(policyStoreStream);
+            }
+            finally {
+                policyStoreStream.close();
             }
-
-            List<String> policies = FileReaderUtil.readFile(policyStorePath);
             List<PolicyDef> policyDef = parser.parsePolicies(policies);
 
-            userReadMap = util.createPermissionMap(policyDef, AtlasActionTypes.READ, AtlasAccessorTypes.USER);
-            userWriteMap = util.createPermissionMap(policyDef, AtlasActionTypes.CREATE, AtlasAccessorTypes.USER);
-            userUpdateMap = util.createPermissionMap(policyDef, AtlasActionTypes.UPDATE, AtlasAccessorTypes.USER);
-            userDeleteMap = util.createPermissionMap(policyDef, AtlasActionTypes.DELETE, AtlasAccessorTypes.USER);
+            userReadMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.READ, AtlasAccessorTypes.USER);
+            userWriteMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.CREATE, AtlasAccessorTypes.USER);
+            userUpdateMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.UPDATE, AtlasAccessorTypes.USER);
+            userDeleteMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.DELETE, AtlasAccessorTypes.USER);
 
-            groupReadMap = util.createPermissionMap(policyDef, AtlasActionTypes.READ, AtlasAccessorTypes.GROUP);
-            groupWriteMap = util.createPermissionMap(policyDef, AtlasActionTypes.CREATE, AtlasAccessorTypes.GROUP);
-            groupUpdateMap = util.createPermissionMap(policyDef, AtlasActionTypes.UPDATE, AtlasAccessorTypes.GROUP);
-            groupDeleteMap = util.createPermissionMap(policyDef, AtlasActionTypes.DELETE, AtlasAccessorTypes.GROUP);
+            groupReadMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.READ, AtlasAccessorTypes.GROUP);
+            groupWriteMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.CREATE, AtlasAccessorTypes.GROUP);
+            groupUpdateMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.UPDATE, AtlasAccessorTypes.GROUP);
+            groupDeleteMap = PolicyUtil.createPermissionMap(policyDef, AtlasActionTypes.DELETE, AtlasAccessorTypes.GROUP);
 
             if (isDebugEnabled) {
                 LOG.debug("\n\nUserReadMap :: {}\nGroupReadMap :: {}", userReadMap, groupReadMap);

common/src/main/java/org/apache/atlas/ApplicationProperties.java

@@ -25,6 +25,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
 import java.net.URL;
 import java.util.Iterator;
 
@@ -32,6 +35,8 @@ import java.util.Iterator;
  * Application properties used by Atlas.
  */
 public final class ApplicationProperties extends PropertiesConfiguration {
+    public static final String ATLAS_CONFIGURATION_DIRECTORY_PROPERTY = "atlas.conf";
+
     private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class);
 
     public static final String APPLICATION_PROPERTIES = "atlas-application.properties";
@@ -65,7 +70,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
     }
 
     public static Configuration get(String fileName) throws AtlasException {
-        String confLocation = System.getProperty("atlas.conf");
+        String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY);
         try {
             URL url = null;
 
@@ -125,4 +130,74 @@ public final class ApplicationProperties extends PropertiesConfiguration {
             throw new AtlasException(e);
         }
     }
+
+    /**
+     * Get the specified property as an {@link InputStream}.
+     * If the property is not set, then the specified default filename
+     * is searched for in the following locations, in order of precedence:
+     * 1. Atlas configuration directory specified by the {@link #ATLAS_CONFIGURATION_DIRECTORY_PROPERTY} system property
+     * 2. relative to the working directory if {@link #ATLAS_CONFIGURATION_DIRECTORY_PROPERTY} is not set
+     * 3. as a classloader resource
+     *
+     * @param configuration
+     * @param propertyName
+     * @param defaultFileName name of file to use by default if specified property is not set in the configuration- if null,
+     * an {@link AtlasException} is thrown if the property is not set
+     * @return an {@link InputStream}
+     * @throws AtlasException if no file was found or if there was an error loading the file
+     */
+    public static InputStream getFileAsInputStream(Configuration configuration, String propertyName, String defaultFileName) throws AtlasException {
+        File fileToLoad = null;
+        String fileName = configuration.getString(propertyName);
+        if (fileName == null) {
+            if (defaultFileName == null) {
+                throw new AtlasException(propertyName + " property not set and no default value specified");
+            }
+            fileName = defaultFileName;
+            String atlasConfDir = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY);
+            if (atlasConfDir != null) {
+                // Look for default filename in Atlas config directory
+                fileToLoad = new File(atlasConfDir, fileName);
+            }
+            else {
+                // Look for default filename under the working directory
+                fileToLoad = new File(fileName);
+            }
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("{} property not set - defaulting to {}", propertyName, fileToLoad.getPath());
+            }
+        }
+        else {
+            // Look for configured filename
+            fileToLoad = new File(fileName);
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Using {} property setting: {}", propertyName, fileToLoad.getPath());
+            }
+        }
+
+        InputStream inStr = null;
+        if (fileToLoad.exists()) {
+            try {
+                inStr = new FileInputStream(fileToLoad);
+            } catch (FileNotFoundException e) {
+                throw new AtlasException("Error loading file " + fileName, e);
+            }
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Loaded file from : {}", fileToLoad.getPath());
+            }
+        }
+        else {
+            // Look for file as class loader resource
+            inStr = Thread.currentThread().getContextClassLoader().getResourceAsStream(fileName);
+            if (inStr == null) {
+                String msg = fileName + " not found in file system or as class loader resource";
+                LOG.error(msg);
+                throw new AtlasException(msg);
+            }
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Loaded {} as resource from : {}", fileName, Thread.currentThread().getContextClassLoader().getResource(fileName).toString());
+            }
+        }
+        return inStr;
+    }
 }

common/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas;
+
+import java.io.InputStream;
+
+import org.apache.commons.configuration.Configuration;
+import org.testng.annotations.Test;
+import static org.testng.Assert.*;
+
+/**
+ * Unit test for {@link ApplicationProperties}
+ *
+ */
+public class ApplicationPropertiesTest {
+
+    @Test
+    public void testGetFileAsInputStream() throws Exception {
+        Configuration props = ApplicationProperties.get("test.properties");
+
+        // configured file as class loader resource
+        InputStream inStr = ApplicationProperties.getFileAsInputStream(props, "jaas.properties.file", null);
+        assertNotNull(inStr);
+
+        // configured file from file system path
+        props.setProperty("jaas.properties.file", "src/test/resources/atlas-jaas.properties");
+        inStr = ApplicationProperties.getFileAsInputStream(props, "jaas.properties.file", null);
+        assertNotNull(inStr);
+
+        // default file as class loader resource
+        inStr = ApplicationProperties.getFileAsInputStream(props, "property.not.specified.in.config", "atlas-jaas.properties");
+        assertNotNull(inStr);
+
+        // default file relative to working directory
+        inStr = ApplicationProperties.getFileAsInputStream(props, "property.not.specified.in.config", "src/test/resources/atlas-jaas.properties");
+        assertNotNull(inStr);
+
+        // default file relative to atlas configuration directory
+        String originalConfDirSetting = System.setProperty(ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY, "src/test/resources");
+        try {
+            inStr = ApplicationProperties.getFileAsInputStream(props, "property.not.specified.in.config", "atlas-jaas.properties");
+            assertNotNull(inStr);
+        }
+        finally {
+            if (originalConfDirSetting != null) {
+                System.setProperty(ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY, originalConfDirSetting);
+            }
+            else {
+                System.clearProperty(ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY);
+            }
+        }
+
+        // non-existent property and no default file
+        try {
+            ApplicationProperties.getFileAsInputStream(props, "property.not.specified.in.config", null);
+            fail("Expected " + AtlasException.class.getSimpleName() + " but none thrown");
+        }
+        catch (AtlasException e) {
+            // good
+        }
+
+        // configured file not found in file system or classpath
+        props.setProperty("jaas.properties.file", "does_not_exist.txt");
+        try {
+            ApplicationProperties.getFileAsInputStream(props, "jaas.properties.file", null);
+            fail("Expected " + AtlasException.class.getSimpleName() + " but none thrown");
+        }
+        catch (AtlasException e) {
+            // good
+        }
+    }
+}

common/src/test/resources/test.properties

+jaas.properties.file=atlas-jaas.properties
\ No newline at end of file

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1539 Integration tests in projects which use the typesystem test jar (e.g. webapp) can now be run successfully when invoked in the project directory (dkantor)
 ATLAS-1542 Atlas server fails to start if duplicate types are found during Typesystem bootstrap (svimal2106)
 ATLAS-1535 Some webapp tests are failing due to a stale Titan transaction (jnhagelberg)
 ATLAS-1401 Document in detail how to set up Eclipse for Atlas dev environment

typesystem/src/test/resources/atlas-application.properties

@@ -127,9 +127,9 @@ atlas.server.ha.enabled=false
 #atlas.server.address.id1=localhost:21000
 
 #########POLICY FILE PATH #########
-atlas.auth.policy.file=${sys:user.dir}/distro/src/conf/policy-store.txt
+# atlas.auth.policy.file=policy-store.txt
 
 atlas.authentication.method.file=true
 atlas.authentication.method.ldap.type=none
-atlas.authentication.method.file.filename=${sys:user.dir}/distro/src/conf/users-credentials.properties
+# atlas.authentication.method.file.filename=users-credentials.properties
 atlas.authentication.method.kerberos=false

typesystem/src/test/resources/policy-store.txt

+##Policy Format
+##r-READ, w-WRITE, u-UPDATE, d-DELETE
+##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name
+##
+adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:*,taxonomy:*,term:*
+dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:*,taxonomy:*,term:*
+dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:*,taxonomy:*,term:*
+hadoopPolicy;;;;hadoop:rwud;;type:*,entity:*,operation:*,taxonomy:*,term:*
+rangerTagSyncPolicy;;;;RANGER_TAG_SYNC:r;;type:*,entity:*

typesystem/src/test/resources/users-credentials.properties

+#username=group::sha256-password
+admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
+rangertagsync=RANGER_TAG_SYNC::e3f67240f5117d1753c940dae9eea772d36ed5fe9bd9c94a300e40413f1afb9d

webapp/src/main/java/org/apache/atlas/web/dao/UserDao.java

@@ -18,7 +18,7 @@
 package org.apache.atlas.web.dao;
 
 import com.google.common.annotations.VisibleForTesting;
-import java.io.FileInputStream;
+
 import java.io.InputStream;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,6 +44,8 @@ import org.springframework.util.StringUtils;
 @Repository
 public class UserDao {
 
+    private static final String DEFAULT_USER_CREDENTIALS_PROPERTIES = "users-credentials.properties";
+
     private static final Logger LOG = LoggerFactory.getLogger(UserDao.class);
 
     private Properties userLogins;
@@ -54,24 +56,14 @@ public class UserDao {
     }
 
     void loadFileLoginsDetails() {
-        String PROPERTY_FILE_PATH = null;
         InputStream inStr = null;
-
         try {
-
             Configuration configuration = ApplicationProperties.get();
-            PROPERTY_FILE_PATH = configuration
-                    .getString("atlas.authentication.method.file.filename");
-            if (PROPERTY_FILE_PATH != null && !"".equals(PROPERTY_FILE_PATH)) {
-                userLogins = new Properties();
-                inStr = new FileInputStream(PROPERTY_FILE_PATH);
-                userLogins.load(inStr);
-            }else {
-                LOG.error("Error while reading user.properties file, filepath={}", PROPERTY_FILE_PATH);
-            }
-
+            inStr = ApplicationProperties.getFileAsInputStream(configuration, "atlas.authentication.method.file.filename", DEFAULT_USER_CREDENTIALS_PROPERTIES);
+            userLogins = new Properties();
+            userLogins.load(inStr);
         } catch (IOException | AtlasException e) {
-            LOG.error("Error while reading user.properties file, filepath={}", PROPERTY_FILE_PATH, e);
+            LOG.error("Error while reading user.properties file", e);
             throw new RuntimeException(e);
         } finally {
             if(inStr != null) {