Merge pull request #1047 from epfl-scitas/source_diffs_in_environment_modifications

feature : construct environment modifications sourcing files

lib/spack/spack/environment.py

@@ -1,4 +1,4 @@
-##############################################################################
+#
 # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory.
 #
@@ -21,14 +21,17 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
+#
 import collections
 import inspect
+import json
 import os
 import os.path
+import subprocess
 
 
 class NameModifier(object):
+
     def __init__(self, name, **kwargs):
         self.name = name
         self.args = {'name': name}
@@ -36,6 +39,7 @@ def __init__(self, name, **kwargs):
 
 
 class NameValueModifier(object):
+
     def __init__(self, name, value, **kwargs):
         self.name = name
         self.value = value
@@ -45,23 +49,27 @@ def __init__(self, name, value, **kwargs):
 
 
 class SetEnv(NameValueModifier):
+
     def execute(self):
         os.environ[self.name] = str(self.value)
 
 
 class UnsetEnv(NameModifier):
+
     def execute(self):
         # Avoid throwing if the variable was not set
         os.environ.pop(self.name, None)
 
 
 class SetPath(NameValueModifier):
+
     def execute(self):
         string_path = concatenate_paths(self.value, separator=self.separator)
         os.environ[self.name] = string_path
 
 
 class AppendPath(NameValueModifier):
+
     def execute(self):
         environment_value = os.environ.get(self.name, '')
         directories = environment_value.split(
@@ -71,6 +79,7 @@ def execute(self):
 
 
 class PrependPath(NameValueModifier):
+
     def execute(self):
         environment_value = os.environ.get(self.name, '')
         directories = environment_value.split(
@@ -80,6 +89,7 @@ def execute(self):
 
 
 class RemovePath(NameValueModifier):
+
     def execute(self):
         environment_value = os.environ.get(self.name, '')
         directories = environment_value.split(
@@ -90,6 +100,7 @@ def execute(self):
 
 
 class EnvironmentModifications(object):
+
     """
     Keeps track of requests to modify the current environment.
 
@@ -240,6 +251,126 @@ def apply_modifications(self):
             for x in actions:
                 x.execute()
 
+    @staticmethod
+    def from_sourcing_files(*args, **kwargs):
+        """
+        Creates an instance of EnvironmentModifications that, if executed,
+        has the same effect on the environment as sourcing the files passed as
+        parameters
+
+        Args:
+            *args: list of files to be sourced
+
+        Returns:
+            instance of EnvironmentModifications
+        """
+        env = EnvironmentModifications()
+        # Check if the files are actually there
+        if not all(os.path.isfile(file) for file in args):
+            raise RuntimeError('trying to source non-existing files')
+        # Relevant kwd parameters and formats
+        info = dict(kwargs)
+        info.setdefault('shell', '/bin/bash')
+        info.setdefault('shell_options', '-c')
+        info.setdefault('source_command', 'source')
+        info.setdefault('suppress_output', '&> /dev/null')
+        info.setdefault('concatenate_on_success', '&&')
+
+        shell = '{shell}'.format(**info)
+        shell_options = '{shell_options}'.format(**info)
+        source_file = '{source_command} {file} {concatenate_on_success}'
+        dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"'  # NOQA: ignore=E501
+        # Construct the command that will be executed
+        command = [source_file.format(file=file, **info) for file in args]
+        command.append(dump_environment)
+        command = ' '.join(command)
+        command = [
+            shell,
+            shell_options,
+            command
+        ]
+
+        # Try to source all the files,
+        proc = subprocess.Popen(
+            command, stdout=subprocess.PIPE, env=os.environ)
+        proc.wait()
+        if proc.returncode != 0:
+            raise RuntimeError('sourcing files returned a non-zero exit code')
+        output = ''.join([line for line in proc.stdout])
+        # Construct a dictionary with all the variables in the new environment
+        after_source_env = dict(json.loads(output))
+        this_environment = dict(os.environ)
+
+        # Filter variables that are not related to sourcing a file
+        to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD'
+        for d in after_source_env, this_environment:
+            for name in to_be_filtered:
+                d.pop(name, None)
+
+        # Fill the EnvironmentModifications instance
+
+        # New variables
+        new_variables = set(after_source_env) - set(this_environment)
+        for x in new_variables:
+            env.set(x, after_source_env[x])
+        # Variables that have been unset
+        unset_variables = set(this_environment) - set(after_source_env)
+        for x in unset_variables:
+            env.unset(x)
+        # Variables that have been modified
+        common_variables = set(this_environment).intersection(set(after_source_env))  # NOQA: ignore=E501
+        modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]]  # NOQA: ignore=E501
+
+        def return_separator_if_any(first_value, second_value):
+            separators = ':', ';'
+            for separator in separators:
+                if separator in first_value and separator in second_value:
+                    return separator
+            return None
+
+        for x in modified_variables:
+            current = this_environment[x]
+            modified = after_source_env[x]
+            sep = return_separator_if_any(current, modified)
+            if sep is None:
+                # We just need to set the variable to the new value
+                env.set(x, after_source_env[x])
+            else:
+                current_list = current.split(sep)
+                modified_list = modified.split(sep)
+                # Paths that have been removed
+                remove_list = [
+                    ii for ii in current_list if ii not in modified_list]
+                # Check that nothing has been added in the middle of vurrent
+                # list
+                remaining_list = [
+                    ii for ii in current_list if ii in modified_list]
+                start = modified_list.index(remaining_list[0])
+                end = modified_list.index(remaining_list[-1])
+                search = sep.join(modified_list[start:end + 1])
+                if search not in current:
+                    # We just need to set the variable to the new value
+                    env.set(x, after_source_env[x])
+                    break
+                else:
+                    try:
+                        prepend_list = modified_list[:start]
+                    except KeyError:
+                        prepend_list = []
+                    try:
+                        append_list = modified_list[end + 1:]
+                    except KeyError:
+                        append_list = []
+
+                    for item in remove_list:
+                        env.remove_path(x, item)
+                    for item in append_list:
+                        env.append_path(x, item)
+                    for item in prepend_list:
+                        env.prepend_path(x, item)
+
+        return env
+
 
 def concatenate_paths(paths, separator=':'):
     """

lib/spack/spack/test/data/sourceme_first.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+export NEW_VAR='new'
+export UNSET_ME='overridden'

lib/spack/spack/test/data/sourceme_second.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+export PATH_LIST='/path/first:/path/second:/path/fourth'
+unset EMPTY_PATH_LIST

lib/spack/spack/test/environment.py

@@ -24,16 +24,21 @@
 ##############################################################################
 import unittest
 import os
-import copy
+
+from spack import spack_root
+from llnl.util.filesystem import join_path
 from spack.environment import EnvironmentModifications
+from spack.environment import SetEnv, UnsetEnv
+from spack.environment import RemovePath, PrependPath, AppendPath
 
 
 class EnvironmentTest(unittest.TestCase):
+
     def setUp(self):
         os.environ['UNSET_ME'] = 'foo'
         os.environ['EMPTY_PATH_LIST'] = ''
         os.environ['PATH_LIST'] = '/path/second:/path/third'
-        os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g'
+        os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g'  # NOQA: ignore=E501
 
     def tearDown(self):
         pass
@@ -77,9 +82,18 @@ def test_path_manipulation(self):
         env.remove_path('REMOVE_PATH_LIST', '/duplicate/')
 
         env.apply_modifications()
-        self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST'])
+        self.assertEqual(
-        self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST'])
+            '/path/first:/path/second:/path/third:/path/last',
-        self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST'])
+            os.environ['PATH_LIST']
+        )
+        self.assertEqual(
+            '/path/first:/path/middle:/path/last',
+            os.environ['EMPTY_PATH_LIST']
+        )
+        self.assertEqual(
+            '/path/first:/path/middle:/path/last',
+            os.environ['NEWLY_CREATED_PATH_LIST']
+        )
         self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST'])
 
     def test_extra_arguments(self):
@@ -98,3 +112,41 @@ def test_extend(self):
         self.assertEqual(len(copy_construct), 2)
         for x, y in zip(env, copy_construct):
             assert x is y
+
+    def test_source_files(self):
+        datadir = join_path(spack_root, 'lib', 'spack',
+                            'spack', 'test', 'data')
+        files = [
+            join_path(datadir, 'sourceme_first.sh'),
+            join_path(datadir, 'sourceme_second.sh')
+        ]
+        env = EnvironmentModifications.from_sourcing_files(*files)
+        modifications = env.group_by_name()
+
+        self.assertEqual(len(modifications), 4)
+        # Set new variables
+        self.assertEqual(len(modifications['NEW_VAR']), 1)
+        self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv))
+        self.assertEqual(modifications['NEW_VAR'][0].value, 'new')
+        # Unset variables
+        self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1)
+        self.assertTrue(isinstance(
+            modifications['EMPTY_PATH_LIST'][0], UnsetEnv))
+        # Modified variables
+        self.assertEqual(len(modifications['UNSET_ME']), 1)
+        self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv))
+        self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden')
+
+        self.assertEqual(len(modifications['PATH_LIST']), 3)
+        self.assertTrue(
+            isinstance(modifications['PATH_LIST'][0], RemovePath)
+        )
+        self.assertEqual(modifications['PATH_LIST'][0].value, '/path/third')
+        self.assertTrue(
+            isinstance(modifications['PATH_LIST'][1], AppendPath)
+        )
+        self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth')
+        self.assertTrue(
+            isinstance(modifications['PATH_LIST'][2], PrependPath)
+        )
+        self.assertEqual(modifications['PATH_LIST'][2].value, '/path/first')