From 4e6fdd12e26273b15c081400b60791e9d1bd6472 Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Tue, 26 Jul 2016 17:09:25 -0400 Subject: [PATCH 01/10] Adds targets config file --- lib/spack/spack/config.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e2e7dbc0ee4..31f0eb3a560 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -266,6 +266,19 @@ ], }, },},},},},}, + 'targets': { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack target configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'targets:?': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { # target name + 'type': 'string' ,},},},},}, 'modules': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack module file configuration file schema', From b1e6c58ff2de8ff5d729dd9a10e28bec41aebeeb Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Tue, 26 Jul 2016 17:11:01 -0400 Subject: [PATCH 02/10] Adds __str__ method to CNL operating system class. --- lib/spack/spack/operating_systems/cnl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index c160a60be82..5ee8599e1de 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -19,6 +19,8 @@ def __init__(self): version = '10' super(Cnl, self).__init__(name, version) + def __str__(self): + return self.name def find_compilers(self, *paths): types = spack.compilers.all_compiler_types() From 3040381f03d962513d49406a6c45e12a952f3bc6 Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Tue, 26 Jul 2016 17:11:32 -0400 Subject: [PATCH 03/10] Front-end unification for Cray systems. A platform to generically cover all Cray systems is introduced to avoid having specific platforms for each of XK (XE, XT), XC, and future systems using CrayPE and CNL. The platform searches for 'front_end' and 'back_end' targets, in order, from: * Environment variables 'SPACK_FRONT_END' and 'SPACK_BACK_END' * A spack configuration file 'targets.yaml' * Parsing `/etc/bash.bashrc.local` for first loaded CrayPE CPU target. If a back_end target is not found through one of these methods, an exception is raised. Otherwise, the back_end is set as the default target. The shell init script search is based on recommendations in Crayports case #144359. No automagic way of determining front_end targets has been found (to date) so if a front_end is not specified through configuration, it is ignored which should leave a spack instance in a consistant state. --- lib/spack/spack/platforms/cray.py | 87 +++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 lib/spack/spack/platforms/cray.py diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py new file mode 100644 index 00000000000..279009be5f7 --- /dev/null +++ b/lib/spack/spack/platforms/cray.py @@ -0,0 +1,87 @@ +import os +import re +import platform as py_platform +#from subprocess import check_output +import spack.config +from spack.util.executable import which +from spack.architecture import Platform, Target, NoPlatformError +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl + + +# Craype- module prefixes that are not valid CPU targets. +NON_TARGETS = ('hugepages', 'network', 'target', 'accel', 'xtpe') + + +def _target_from_init(name): + matches = [] + if name != 'front_end': + pattern = 'craype-(?!{0})(\S*)'.format('|'.join(NON_TARGETS)) + with open('/etc/bash.bashrc.local', 'r') as conf: + for line in conf: + if re.search('^[^\#]*module[\s]*(?:add|load)', line): + matches.extend(re.findall(pattern, line)) + return matches[0] if matches else None + + +class Cray(Platform): + priority = 10 + + def __init__(self): + ''' Create a Cray system platform. + + Target names should use craype target names but not include the + 'craype-' prefix. Uses first viable target from: + self + envars [SPACK_FRONT_END, SPACK_BACK_END] + configuration file "targets.yaml" with keys 'front_end', 'back_end' + scanning /etc/bash/bashrc.local for back_end only + ''' + super(Cray, self).__init__('cray') + + # Get targets from config or make best guess from environment: + conf = spack.config.get_config('targets') + for name in ('front_end', 'back_end'): + _target = getattr(self, name, None) + if _target is None: + _target = os.environ.get('SPACK_' + name.upper()) + if _target is None: + _target = conf.get(name) + if _target is None: + _target = _target_from_init(name) + setattr(self, name, _target) + + if _target is not None: + self.add_target(name, Target(_target, 'craype-' + _target)) + self.add_target(_target, Target(_target, 'craype-' + _target)) + + if self.back_end is not None: + self.default = self.back_end + self.add_target('default', Target(self.default, 'craype-' + self.default)) + else: + raise NoPlatformError() + + front_distro = LinuxDistro() + back_distro = Cnl() + + self.default_os = str(back_distro) + self.back_os = self.default_os + self.front_os = str(front_distro) + + self.add_operating_system(self.back_os, back_distro) + self.add_operating_system(self.front_os, front_distro) + + @classmethod + def setup_platform_environment(self, pkg, env): + """ Change the linker to default dynamic to be more + similar to linux/standard linker behavior + """ + env.set('CRAYPE_LINK_TYPE', 'dynamic') + cray_wrapper_names = join_path(spack.build_env_path, 'cray') + if os.path.isdir(cray_wrapper_names): + env.prepend_path('PATH', cray_wrapper_names) + env.prepend_path('SPACK_ENV_PATHS', cray_wrapper_names) + + @classmethod + def detect(self): + return os.environ.get('CRAYPE_VERSION') is not None From eab56b71bee2cc4ef75fdd68a2398deb41203beb Mon Sep 17 00:00:00 2001 From: robertdfrench Date: Wed, 27 Jul 2016 13:11:24 -0400 Subject: [PATCH 04/10] PEP8 Goodness --- lib/spack/spack/operating_systems/cnl.py | 22 +++++++++++++--------- lib/spack/spack/platforms/cray.py | 12 +++++------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 5ee8599e1de..dbd27758617 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -7,11 +7,12 @@ from spack.util.multiproc import parmap import spack.compilers + class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC series super computers. It is a very stripped down version of GNU/Linux. Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are + modules. If updated, user must make sure that version and name are updated to indicate that OS has been upgraded (or downgraded) """ def __init__(self): @@ -24,14 +25,14 @@ def __str__(self): def find_compilers(self, *paths): types = spack.compilers.all_compiler_types() - compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) + compiler_lists = parmap( + lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) # ensure all the version calls we made are cached in the parent # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) + clist = reduce(lambda x, y: x + y, compiler_lists) return clist - def find_compiler(self, cmp_cls, *paths): compilers = [] if cmp_cls.PrgEnv: @@ -47,13 +48,16 @@ def find_compiler(self, cmp_cls, *paths): if paths: module_paths = ':' + ':'.join(p for p in paths) os.environ['MODULEPATH'] = module_paths - - output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str) - matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) + + output = modulecmd( + 'avail', cmp_cls.PrgEnv_compiler, output=str, error=str) + matches = re.findall( + r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) for name, version in matches: v = version - comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self, - ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v]) + comp = cmp_cls( + spack.spec.CompilerSpec(name + '@' + v), self, + ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name + '/' + v]) compilers.append(comp) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 279009be5f7..79a53d887cc 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -1,9 +1,6 @@ import os import re -import platform as py_platform -#from subprocess import check_output import spack.config -from spack.util.executable import which from spack.architecture import Platform, Target, NoPlatformError from spack.operating_systems.linux_distro import LinuxDistro from spack.operating_systems.cnl import Cnl @@ -20,7 +17,7 @@ def _target_from_init(name): with open('/etc/bash.bashrc.local', 'r') as conf: for line in conf: if re.search('^[^\#]*module[\s]*(?:add|load)', line): - matches.extend(re.findall(pattern, line)) + matches.extend(re.findall(pattern, line)) return matches[0] if matches else None @@ -29,7 +26,7 @@ class Cray(Platform): def __init__(self): ''' Create a Cray system platform. - + Target names should use craype target names but not include the 'craype-' prefix. Uses first viable target from: self @@ -50,14 +47,15 @@ def __init__(self): if _target is None: _target = _target_from_init(name) setattr(self, name, _target) - + if _target is not None: self.add_target(name, Target(_target, 'craype-' + _target)) self.add_target(_target, Target(_target, 'craype-' + _target)) if self.back_end is not None: self.default = self.back_end - self.add_target('default', Target(self.default, 'craype-' + self.default)) + self.add_target( + 'default', Target(self.default, 'craype-' + self.default)) else: raise NoPlatformError() From 5e97eb5ec4c49f5199b47d4b43fb1584046785e4 Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Fri, 29 Jul 2016 12:36:50 -0400 Subject: [PATCH 05/10] Obtains default modules from a clean subshell. The list of default environment modules is obtained by calling `module list -lt` from a subshell with a wiped environment. This allows `/etc/profile` and other init scripts to be fully sourced which should generally include loading the default modules. The list of default modules is then parsed for the first acceptable CPU target, assumed to be the back_end target. --- lib/spack/spack/platforms/cray.py | 37 ++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 79a53d887cc..dc959c7eafe 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -1,6 +1,7 @@ import os import re import spack.config +from spack.util.executable import which from spack.architecture import Platform, Target, NoPlatformError from spack.operating_systems.linux_distro import LinuxDistro from spack.operating_systems.cnl import Cnl @@ -10,15 +11,35 @@ NON_TARGETS = ('hugepages', 'network', 'target', 'accel', 'xtpe') -def _target_from_init(name): - matches = [] +def _target_from_clean_env(name): + '''Return the default back_end target as loaded in a clean login session. + + A bash subshell is launched with a wiped environment and the list of loaded + modules is parsed for the first acceptable CrayPE target. + ''' + # Based on the incantation: + # echo "$(env - USER=$USER /bin/bash -l -c 'module list -lt')" + default_modules = [] + targets = [] if name != 'front_end': + env = which('env') + env.add_default_arg('-') + # CAUTION - $USER is generally needed to initialize the environment. + # There may be other variables needed for general success. + output = env('USER=%s' % os.environ['USER'], + '/bin/bash', '-l', '-c', 'module list -lt', + output=str, error=str) pattern = 'craype-(?!{0})(\S*)'.format('|'.join(NON_TARGETS)) - with open('/etc/bash.bashrc.local', 'r') as conf: - for line in conf: - if re.search('^[^\#]*module[\s]*(?:add|load)', line): - matches.extend(re.findall(pattern, line)) - return matches[0] if matches else None + for line in output.splitlines(): + if 'craype-' in line: + targets.extend(re.findall(pattern, line)) + if len(line.split()) == 1: + default_modules.append(line) + # if default_modules: + # print 'Found default modules:' + # for defmod in default_modules: + # print ' ', defmod + return targets[0] if targets else None class Cray(Platform): @@ -45,7 +66,7 @@ def __init__(self): if _target is None: _target = conf.get(name) if _target is None: - _target = _target_from_init(name) + _target = _target_from_clean_env(name) setattr(self, name, _target) if _target is not None: From dc7e0899a05cf4464552f4b3d41005e79571a15a Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Sat, 30 Jul 2016 11:38:34 -0400 Subject: [PATCH 06/10] Invokes subshell without user init scripts. --- lib/spack/spack/platforms/cray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index dc959c7eafe..fd58915c575 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -27,7 +27,8 @@ def _target_from_clean_env(name): # CAUTION - $USER is generally needed to initialize the environment. # There may be other variables needed for general success. output = env('USER=%s' % os.environ['USER'], - '/bin/bash', '-l', '-c', 'module list -lt', + '/bin/sh', '--noprofile', '-c', + 'source /etc/profile; module list -lt', output=str, error=str) pattern = 'craype-(?!{0})(\S*)'.format('|'.join(NON_TARGETS)) for line in output.splitlines(): From aaa5c9e8a473cccbe3b5e7258bc5f9b7ff4d8c6c Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Mon, 1 Aug 2016 12:05:29 -0400 Subject: [PATCH 07/10] Writes default module list to terminal when debugging. --- lib/spack/spack/platforms/cray.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index fd58915c575..d43580df06d 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -1,6 +1,7 @@ import os import re import spack.config +import llnl.util.tty as tty from spack.util.executable import which from spack.architecture import Platform, Target, NoPlatformError from spack.operating_systems.linux_distro import LinuxDistro @@ -19,7 +20,6 @@ def _target_from_clean_env(name): ''' # Based on the incantation: # echo "$(env - USER=$USER /bin/bash -l -c 'module list -lt')" - default_modules = [] targets = [] if name != 'front_end': env = which('env') @@ -30,16 +30,14 @@ def _target_from_clean_env(name): '/bin/sh', '--noprofile', '-c', 'source /etc/profile; module list -lt', output=str, error=str) + default_modules = [i for i in output.splitlines() + if len(i.split()) == 1] + tty.debug("Found default modules:", + *[" " + mod for mod in default_modules]) pattern = 'craype-(?!{0})(\S*)'.format('|'.join(NON_TARGETS)) - for line in output.splitlines(): - if 'craype-' in line: - targets.extend(re.findall(pattern, line)) - if len(line.split()) == 1: - default_modules.append(line) - # if default_modules: - # print 'Found default modules:' - # for defmod in default_modules: - # print ' ', defmod + for mod in default_modules: + if 'craype-' in mod: + targets.extend(re.findall(pattern, mod)) return targets[0] if targets else None From e864d2764171b2dbd539bac16dbb7a28910407a9 Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Mon, 1 Aug 2016 17:03:50 -0400 Subject: [PATCH 08/10] Replaces bash-ism `source` for POSIX-compliant `.` Change is made in order to use `/bin/sh` on systems where `/bin/sh` is not simply an alias for `/bin/bash --norc`. --- lib/spack/spack/platforms/cray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index d43580df06d..68f1453edf1 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -28,7 +28,7 @@ def _target_from_clean_env(name): # There may be other variables needed for general success. output = env('USER=%s' % os.environ['USER'], '/bin/sh', '--noprofile', '-c', - 'source /etc/profile; module list -lt', + '. /etc/profile; module list -lt', output=str, error=str) default_modules = [i for i in output.splitlines() if len(i.split()) == 1] From ae167c09fc4d3dc4b4ff90840bd70cd8648261ab Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Mon, 1 Aug 2016 17:17:24 -0400 Subject: [PATCH 09/10] Prefers `bash` over `sh`. Assuming a bash interactive environment will be correctly formed on login, we should prefer to probe the environment using a shell that reports itself as `bash` instead of `sh` which may not source files that set the environment modules in statements like: ``` case "$is" in bash) test -s /etc/bash.bashrc.local && . /etc/bash.bashrc.local ;; ksh) test -s /etc/ksh.kshrc.local && . /etc/ksh.kshrc.local ;; zsh) test -s /etc/zsh.zshrc.local && . /etc/zsh.zshrc.local ;; ash) test -s /etc/ash.ashrc.local && . /etc/ash.ashrc.local esac test -s /etc/sh.shrc.local && . /etc/sh.shrc.local ``` --- lib/spack/spack/platforms/cray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 68f1453edf1..2883a35ec7b 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -27,7 +27,7 @@ def _target_from_clean_env(name): # CAUTION - $USER is generally needed to initialize the environment. # There may be other variables needed for general success. output = env('USER=%s' % os.environ['USER'], - '/bin/sh', '--noprofile', '-c', + '/bin/bash', '--noprofile', '--norc', '-c', '. /etc/profile; module list -lt', output=str, error=str) default_modules = [i for i in output.splitlines() From 661708b7facbcc4c4fab7b7592e605e1d63de7b4 Mon Sep 17 00:00:00 2001 From: Matt Belhorn Date: Mon, 1 Aug 2016 17:27:33 -0400 Subject: [PATCH 10/10] Fixes Flake8 indentation errors. --- lib/spack/spack/platforms/cray.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 2883a35ec7b..2bd2a404634 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -27,9 +27,9 @@ def _target_from_clean_env(name): # CAUTION - $USER is generally needed to initialize the environment. # There may be other variables needed for general success. output = env('USER=%s' % os.environ['USER'], - '/bin/bash', '--noprofile', '--norc', '-c', - '. /etc/profile; module list -lt', - output=str, error=str) + '/bin/bash', '--noprofile', '--norc', '-c', + '. /etc/profile; module list -lt', + output=str, error=str) default_modules = [i for i in output.splitlines() if len(i.split()) == 1] tty.debug("Found default modules:",