zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Python flake8 overhaul

Browse Source
This commit is contained in:
Adam J. Stewart 2016-06-17 13:33:09 -05:00
parent 80d444624b
commit 536fa73d5f
1 changed files with 113 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
var/spack/repos/builtin/packages/python/package.py
View File

@ -22,9 +22,6 @@
# License along with this program; if not, write to the Free Software # License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
import functools
import glob
import inspect
import os import os
import re import re
from contextlib import closing from contextlib import closing
@ -37,11 +34,10 @@
class Python(Package): class Python(Package):
"""The Python programming language.""" """The Python programming language."""
homepage = "http://www.python.org" homepage = "http://www.python.org"
url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz" url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
extendable = True
version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe') version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe')
version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36') version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36')
version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True) version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True)
@ -49,6 +45,8 @@ class Python(Package):
version('2.7.9', '5eebcaa0030dc4061156d3429657fb83') version('2.7.9', '5eebcaa0030dc4061156d3429657fb83')
version('2.7.8', 'd4bca0159acb0b44a781292b5231936f') version('2.7.8', 'd4bca0159acb0b44a781292b5231936f')
extendable = True
depends_on("openssl") depends_on("openssl")
depends_on("bzip2") depends_on("bzip2")
depends_on("readline") depends_on("readline")
@ -64,39 +62,63 @@ def install(self, spec, prefix):
# Rest of install is pretty standard except setup.py needs to # Rest of install is pretty standard except setup.py needs to
# be able to read the CPPFLAGS and LDFLAGS as it scans for the # be able to read the CPPFLAGS and LDFLAGS as it scans for the
# library and headers to build # library and headers to build
configure_args= [ cppflags = ' -I'.join([
"--prefix=%s" % prefix, spec['openssl'].prefix.include, spec['bzip2'].prefix.include,
spec['readline'].prefix.include, spec['ncurses'].prefix.include,
spec['sqlite'].prefix.include, spec['zlib'].prefix.include
])
ldflags = ' -L'.join([
spec['openssl'].prefix.lib, spec['bzip2'].prefix.lib,
spec['readline'].prefix.lib, spec['ncurses'].prefix.lib,
spec['sqlite'].prefix.lib, spec['zlib'].prefix.lib
])
config_args = [
"--prefix={0}".format(prefix),
"--with-threads", "--with-threads",
"--enable-shared", "--enable-shared",
"CPPFLAGS=-I%s/include -I%s/include -I%s/include -I%s/include -I%s/include -I%s/include" % ( "CPPFLAGS=-I{0}".format(cppflags),
spec['openssl'].prefix, spec['bzip2'].prefix, "LDFLAGS=-L{0}".format(ldflags)
spec['readline'].prefix, spec['ncurses'].prefix,
spec['sqlite'].prefix, spec['zlib'].prefix),
"LDFLAGS=-L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib" % (
spec['openssl'].prefix, spec['bzip2'].prefix,
spec['readline'].prefix, spec['ncurses'].prefix,
spec['sqlite'].prefix, spec['zlib'].prefix)
] ]
if spec.satisfies('@3:'): if spec.satisfies('@3:'):
configure_args.append('--without-ensurepip') config_args.append('--without-ensurepip')
configure(*configure_args)
configure(*config_args)
make() make()
make("install") make("install")
# Modify compiler paths in configuration files. This is necessary for self.filter_compilers(spec, prefix)
# building site packages outside of spack
filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)',
(r'\4\5'),
join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py'))
python3_version = '' def filter_compilers(self, spec, prefix):
"""Run after install to tell the configuration files and Makefiles
to use the compilers that Spack built the package with.
If this isn't done, they'll have CC and CXX set to Spack's generic
cc and c++. We want them to be bound to whatever compiler
they were built with."""
kwargs = {'ignore_absent': True, 'backup': False, 'string': True}
dirname = join_path(prefix.lib,
'python{0}'.format(self.version.up_to(2)))
config = 'config'
if spec.satisfies('@3:'): if spec.satisfies('@3:'):
python3_version = '-%d.%dm' % self.version[:2] config = 'config-{0}m'.format(self.version.up_to(2))
makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile')
filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)',
(r'\4\5'),
makefile_filepath)
files = [
'_sysconfigdata.py',
join_path(config, 'Makefile')
]
for filename in files:
filter_file(env['CC'], self.compiler.cc,
join_path(dirname, filename), **kwargs)
filter_file(env['CXX'], self.compiler.cxx,
join_path(dirname, filename), **kwargs)
# ======================================================================== # ========================================================================
# Set up environment to make install easy for python extensions. # Set up environment to make install easy for python extensions.
@ -104,57 +126,59 @@ def install(self, spec, prefix):
@property @property
def python_lib_dir(self): def python_lib_dir(self):
return os.path.join('lib', 'python%d.%d' % self.version[:2]) return join_path('lib', 'python{0}'.format(self.version.up_to(2)))
@property @property
def python_include_dir(self): def python_include_dir(self):
return os.path.join('include', 'python%d.%d' % self.version[:2]) return join_path('include', 'python{0}'.format(self.version.up_to(2)))
@property @property
def site_packages_dir(self): def site_packages_dir(self):
return os.path.join(self.python_lib_dir, 'site-packages') return join_path(self.python_lib_dir, 'site-packages')
def setup_dependent_environment(self, spack_env, run_env, extension_spec): def setup_dependent_environment(self, spack_env, run_env, extension_spec):
# TODO: do this only for actual extensions. """Set PYTHONPATH to include site-packages dir for the
extension and any other python extensions it depends on."""
# Set PYTHONPATH to include site-packages dir for the
# extension and any other python extensions it depends on.
python_paths = [] python_paths = []
for d in extension_spec.traverse(): for d in extension_spec.traverse():
if d.package.extends(self.spec): if d.package.extends(self.spec):
python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) python_paths.append(join_path(d.prefix,
self.site_packages_dir))
pythonpath = ':'.join(python_paths) pythonpath = ':'.join(python_paths)
spack_env.set('PYTHONPATH', pythonpath) spack_env.set('PYTHONPATH', pythonpath)
# For run time environment set only the path for extension_spec and prepend it to PYTHONPATH # For run time environment set only the path for
# extension_spec and prepend it to PYTHONPATH
if extension_spec.package.extends(self.spec): if extension_spec.package.extends(self.spec):
run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) run_env.prepend_path('PYTHONPATH', join_path(
extension_spec.prefix, self.site_packages_dir))
def setup_dependent_package(self, module, ext_spec): def setup_dependent_package(self, module, ext_spec):
""" """Called before python modules' install() methods.
Called before python modules' install() methods.
In most cases, extensions will only need to have one line:: In most cases, extensions will only need to have one line::
python('setup.py', 'install', '--prefix=%s' % prefix) python('setup.py', 'install', '--prefix={0}'.format(prefix))"""
"""
# Python extension builds can have a global python executable function # Python extension builds can have a global python executable function
if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): if Version("3.0.0") <= self.version < Version("4.0.0"):
module.python = Executable(join_path(self.spec.prefix.bin, 'python3')) module.python = Executable(join_path(self.spec.prefix.bin,
'python3'))
else: else:
module.python = Executable(join_path(self.spec.prefix.bin, 'python')) module.python = Executable(join_path(self.spec.prefix.bin,
'python'))
# Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs.
module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) module.python_lib_dir = join_path(ext_spec.prefix,
module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) self.python_lib_dir)
module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) module.python_include_dir = join_path(ext_spec.prefix,
self.python_include_dir)
module.site_packages_dir = join_path(ext_spec.prefix,
self.site_packages_dir)
# Make the site packages directory for extensions, if it does not exist already. # Make the site packages directory for extensions
if ext_spec.package.is_extension: if ext_spec.package.is_extension:
mkdirp(module.site_packages_dir) mkdirp(module.site_packages_dir)
@ -170,21 +194,25 @@ def python_ignore(self, ext_pkg, args):
patterns = [r'site-packages/easy-install\.pth$'] patterns = [r'site-packages/easy-install\.pth$']
# Ignore pieces of setuptools installed by other packages. # Ignore pieces of setuptools installed by other packages.
# Must include directory name or it will remove all site*.py files.
if ext_pkg.name != 'py-setuptools': if ext_pkg.name != 'py-setuptools':
patterns.append(r'bin/easy_install[^/]*$') patterns.extend([
patterns.append(r'site-packages/setuptools\.pth$') r'bin/easy_install[^/]*$',
patterns.append(r'site-packages/site[^/]*\.pyc?$') r'site-packages/setuptools[^/]*\.egg$',
r'site-packages/setuptools\.pth$',
r'site-packages/site[^/]*\.pyc?$',
r'site-packages/__pycache__/site[^/]*\.pyc?$'
])
if ext_pkg.name != 'py-numpy': if ext_pkg.name != 'py-numpy':
patterns.append(r'bin/f2py$') patterns.append(r'bin/f2py$')
return match_predicate(ignore_arg, patterns) return match_predicate(ignore_arg, patterns)
def write_easy_install_pth(self, exts): def write_easy_install_pth(self, exts):
paths = [] paths = []
for ext in sorted(exts.values()): for ext in sorted(exts.values()):
ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) ext_site_packages = join_path(ext.prefix, self.site_packages_dir)
easy_pth = "%s/easy-install.pth" % ext_site_packages easy_pth = join_path(ext_site_packages, "easy-install.pth")
if not os.path.isfile(easy_pth): if not os.path.isfile(easy_pth):
continue continue
@ -194,15 +222,18 @@ def write_easy_install_pth(self, exts):
line = line.rstrip() line = line.rstrip()
# Skip lines matching these criteria # Skip lines matching these criteria
if not line: continue if not line:
if re.search(r'^(import|#)', line): continue continue
if (ext.name != 'py-setuptools' and if re.search(r'^(import|#)', line):
re.search(r'setuptools.*egg$', line)): continue continue
if ((ext.name != 'py-setuptools' and
re.search(r'setuptools.*egg$', line))):
continue
paths.append(line) paths.append(line)
site_packages = os.path.join(self.prefix, self.site_packages_dir) site_packages = join_path(self.prefix, self.site_packages_dir)
main_pth = "%s/easy-install.pth" % site_packages main_pth = join_path(site_packages, "easy-install.pth")
if not paths: if not paths:
if os.path.isfile(main_pth): if os.path.isfile(main_pth):
@ -210,12 +241,19 @@ def write_easy_install_pth(self, exts):
else: else:
with closing(open(main_pth, 'w')) as f: with closing(open(main_pth, 'w')) as f:
f.write("import sys; sys.__plen = len(sys.path)\n") f.write("""
import sys
sys.__plen = len(sys.path)
""")
for path in paths: for path in paths:
f.write("%s\n" % path) f.write("{0}\n".format(path))
f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; " f.write("""
"p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n") new = sys.path[sys.__plen:]
del sys.path[sys.__plen:]
p = getattr(sys, '__egginsert', 0)
sys.path[p:p] = new
sys.__egginsert = p + len(new)
""")
def activate(self, ext_pkg, **args): def activate(self, ext_pkg, **args):
ignore = self.python_ignore(ext_pkg, args) ignore = self.python_ignore(ext_pkg, args)
@ -227,12 +265,12 @@ def activate(self, ext_pkg, **args):
exts[ext_pkg.name] = ext_pkg.spec exts[ext_pkg.name] = ext_pkg.spec
self.write_easy_install_pth(exts) self.write_easy_install_pth(exts)
def deactivate(self, ext_pkg, **args): def deactivate(self, ext_pkg, **args):
args.update(ignore=self.python_ignore(ext_pkg, args)) args.update(ignore=self.python_ignore(ext_pkg, args))
super(Python, self).deactivate(ext_pkg, **args) super(Python, self).deactivate(ext_pkg, **args)
exts = spack.install_layout.extension_map(self.spec) exts = spack.install_layout.extension_map(self.spec)
if ext_pkg.name in exts: # Make deactivate idempotent. # Make deactivate idempotent
if ext_pkg.name in exts:
del exts[ext_pkg.name] del exts[ext_pkg.name]
self.write_easy_install_pth(exts) self.write_easy_install_pth(exts)