clingo: modify recipe for bootstrapping (#22354)

* clingo: modify recipe for bootstrapping

Modifications:
- clingo builds with shared Python only if ^python+shared
- avoid building the clingo app for bootstrapping
- don't link to libpython when bootstrapping

* Remove option that breaks on linux

* Give more hints for the current Python

* Disable CLINGO_BUILD_PY_SHARED for bootstrapping

* bootstrapping: try to detect the current python from std library

This is much faster than calling external executables

* Fix compatibility with Python 2.6

* Give hints on which compiler and OS to use when bootstrapping

This change hints which compiler to use for bootstrapping clingo
(either GCC or Apple Clang on MacOS). On Cray platforms it also
hints to build for the frontend system, where software is meant
to be installed.

* Use spec_for_current_python to constrain module requirement

lib/spack/spack/bootstrap.py

@@ -5,6 +5,13 @@
 import contextlib
 import os
 import sys
+try:
+    import sysconfig  # novm
+except ImportError:
+    # Not supported on Python 2.6
+    pass
+
+import archspec.cpu
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
@@ -20,17 +27,33 @@
 from spack.util.environment import EnvironmentModifications
 
 
+def spec_for_current_python():
+    """For bootstrapping purposes we are just interested in the Python
+    minor version (all patches are ABI compatible with the same minor)
+    and on whether ucs4 support has been enabled for Python 2.7
+
+    See:
+      https://www.python.org/dev/peps/pep-0513/
+      https://stackoverflow.com/a/35801395/771663
+    """
+    version_str = '.'.join(str(x) for x in sys.version_info[:2])
+    variant_str = ''
+    if sys.version_info[0] == 2 and sys.version_info[1] == 7:
+        unicode_size = sysconfig.get_config_var('Py_UNICODE_SIZE')
+        variant_str = '+ucs4' if unicode_size == 4 else '~ucs4'
+
+    spec_fmt = 'python@{0} {1}'
+    return spec_fmt.format(version_str, variant_str)
+
+
 @contextlib.contextmanager
 def spack_python_interpreter():
     """Override the current configuration to set the interpreter under
     which Spack is currently running as the only Python external spec
     available.
     """
-    python_cls = type(spack.spec.Spec('python').package)
     python_prefix = os.path.dirname(os.path.dirname(sys.executable))
-    externals = python_cls.determine_spec_details(
+    external_python = spec_for_current_python()
-        python_prefix, [os.path.basename(sys.executable)])
-    external_python = externals[0]
 
     entry = {
         'buildable': False,
@@ -60,9 +83,10 @@ def make_module_available(module, spec=None, install=False):
     # We can constrain by a shortened version in place of a version range
     # because this spec is only used for querying or as a placeholder to be
     # replaced by an external that already has a concrete version. This syntax
-    # is not suffucient when concretizing without an external, as it will
+    # is not sufficient when concretizing without an external, as it will
     # concretize to python@X.Y instead of python@X.Y.Z
-    spec.constrain('^python@%d.%d' % sys.version_info[:2])
+    python_requirement = '^' + spec_for_current_python()
+    spec.constrain(python_requirement)
     installed_specs = spack.store.db.query(spec, installed=True)
 
     for ispec in installed_specs:
@@ -197,3 +221,27 @@ def ensure_bootstrap_configuration():
                 with spack.store.use_store(spack.paths.user_bootstrap_store):
                     with spack_python_interpreter():
                         yield
+
+
+def clingo_root_spec():
+    # Construct the root spec that will be used to bootstrap clingo
+    spec_str = 'clingo-bootstrap@spack+python'
+
+    # Add a proper compiler hint to the root spec. We use GCC for
+    # everything but MacOS.
+    if str(spack.architecture.platform()) == 'darwin':
+        spec_str += ' %apple-clang'
+    else:
+        spec_str += ' %gcc'
+
+    # Add hint to use frontend operating system on Cray
+    if str(spack.architecture.platform()) == 'cray':
+        spec_str += ' os=fe'
+
+    # Add the generic target
+    generic_target = archspec.cpu.host().family
+    spec_str += ' target={0}'.format(str(generic_target))
+
+    tty.debug('[BOOTSTRAP ROOT SPEC] clingo: {0}'.format(spec_str))
+
+    return spack.spec.Spec(spec_str)

lib/spack/spack/solver/asp.py

@@ -253,14 +253,11 @@ def __init__(self, cores=True, asp=None):
             # TODO: Find a way to vendor the concrete spec
             # in a cross-platform way
             with spack.bootstrap.ensure_bootstrap_configuration():
-                generic_target = archspec.cpu.host().family
+                clingo_spec = spack.bootstrap.clingo_root_spec()
-                spec_str = 'clingo-bootstrap@spack+python target={0}'.format(
-                    str(generic_target)
-                )
-                clingo_spec = spack.spec.Spec(spec_str)
                 clingo_spec._old_concretize()
                 spack.bootstrap.make_module_available(
-                    'clingo', spec=clingo_spec, install=True)
+                    'clingo', spec=clingo_spec, install=True
+                )
                 import clingo
         self.out = asp or llnl.util.lang.Devnull()
         self.cores = cores

var/spack/repos/builtin/packages/clingo-bootstrap/package.py

@@ -9,6 +9,9 @@
 
 class ClingoBootstrap(Clingo):
     """Clingo with some options used for bootstrapping"""
+
+    maintainers = ['alalazo']
+
     variant('build_type', default='Release', values=('Release',),
             description='CMake build type')
 
@@ -41,6 +44,18 @@ class ClingoBootstrap(Clingo):
     # Clingo needs the Python module to be usable by Spack
     conflicts('~python', msg='Python support is required to bootstrap Spack')
 
+    @property
+    def cmake_py_shared(self):
+        return self.define('CLINGO_BUILD_PY_SHARED', 'OFF')
+
+    def cmake_args(self):
+        args = super(ClingoBootstrap, self).cmake_args()
+        args.extend([
+            # Avoid building the clingo executable
+            self.define('CLINGO_BUILD_APPS', 'OFF'),
+        ])
+        return args
+
     def setup_build_environment(self, env):
         if '%apple-clang platform=darwin' in self.spec:
             opts = '-mmacosx-version-min=10.13'

var/spack/repos/builtin/packages/clingo/package.py

@@ -21,7 +21,7 @@ class Clingo(CMakePackage):
     url      = "https://github.com/potassco/clingo/archive/v5.2.2.tar.gz"
     git      = 'https://github.com/potassco/clingo.git'
 
-    maintainers = ["tgamblin"]
+    maintainers = ["tgamblin", "alalazo"]
 
     version('master', branch='master', submodules=True, preferred=True)
     version('spack', commit='2a025667090d71b2c9dce60fe924feb6bde8f667', submodules=True)
@@ -56,10 +56,17 @@ def cmake_python_hints(self):
         """Return standard CMake defines to ensure that the
         current spec is the one found by CMake find_package(Python, ...)
         """
+        python_spec = self.spec['python']
+        include_dir = python_spec.package.get_python_inc()
         return [
-            '-DPython_EXECUTABLE={0}'.format(str(self.spec['python'].command))
+            self.define('Python_EXECUTABLE', str(python_spec.command)),
+            self.define('Python_INCLUDE_DIR', include_dir)
         ]
 
+    @property
+    def cmake_py_shared(self):
+        return self.define('CLINGO_BUILD_PY_SHARED', 'ON')
+
     def cmake_args(self):
         try:
             self.compiler.cxx14_flag
@@ -69,10 +76,10 @@ def cmake_args(self):
         args = [
             '-DCLINGO_REQUIRE_PYTHON=ON',
             '-DCLINGO_BUILD_WITH_PYTHON=ON',
-            '-DCLINGO_BUILD_PY_SHARED=ON',
             '-DPYCLINGO_USER_INSTALL=OFF',
             '-DPYCLINGO_USE_INSTALL_PREFIX=ON',
-            '-DCLINGO_BUILD_WITH_LUA=OFF'
+            '-DCLINGO_BUILD_WITH_LUA=OFF',
+            self.cmake_py_shared
         ]
         if self.spec['cmake'].satisfies('@3.16.0:'):
             args += self.cmake_python_hints