buildcache: Add environment-aware buildcache sync command (#25470)

lib/spack/spack/cmd/buildcache.py

@@ -6,6 +6,7 @@
 import os
 import shutil
 import sys
+import tempfile
 
 import llnl.util.tty as tty
 
@@ -15,6 +16,7 @@
 import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
+import spack.fetch_strategy as fs
 import spack.hash_types as ht
 import spack.mirror
 import spack.relocate
@@ -23,9 +25,11 @@
 import spack.store
 import spack.util.crypto
 import spack.util.url as url_util
+import spack.util.web as web_util
 from spack.cmd import display_specs
 from spack.error import SpecError
 from spack.spec import Spec, save_dependency_spec_yamls
+from spack.stage import Stage
 from spack.util.string import plural
 
 description = "create, download and install binary packages"
@@ -226,6 +230,36 @@ def setup_parser(subparser):
         help='Destination mirror url')
     copy.set_defaults(func=buildcache_copy)
 
+    # Sync buildcache entries from one mirror to another
+    sync = subparsers.add_parser('sync', help=buildcache_sync.__doc__)
+    source = sync.add_mutually_exclusive_group(required=True)
+    source.add_argument('--src-directory',
+                        metavar='DIRECTORY',
+                        type=str,
+                        help="Source mirror as a local file path")
+    source.add_argument('--src-mirror-name',
+                        metavar='MIRROR_NAME',
+                        type=str,
+                        help="Name of the source mirror")
+    source.add_argument('--src-mirror-url',
+                        metavar='MIRROR_URL',
+                        type=str,
+                        help="URL of the source mirror")
+    dest = sync.add_mutually_exclusive_group(required=True)
+    dest.add_argument('--dest-directory',
+                      metavar='DIRECTORY',
+                      type=str,
+                      help="Destination mirror as a local file path")
+    dest.add_argument('--dest-mirror-name',
+                      metavar='MIRROR_NAME',
+                      type=str,
+                      help="Name of the destination mirror")
+    dest.add_argument('--dest-mirror-url',
+                      metavar='MIRROR_URL',
+                      type=str,
+                      help="URL of the destination mirror")
+    sync.set_defaults(func=buildcache_sync)
+
     # Update buildcache index without copying any additional packages
     update_index = subparsers.add_parser(
         'update-index', help=buildcache_update_index.__doc__)
@@ -779,6 +813,123 @@ def buildcache_copy(args):
         shutil.copyfile(cdashid_src_path, cdashid_dest_path)
 
 
+def buildcache_sync(args):
+    """ Syncs binaries (and associated metadata) from one mirror to another.
+    Requires an active environment in order to know which specs to sync.
+
+    Args:
+        src (str): Source mirror URL
+        dest (str): Destination mirror URL
+    """
+    # Figure out the source mirror
+    source_location = None
+    if args.src_directory:
+        source_location = args.src_directory
+        scheme = url_util.parse(source_location, scheme='<missing>').scheme
+        if scheme != '<missing>':
+            raise ValueError(
+                '"--src-directory" expected a local path; got a URL, instead')
+        # Ensure that the mirror lookup does not mistake this for named mirror
+        source_location = 'file://' + source_location
+    elif args.src_mirror_name:
+        source_location = args.src_mirror_name
+        result = spack.mirror.MirrorCollection().lookup(source_location)
+        if result.name == "<unnamed>":
+            raise ValueError(
+                'no configured mirror named "{name}"'.format(
+                    name=source_location))
+    elif args.src_mirror_url:
+        source_location = args.src_mirror_url
+        scheme = url_util.parse(source_location, scheme='<missing>').scheme
+        if scheme == '<missing>':
+            raise ValueError(
+                '"{url}" is not a valid URL'.format(url=source_location))
+
+    src_mirror = spack.mirror.MirrorCollection().lookup(source_location)
+    src_mirror_url = url_util.format(src_mirror.fetch_url)
+
+    # Figure out the destination mirror
+    dest_location = None
+    if args.dest_directory:
+        dest_location = args.dest_directory
+        scheme = url_util.parse(dest_location, scheme='<missing>').scheme
+        if scheme != '<missing>':
+            raise ValueError(
+                '"--dest-directory" expected a local path; got a URL, instead')
+        # Ensure that the mirror lookup does not mistake this for named mirror
+        dest_location = 'file://' + dest_location
+    elif args.dest_mirror_name:
+        dest_location = args.dest_mirror_name
+        result = spack.mirror.MirrorCollection().lookup(dest_location)
+        if result.name == "<unnamed>":
+            raise ValueError(
+                'no configured mirror named "{name}"'.format(
+                    name=dest_location))
+    elif args.dest_mirror_url:
+        dest_location = args.dest_mirror_url
+        scheme = url_util.parse(dest_location, scheme='<missing>').scheme
+        if scheme == '<missing>':
+            raise ValueError(
+                '"{url}" is not a valid URL'.format(url=dest_location))
+
+    dest_mirror = spack.mirror.MirrorCollection().lookup(dest_location)
+    dest_mirror_url = url_util.format(dest_mirror.fetch_url)
+
+    # Get the active environment
+    env = ev.get_env(args, 'buildcache sync', required=True)
+
+    tty.msg('Syncing environment buildcache files from {0} to {1}'.format(
+        src_mirror_url, dest_mirror_url))
+
+    build_cache_dir = bindist.build_cache_relative_path()
+    buildcache_rel_paths = []
+
+    tty.debug('Syncing the following specs:')
+    for s in env.all_specs():
+        tty.debug('  {0}{1}: {2}'.format(
+            '* ' if s in env.roots() else '  ', s.name, s.dag_hash()))
+
+        buildcache_rel_paths.extend([
+            os.path.join(
+                build_cache_dir, bindist.tarball_path_name(s, '.spack')),
+            os.path.join(
+                build_cache_dir, bindist.tarball_name(s, '.spec.yaml')),
+            os.path.join(
+                build_cache_dir, bindist.tarball_name(s, '.cdashid'))
+        ])
+
+    tmpdir = tempfile.mkdtemp()
+
+    try:
+        for rel_path in buildcache_rel_paths:
+            src_url = url_util.join(src_mirror_url, rel_path)
+            local_path = os.path.join(tmpdir, rel_path)
+            dest_url = url_util.join(dest_mirror_url, rel_path)
+
+            tty.debug('Copying {0} to {1} via {2}'.format(
+                src_url, dest_url, local_path))
+
+            stage = Stage(src_url,
+                          name="temporary_file",
+                          path=os.path.dirname(local_path),
+                          keep=True)
+
+            try:
+                stage.create()
+                stage.fetch()
+                web_util.push_to_url(
+                    local_path,
+                    dest_url,
+                    keep_original=True)
+            except fs.FetchError as e:
+                tty.debug('spack buildcache unable to sync {0}'.format(rel_path))
+                tty.debug(e)
+            finally:
+                stage.destroy()
+    finally:
+        shutil.rmtree(tmpdir)
+
+
 def update_index(mirror_url, update_keys=False):
     mirror = spack.mirror.MirrorCollection().lookup(mirror_url)
     outdir = url_util.format(mirror.push_url)

lib/spack/spack/test/cmd/buildcache.py

@@ -6,6 +6,7 @@
 import errno
 import os
 import platform
+import shutil
 
 import pytest
 
@@ -172,3 +173,79 @@ def test_update_key_index(tmpdir, mutable_mock_env_path,
     mirror('rm', 'test-mirror')
 
     assert 'index.json' in key_dir_list
+
+
+def test_buildcache_sync(mutable_mock_env_path, install_mockery_mutable_config,
+                         mock_packages, mock_fetch, mock_stage, tmpdir):
+    """
+    Make sure buildcache sync works in an environment-aware manner, ignoring
+    any specs that may be in the mirror but not in the environment.
+    """
+    working_dir = tmpdir.join('working_dir')
+
+    src_mirror_dir = working_dir.join('src_mirror').strpath
+    src_mirror_url = 'file://{0}'.format(src_mirror_dir)
+
+    dest_mirror_dir = working_dir.join('dest_mirror').strpath
+    dest_mirror_url = 'file://{0}'.format(dest_mirror_dir)
+
+    in_env_pkg = 'trivial-install-test-package'
+    out_env_pkg = 'libdwarf'
+
+    def verify_mirror_contents():
+        dest_list = os.listdir(
+            os.path.join(dest_mirror_dir, 'build_cache'))
+
+        found_pkg = False
+
+        for p in dest_list:
+            assert(out_env_pkg not in p)
+            if in_env_pkg in p:
+                found_pkg = True
+
+        if not found_pkg:
+            print('Expected to find {0} in {1}'.format(
+                in_env_pkg, dest_mirror_dir))
+            assert(False)
+
+    # Install a package and put it in the buildcache
+    s = Spec(out_env_pkg).concretized()
+    install(s.name)
+    buildcache(
+        'create', '-u', '-f', '-a', '--mirror-url', src_mirror_url, s.name)
+
+    env('create', 'test')
+    with ev.read('test'):
+        add(in_env_pkg)
+        install()
+        buildcache(
+            'create', '-u', '-f', '-a', '--mirror-url', src_mirror_url, in_env_pkg)
+
+        # Now run the spack buildcache sync command with all the various options
+        # for specifying mirrors
+
+        # Use urls to specify mirrors
+        buildcache('sync',
+                   '--src-mirror-url', src_mirror_url,
+                   '--dest-mirror-url', dest_mirror_url)
+
+        verify_mirror_contents()
+        shutil.rmtree(dest_mirror_dir)
+
+        # Use local directory paths to specify fs locations
+        buildcache('sync',
+                   '--src-directory', src_mirror_dir,
+                   '--dest-directory', dest_mirror_dir)
+
+        verify_mirror_contents()
+        shutil.rmtree(dest_mirror_dir)
+
+        # Use mirror names to specify mirrors
+        mirror('add', 'src', src_mirror_url)
+        mirror('add', 'dest', dest_mirror_url)
+
+        buildcache('sync',
+                   '--src-mirror-name', 'src',
+                   '--dest-mirror-name', 'dest')
+
+        verify_mirror_contents()

share/spack/spack-completion.bash

@@ -482,7 +482,7 @@ _spack_buildcache() {
     then
         SPACK_COMPREPLY="-h --help"
     else
-        SPACK_COMPREPLY="create install list keys preview check download get-buildcache-name save-yaml copy update-index"
+        SPACK_COMPREPLY="create install list keys preview check download get-buildcache-name save-yaml copy sync update-index"
     fi
 }
 
@@ -546,6 +546,10 @@ _spack_buildcache_copy() {
     SPACK_COMPREPLY="-h --help --base-dir --spec-yaml --destination-url"
 }
 
+_spack_buildcache_sync() {
+    SPACK_COMPREPLY="-h --help --src-directory --src-mirror-name --src-mirror-url --dest-directory --dest-mirror-name --dest-mirror-url"
+}
+
 _spack_buildcache_update_index() {
     SPACK_COMPREPLY="-h --help -d --mirror-url -k --keys"
 }