move gpg back into Spack, fix style

This commit is contained in:
Gregory Becker 2024-07-16 17:30:40 -07:00
parent 1ff78a7959
commit 7ee62d8202
23 changed files with 91 additions and 85 deletions

View File

@ -38,6 +38,7 @@
import spack.config as config
import spack.database as spack_db
import spack.error
import spack.gpg
import spack.hooks
import spack.hooks.sbang
import spack.mirror
@ -53,7 +54,6 @@
import spack.traverse as traverse
import spack.util.crypto
import spack.util.file_cache as file_cache
import spack.util.gpg
import spack.util.path
import spack.util.spack_json as sjson
import spack.util.spack_yaml as syaml
@ -833,7 +833,7 @@ def tarball_path_name(spec, ext):
def select_signing_key(key=None):
if key is None:
keys = spack.util.gpg.signing_keys()
keys = spack.gpg.signing_keys()
if len(keys) == 1:
key = keys[0]
@ -858,7 +858,7 @@ def sign_specfile(key, force, specfile_path):
raise NoOverwriteException(signed_specfile_path)
key = select_signing_key(key)
spack.util.gpg.sign(key, specfile_path, signed_specfile_path, clearsign=True)
spack.gpg.sign(key, specfile_path, signed_specfile_path, clearsign=True)
def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_dir, concurrency):
@ -1566,7 +1566,7 @@ def try_verify(specfile_path):
suppress = config.get("config:suppress_gpg_warnings", False)
try:
spack.util.gpg.verify(specfile_path, suppress_warnings=suppress)
spack.gpg.verify(specfile_path, suppress_warnings=suppress)
except Exception:
return False
@ -2037,7 +2037,7 @@ def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum
if os.path.exists("%s.asc" % specfile_path):
suppress = config.get("config:suppress_gpg_warnings", False)
try:
spack.util.gpg.verify("%s.asc" % specfile_path, specfile_path, suppress)
spack.gpg.verify("%s.asc" % specfile_path, specfile_path, suppress)
except Exception:
raise NoVerifyException(
"Spack was unable to verify package "
@ -2443,7 +2443,7 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
tty.debug("Found key {0}".format(fingerprint))
if install:
if trust:
spack.util.gpg.trust(stage.save_filename)
spack.gpg.trust(stage.save_filename)
tty.debug("Added this key to trusted keys.")
else:
tty.debug(
@ -2461,7 +2461,7 @@ def push_keys(*mirrors, **kwargs):
tmpdir = kwargs.get("tmpdir")
remove_tmpdir = False
keys = spack.util.gpg.public_keys(*(keys or []))
keys = spack.gpg.public_keys(*(keys or []))
try:
for mirror in mirrors:
@ -2493,7 +2493,7 @@ def push_keys(*mirrors, **kwargs):
export_target = os.path.join(prefix, filename)
# Export public keys (private is set to False)
spack.util.gpg.export_keys(export_target, [fingerprint])
spack.gpg.export_keys(export_target, [fingerprint])
# If mirror is local, the above export writes directly to the
# mirror (export_target points directly to the mirror).

View File

@ -52,7 +52,7 @@ def environment_root(cls) -> pathlib.Path:
return pathlib.Path(
spack.util.path.canonicalize_path(
os.path.join(bootstrap_root_path, "environments", environment_dir),
replacements=spack.paths.path_replacements()
replacements=spack.paths.path_replacements(),
)
)
@ -139,7 +139,9 @@ def _install_with_depfile(self) -> None:
"-C",
str(self.environment_root()),
"-j",
str(spack.util.cpus.determine_number_of_jobs(parallel=True, config=spack.config.CONFIG)),
str(
spack.util.cpus.determine_number_of_jobs(parallel=True, config=spack.config.CONFIG)
),
**kwargs,
)

View File

@ -31,13 +31,13 @@
import spack.binary_distribution as bindist
import spack.config as cfg
import spack.environment as ev
import spack.gpg
import spack.main
import spack.mirror
import spack.paths
import spack.repo
import spack.spec
import spack.util.git
import spack.util.gpg as gpg_util
import spack.util.spack_yaml as syaml
import spack.util.url as url_util
import spack.util.web as web_util
@ -1454,13 +1454,13 @@ def can_sign_binaries():
"""Utility method to determine if this spack instance is capable of
signing binary packages. This is currently only possible if the
spack gpg keystore contains exactly one secret key."""
return len(gpg_util.signing_keys()) == 1
return len(spack.gpg.signing_keys()) == 1
def can_verify_binaries():
"""Utility method to determin if this spack instance is capable (at
least in theory) of verifying signed binaries."""
return len(gpg_util.public_keys()) >= 1
return len(spack.gpg.public_keys()) >= 1
def _push_mirror_contents(input_spec, sign_binaries, mirror_url):

View File

@ -16,9 +16,9 @@
import spack.cmd.buildcache as buildcache
import spack.config as cfg
import spack.environment as ev
import spack.gpg
import spack.hash_types as ht
import spack.mirror
import spack.util.gpg as gpg_util
import spack.util.timer as timer
import spack.util.url as url_util
import spack.util.web as web_util
@ -305,7 +305,7 @@ def ci_rebuild(args):
# Fail early if signing is required but we don't have a signing key
sign_binaries = require_signing is not None and require_signing.lower() == "true"
if sign_binaries and not spack_ci.can_sign_binaries():
gpg_util.list(False, True)
spack.gpg.list(False, True)
tty.die("SPACK_REQUIRE_SIGNING=True => spack must have exactly one signing key")
# Construct absolute paths relative to current $CI_PROJECT_DIR

View File

@ -7,9 +7,9 @@
import os
import spack.binary_distribution
import spack.gpg
import spack.mirror
import spack.paths
import spack.util.gpg
import spack.util.url
from spack.cmd.common import arguments
@ -129,40 +129,40 @@ def setup_parser(subparser):
def gpg_create(args):
"""create a new key"""
if args.export or args.secret:
old_sec_keys = spack.util.gpg.signing_keys()
old_sec_keys = spack.gpg.signing_keys()
# Create the new key
spack.util.gpg.create(
spack.gpg.create(
name=args.name, email=args.email, comment=args.comment, expires=args.expires
)
if args.export or args.secret:
new_sec_keys = set(spack.util.gpg.signing_keys())
new_sec_keys = set(spack.gpg.signing_keys())
new_keys = new_sec_keys.difference(old_sec_keys)
if args.export:
spack.util.gpg.export_keys(args.export, new_keys)
spack.gpg.export_keys(args.export, new_keys)
if args.secret:
spack.util.gpg.export_keys(args.secret, new_keys, secret=True)
spack.gpg.export_keys(args.secret, new_keys, secret=True)
def gpg_export(args):
"""export a gpg key, optionally including secret key"""
keys = args.keys
if not keys:
keys = spack.util.gpg.signing_keys()
spack.util.gpg.export_keys(args.location, keys, args.secret)
keys = spack.gpg.signing_keys()
spack.gpg.export_keys(args.location, keys, args.secret)
def gpg_list(args):
"""list keys available in the keyring"""
spack.util.gpg.list(args.trusted, args.signing)
spack.gpg.list(args.trusted, args.signing)
def gpg_sign(args):
"""sign a package"""
key = args.key
if key is None:
keys = spack.util.gpg.signing_keys()
keys = spack.gpg.signing_keys()
if len(keys) == 1:
key = keys[0]
elif not keys:
@ -173,12 +173,12 @@ def gpg_sign(args):
if not output:
output = args.spec[0] + ".asc"
# TODO: Support the package format Spack creates.
spack.util.gpg.sign(key, " ".join(args.spec), output, args.clearsign)
spack.gpg.sign(key, " ".join(args.spec), output, args.clearsign)
def gpg_trust(args):
"""add a key to the keyring"""
spack.util.gpg.trust(args.keyfile)
spack.gpg.trust(args.keyfile)
def gpg_init(args):
@ -191,12 +191,12 @@ def gpg_init(args):
for filename in filenames:
if not filename.endswith(".key"):
continue
spack.util.gpg.trust(os.path.join(root, filename))
spack.gpg.trust(os.path.join(root, filename))
def gpg_untrust(args):
"""remove a key from the keyring"""
spack.util.gpg.untrust(args.signing, *args.keys)
spack.gpg.untrust(args.signing, *args.keys)
def gpg_verify(args):
@ -205,7 +205,7 @@ def gpg_verify(args):
signature = args.signature
if signature is None:
signature = args.spec[0] + ".asc"
spack.util.gpg.verify(signature, " ".join(args.spec))
spack.gpg.verify(signature, " ".join(args.spec))
def gpg_publish(args):

View File

@ -11,9 +11,9 @@
import spack
import spack.config
import spack.gpg
import spack.paths
import spack.util.git
import spack.util.gpg
from spack.cmd.common import arguments
from spack.util.spack_yaml import syaml_dict
@ -76,7 +76,7 @@ def tutorial(parser, args):
spack.config.set("mirrors", mirror_config, scope="user")
tty.msg("Ensuring that we trust tutorial binaries", f"spack gpg trust {tutorial_key}")
spack.util.gpg.trust(tutorial_key)
spack.gpg.trust(tutorial_key)
# Note that checkout MUST be last. It changes Spack under our feet.
# If you don't put this last, you'll get import errors for the code

View File

@ -2,17 +2,17 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import stat
import warnings
import os
import spack.error
import spack.repo
import spack.paths
import spack.repo
import spack.util.file_permissions as fp
from spack.config import ConfigError
from spack.util.path import canonicalize_path
from spack.version import Version
import spack.util.file_permissions as fp
_lesser_spec_types = {"compiler": spack.spec.CompilerSpec, "version": Version}

View File

@ -9,6 +9,7 @@
throughout Spack and should bring in a minimal number of external
dependencies.
"""
import getpass
import os
import tempfile
from datetime import date
@ -16,7 +17,7 @@
import llnl.util.filesystem
import spack.util.path
from spack.util.path import NOMATCH
#: This file lives in $prefix/lib/spack/spack/__file__
prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4)))
@ -169,7 +170,6 @@ def get_user():
def path_replacements():
# break circular imports
import spack.environment as ev
import spack.paths
arch = architecture()
@ -186,6 +186,5 @@ def path_replacements():
"target": lambda: arch.target,
"target_family": lambda: arch.target.microarchitecture.family,
"date": lambda: date.today().strftime("%Y-%m-%d"),
"env": lambda: ev.active_environment().path if ev.active_environment() else \
spack.util.path.NOMATCH,
"env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH,
}

View File

@ -25,12 +25,12 @@
import spack.caches
import spack.config
import spack.fetch_strategy
import spack.gpg
import spack.hooks.sbang as sbang
import spack.main
import spack.mirror
import spack.repo
import spack.store
import spack.util.gpg
import spack.util.spack_yaml as syaml
import spack.util.url as url_util
import spack.util.web as web_util
@ -344,10 +344,10 @@ def test_push_and_fetch_keys(mock_gnupghome):
# dir 1: create a new key, record its fingerprint, and push it to a new
# mirror
with spack.util.gpg.gnupghome_override(gpg_dir1):
spack.util.gpg.create(name="test-key", email="fake@test.key", expires="0", comment=None)
with spack.gpg.gnupghome_override(gpg_dir1):
spack.gpg.create(name="test-key", email="fake@test.key", expires="0", comment=None)
keys = spack.util.gpg.public_keys()
keys = spack.gpg.public_keys()
assert len(keys) == 1
fpr = keys[0]
@ -355,12 +355,12 @@ def test_push_and_fetch_keys(mock_gnupghome):
# dir 2: import the key from the mirror, and confirm that its fingerprint
# matches the one created above
with spack.util.gpg.gnupghome_override(gpg_dir2):
assert len(spack.util.gpg.public_keys()) == 0
with spack.gpg.gnupghome_override(gpg_dir2):
assert len(spack.gpg.public_keys()) == 0
bindist.get_keys(mirrors=mirrors, install=True, trust=True, force=True)
new_keys = spack.util.gpg.public_keys()
new_keys = spack.gpg.public_keys()
assert len(new_keys) == 1
assert new_keys[0] == fpr

View File

@ -16,9 +16,9 @@
import spack.config
import spack.environment as ev
import spack.error
import spack.gpg
import spack.paths as spack_paths
import spack.util.git
import spack.util.gpg
import spack.util.spack_yaml as syaml

View File

@ -18,11 +18,11 @@
import spack.ci as ci
import spack.config
import spack.environment as ev
import spack.gpg
import spack.hash_types as ht
import spack.main
import spack.paths as spack_paths
import spack.repo as repo
import spack.util.gpg
import spack.util.spack_yaml as syaml
import spack.util.url as url_util
from spack.schema.buildcache_spec import schema as specfile_schema

View File

@ -10,9 +10,9 @@
import llnl.util.filesystem as fs
import spack.bootstrap
import spack.util.executable
import spack.util.gpg
import spack.gpg
import spack.paths
import spack.util.executable
from spack.main import SpackCommand
from spack.paths import mock_gpg_data_path, mock_gpg_keys_path
from spack.util.executable import ProcessError
@ -46,19 +46,19 @@ def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
monkeypatch.setenv("PATH", str(tmpdir))
if version == "undetectable" or version.endswith("1.3.4"):
with pytest.raises(spack.util.gpg.SpackGPGError):
spack.util.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
with pytest.raises(spack.gpg.SpackGPGError):
spack.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
else:
spack.util.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
assert spack.util.gpg.GPG is not None
assert spack.util.gpg.GPGCONF is not None
spack.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
assert spack.gpg.GPG is not None
assert spack.gpg.GPGCONF is not None
def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch, mutable_config):
monkeypatch.setenv("PATH", str(tmpdir))
bootstrap("disable")
with pytest.raises(RuntimeError):
spack.util.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
spack.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
@pytest.mark.maybeslow
@ -106,7 +106,7 @@ def test_gpg(tmpdir, mutable_config, mock_gnupghome):
"Spack testing 1",
"spack@googlegroups.com",
)
keyfp = spack.util.gpg.signing_keys()[0]
keyfp = spack.gpg.signing_keys()[0]
# List the keys.
# TODO: Test the output here.

View File

@ -39,6 +39,7 @@
import spack.directory_layout
import spack.environment as ev
import spack.error
import spack.gpg
import spack.package_base
import spack.package_prefs
import spack.paths
@ -50,7 +51,6 @@
import spack.test.cray_manifest
import spack.util.executable
import spack.util.git
import spack.util.gpg
import spack.util.spack_yaml as syaml
import spack.util.url as url_util
from spack.fetch_strategy import URLFetchStrategy
@ -1074,13 +1074,13 @@ def mock_gnupghome(monkeypatch):
# This comes up because tmp paths on macOS are already long-ish, and
# pytest makes them longer.
try:
spack.util.gpg.init(gpg_path=spack.paths.gpg_path)
except spack.util.gpg.SpackGPGError:
if not spack.util.gpg.GPG:
spack.gpg.init(gpg_path=spack.paths.gpg_path)
except spack.gpg.SpackGPGError:
if not spack.gpg.GPG:
pytest.skip("This test requires gpg")
short_name_tmpdir = tempfile.mkdtemp()
with spack.util.gpg.gnupghome_override(short_name_tmpdir):
with spack.gpg.gnupghome_override(short_name_tmpdir):
yield short_name_tmpdir
# clean up, since we are doing this manually

View File

@ -21,10 +21,10 @@
import spack.binary_distribution as bindist
import spack.cmd.buildcache as buildcache
import spack.error
import spack.gpg
import spack.package_base
import spack.repo
import spack.store
import spack.util.gpg
import spack.util.url as url_util
from spack.fetch_strategy import URLFetchStrategy
from spack.paths import mock_gpg_keys_path
@ -72,7 +72,7 @@ def test_buildcache(mock_archive, tmp_path, monkeypatch, mutable_config):
create_args = ["create", "-f", "--rebuild-index", mirror_path, pkghash]
# Create a private key to sign package with if gpg2 available
spack.util.gpg.create(
spack.gpg.create(
name="test key 1",
expires="0",
email="spack@googlegroups.com",

View File

@ -11,8 +11,8 @@
from llnl.util.filesystem import getuid, group_ids
import spack.config
import spack.util.lock as lk
import spack.util.error
import spack.util.lock as lk
def test_disable_locking(tmpdir):

View File

@ -7,14 +7,14 @@
import pytest
import spack.gpg
import spack.paths
import spack.util.gpg
@pytest.fixture()
def has_socket_dir():
spack.util.gpg.init(gpg_path=spack.paths.gpg_path)
return bool(spack.util.gpg.SOCKET_DIR)
spack.gpg.init(gpg_path=spack.paths.gpg_path)
return bool(spack.gpg.SOCKET_DIR)
def test_parse_gpg_output_case_one():
@ -28,7 +28,7 @@ def test_parse_gpg_output_case_one():
uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) <j.s@s.com>:
ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::
"""
keys = spack.util.gpg._parse_secret_keys_output(output)
keys = spack.gpg._parse_secret_keys_output(output)
assert len(keys) == 2
assert keys[0] == "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
@ -45,7 +45,7 @@ def test_parse_gpg_output_case_two():
fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY:
grp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:
"""
keys = spack.util.gpg._parse_secret_keys_output(output)
keys = spack.gpg._parse_secret_keys_output(output)
assert len(keys) == 1
assert keys[0] == "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
@ -64,7 +64,7 @@ def test_parse_gpg_output_case_three():
ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::
fpr:::::::::ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ:"""
keys = spack.util.gpg._parse_secret_keys_output(output)
keys = spack.gpg._parse_secret_keys_output(output)
assert len(keys) == 2
assert keys[0] == "WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW"
@ -84,8 +84,8 @@ def test_really_long_gnupghome_dir(tmpdir, has_socket_dir):
tdir = tdir[:N].rstrip(os.sep)
tdir += "0" * (N - len(tdir))
with spack.util.gpg.gnupghome_override(tdir):
spack.util.gpg.create(
with spack.gpg.gnupghome_override(tdir):
spack.gpg.create(
name="Spack testing 1", email="test@spack.io", comment="Spack testing key", expires="0"
)
spack.util.gpg.list(True, True)
spack.gpg.list(True, True)

View File

@ -25,8 +25,8 @@ def determine_number_of_jobs(
*,
parallel: bool = False,
max_cpus: int = cpus_available(),
config: Optional["spack.config.Configuration"] = None,
config_path: str = 'config:build_jobs',
config=None,
config_path: str = "config:build_jobs",
) -> int:
"""
Packages that require sequential builds need 1 job. Otherwise we use the

View File

@ -85,4 +85,3 @@ def __repr__(self):
def __reduce__(self):
return type(self), (self.message, self.long_message)

View File

@ -365,6 +365,7 @@ def which(*args, **kwargs):
class EditorError(spack.util.error.UtilityError):
"""Base error for all errors from the executable utility"""
class ProcessError(EditorError):
"""ProcessErrors are raised when Executables exit with an error code."""

View File

@ -8,20 +8,22 @@
TODO: this is really part of spack.config. Consolidate it.
"""
import contextlib
import getpass
import os
import re
import subprocess
import sys
import tempfile
from datetime import date
import llnl.util.tty as tty
from llnl.util.lang import memoized
import spack.util.spack_yaml as syaml
__all__ = ["substitute_config_variables", "substitute_path_variables", "canonicalize_path", "NOMATCH"]
__all__ = [
"substitute_config_variables",
"substitute_path_variables",
"canonicalize_path",
"NOMATCH",
]
# return value for replacements with no match
@ -118,6 +120,7 @@ def substitute_config_variables(path, replacements={}):
replaced if there is an active environment, and should only be used in
environment yaml files.
"""
# Look up replacements
def repl(match):
m = match.group(0)
@ -200,8 +203,10 @@ def canonicalize_path(path, default_wd=None, replacements=None):
_replacements = replacements
if not isinstance(_replacements, dict):
tty.die("Replacements returned by replacements func are of type"
f"{type(replacements)} and not of the expected type of dict.")
tty.die(
"Replacements returned by replacements func are of type"
f"{type(replacements)} and not of the expected type of dict."
)
path = substitute_path_variables(path, replacements=_replacements)
if not os.path.isabs(path):

View File

@ -25,8 +25,8 @@
from llnl.util import lang, tty
from llnl.util.filesystem import mkdirp, rename, working_dir
import spack.util.url as url_util
import spack.util.error
import spack.util.url as url_util
from .executable import CommandNotFoundError, which
from .gcs import GCSBlob, GCSBucket, GCSHandler

View File

@ -10,8 +10,8 @@
import llnl.util.tty as tty
import spack.package_prefs as pp
import spack.filesystem_view
import spack.package_prefs as pp
import spack.store
import spack.util.spack_json as sjson
from spack.package_base import spack_times_log