zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rebuild-index: fix race condition by avoiding parallelism (#39594)

Browse Source
This commit is contained in:
Harmen Stoppels
2023-08-24 08:26:56 +02:00
committed by GitHub
parent fdea5e7624
commit 8914d26867
1 changed files with 10 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
lib/spack/spack/binary_distribution.py
View File

@@ -9,7 +9,6 @@
import io import io
import itertools import itertools
import json import json
import multiprocessing.pool
import os import os
import re import re
import shutil import shutil
@@ -876,32 +875,18 @@ def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_di
db: A spack database used for adding specs and then writing the index. db: A spack database used for adding specs and then writing the index.
temp_dir (str): Location to write index.json and hash for pushing temp_dir (str): Location to write index.json and hash for pushing
concurrency (int): Number of parallel processes to use when fetching concurrency (int): Number of parallel processes to use when fetching
Return:
None
""" """
for file in file_list:
def _fetch_spec_from_mirror(spec_url): contents = read_method(file)
spec_file_contents = read_method(spec_url)
if spec_file_contents:
# Need full spec.json name or this gets confused with index.json. # Need full spec.json name or this gets confused with index.json.
if spec_url.endswith(".json.sig"): if file.endswith(".json.sig"):
specfile_json = Spec.extract_json_from_clearsig(spec_file_contents) specfile_json = Spec.extract_json_from_clearsig(contents)
return Spec.from_dict(specfile_json) fetched_spec = Spec.from_dict(specfile_json)
if spec_url.endswith(".json"): elif file.endswith(".json"):
return Spec.from_json(spec_file_contents) fetched_spec = Spec.from_json(contents)
else:
continue
tp = multiprocessing.pool.ThreadPool(processes=concurrency)
try:
fetched_specs = tp.map(
llnl.util.lang.star(_fetch_spec_from_mirror), [(f,) for f in file_list]
)
finally:
tp.terminate()
tp.join()
for fetched_spec in fetched_specs:
db.add(fetched_spec, None) db.add(fetched_spec, None)
db.mark(fetched_spec, "in_buildcache", True) db.mark(fetched_spec, "in_buildcache", True)