commands: add spack url stats
command
This command prints out stats about all package versions, like so: $ spack url stats ==> 6070 total versions for 2827 packages: ------------------------------------ url 5411 89.1% schemes https 3822 70.6% http 1527 28.2% ftp 27 0.5% file 35 0.6% checksums sha512 6 0.1% sha256 163 3.0% sha1 81 1.5% md5 5161 95.4% ------------------------------------ svn 3 0.0% ------------------------------------ hg 5 0.1% ------------------------------------ go 1 0.0% ------------------------------------ git 650 10.7% security no commit 384 59.1% commit 266 40.9% ------------------------------------
This commit is contained in:
parent
82820efe37
commit
3f0e6d04e0
@ -23,10 +23,15 @@
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
from __future__ import division, print_function
|
||||
|
||||
from collections import defaultdict
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
except ImportError:
|
||||
from urlparse import urlparse
|
||||
|
||||
import spack.fetch_strategy as fs
|
||||
import spack.repo
|
||||
import spack.util.crypto as crypto
|
||||
|
||||
from llnl.util import tty
|
||||
from spack.url import parse_version_offset, parse_name_offset
|
||||
@ -87,12 +92,18 @@ def setup_parser(subparser):
|
||||
'summary',
|
||||
help='print a summary of how well we are parsing package urls')
|
||||
|
||||
# Stats
|
||||
sp.add_parser(
|
||||
'stats',
|
||||
help='print statistics on versions and checksums for all packages')
|
||||
|
||||
|
||||
def url(parser, args):
|
||||
action = {
|
||||
'parse': url_parse,
|
||||
'list': url_list,
|
||||
'summary': url_summary
|
||||
'summary': url_summary,
|
||||
'stats': url_stats,
|
||||
}
|
||||
|
||||
action[args.subcommand](args)
|
||||
@ -246,6 +257,77 @@ def url_summary(args):
|
||||
name_count_dict, version_count_dict)
|
||||
|
||||
|
||||
def url_stats(args):
|
||||
stats = {} # stats about fetchers in packages.
|
||||
nvers = 0 # total number of versions
|
||||
npkgs = 0 # total number of packages
|
||||
|
||||
def inc(fstype, category, attr=None):
|
||||
"""Increment statistics in the stats dict."""
|
||||
categories = stats.setdefault(fstype, {})
|
||||
if attr:
|
||||
cat_stats = categories.setdefault(category, {})
|
||||
val = cat_stats.setdefault(attr, 0)
|
||||
stats[fstype][category][attr] = val + 1
|
||||
else:
|
||||
val = categories.setdefault(category, 0)
|
||||
stats[fstype][category] = val + 1
|
||||
|
||||
# over all packages
|
||||
for pkg in spack.repo.path.all_packages():
|
||||
npkgs += 1
|
||||
|
||||
# look at each version
|
||||
for v, args in pkg.versions.items():
|
||||
# figure out what type of fetcher it is
|
||||
fetcher = fs.for_package_version(pkg, v)
|
||||
nvers += 1
|
||||
|
||||
fstype = fetcher.url_attr
|
||||
inc(fstype, 'total')
|
||||
|
||||
# put some special stats in for particular types of fetchers.
|
||||
if fstype == 'git':
|
||||
if 'commit' in args:
|
||||
inc('git', 'security', 'commit')
|
||||
else:
|
||||
inc('git', 'security', 'no commit')
|
||||
elif fstype == 'url':
|
||||
for h in crypto.hashes:
|
||||
if h in args:
|
||||
inc('url', 'checksums', h)
|
||||
break
|
||||
else:
|
||||
if 'checksum' in args:
|
||||
h = crypto.hash_algo_for_digest(args['checksum'])
|
||||
inc('url', 'checksums', h)
|
||||
else:
|
||||
inc('url', 'checksums', 'no checksum')
|
||||
|
||||
# parse out the URL scheme (https/http/ftp/etc.)
|
||||
urlinfo = urlparse(fetcher.url)
|
||||
inc('url', 'schemes', urlinfo.scheme)
|
||||
|
||||
# print a nice summary table
|
||||
tty.msg("%d total versions for %d packages:" % (nvers, npkgs))
|
||||
line_width = 36
|
||||
print("-" * line_width)
|
||||
for fetcher, fetcher_stats in sorted(stats.items(), reverse=True):
|
||||
fs_total = fetcher_stats['total']
|
||||
fs_pct = float(fs_total) / nvers * 100
|
||||
print("%-22s%5d%8.1f%%" % (fetcher, fs_total, fs_pct))
|
||||
|
||||
for category, cat_stats in sorted(fetcher_stats.items(), reverse=True):
|
||||
if category == 'total':
|
||||
continue
|
||||
print(" %s" % category)
|
||||
|
||||
for name, number in sorted(cat_stats.items(), reverse=True):
|
||||
pct = float(number) / fs_total * 100
|
||||
print(" %-18s%5d%8.1f%%" % (name, number, pct))
|
||||
print("-" * line_width)
|
||||
|
||||
|
||||
def print_name_and_version(url):
|
||||
"""Prints a URL. Underlines the detected name with dashes and
|
||||
the detected version with tildes.
|
||||
|
@ -24,6 +24,8 @@
|
||||
##############################################################################
|
||||
import re
|
||||
import pytest
|
||||
|
||||
import spack.repo
|
||||
from spack.url import UndetectableVersionError
|
||||
from spack.main import SpackCommand
|
||||
from spack.cmd.url import name_parsed_correctly, version_parsed_correctly
|
||||
@ -141,3 +143,11 @@ def test_url_summary():
|
||||
out_correct_versions = int(
|
||||
re.search(r'Versions correctly parsed:\s*(\d+)', out).group(1))
|
||||
assert out_correct_versions == correct_versions
|
||||
|
||||
|
||||
def test_url_stats(capfd):
|
||||
with capfd.disabled():
|
||||
output = url('stats')
|
||||
npkgs = '%d packages' % len(spack.repo.all_package_names())
|
||||
assert npkgs in output
|
||||
assert 'total versions' in output
|
||||
|
Loading…
Reference in New Issue
Block a user