zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Format-agnostic data structure for reports

Browse Source 
Replace the JUnit-specific terms 'testsuite' and 'testcase' with
'spec' and 'package', respectively.
This commit is contained in:
Zack Galbreath 2018-02-26 13:18:11 -05:00 committed by Todd Gamblin
parent 0598f70de2
commit 6c5dbdd9cd
2 changed files with 68 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
lib/spack/spack/report.py
View File

@ -64,7 +64,7 @@ class InfoCollector(object):
When exiting the context this change will be rolled-back. When exiting the context this change will be rolled-back.
The data collected is available through the ``test_suites`` The data collected is available through the ``specs``
attribute once exited, and it's organized as a list where attribute once exited, and it's organized as a list where
each item represents the installation of one of the spec. each item represents the installation of one of the spec.
@ -77,49 +77,51 @@ class InfoCollector(object):
def __init__(self, specs): def __init__(self, specs):
#: Specs that will be installed #: Specs that will be installed
self.specs = specs self.input_specs = specs
#: Context that will be used to stamp the report from #: This is where we record the data that will be included
#: the template file #: in our report.
self.test_suites = [] self.specs = []
def __enter__(self): def __enter__(self):
# Initialize the test suites with the data that # Initialize the spec report with the data that is available upfront.
# is available upfront for input_spec in self.input_specs:
for spec in self.specs:
name_fmt = '{0}_{1}' name_fmt = '{0}_{1}'
name = name_fmt.format(spec.name, spec.dag_hash(length=7)) name = name_fmt.format(input_spec.name,
input_spec.dag_hash(length=7))
suite = { spec = {
'name': name, 'name': name,
'nerrors': None, 'nerrors': None,
'nfailures': None, 'nfailures': None,
'ntests': None, 'npackages': None,
'time': None, 'time': None,
'timestamp': time.strftime( 'timestamp': time.strftime(
"%a, %d %b %Y %H:%M:%S", time.gmtime() "%a, %d %b %Y %H:%M:%S", time.gmtime()
), ),
'properties': [], 'properties': [],
'testcases': [] 'packages': []
} }
self.test_suites.append(suite) self.specs.append(spec)
Property = collections.namedtuple('Property', ['name', 'value']) Property = collections.namedtuple('Property', ['name', 'value'])
suite['properties'].append( spec['properties'].append(
Property('architecture', spec.architecture) Property('architecture', input_spec.architecture)
) )
suite['properties'].append(Property('compiler', spec.compiler)) spec['properties'].append(
Property('compiler', input_spec.compiler))
# Check which specs are already installed and mark them as skipped # Check which specs are already installed and mark them as skipped
for dep in filter(lambda x: x.package.installed, spec.traverse()): for dep in filter(lambda x: x.package.installed,
test_case = { input_spec.traverse()):
package = {
'name': dep.name, 'name': dep.name,
'id': dep.dag_hash(), 'id': dep.dag_hash(),
'elapsed_time': '0.0', 'elapsed_time': '0.0',
'result': 'skipped', 'result': 'skipped',
'message': 'Spec already installed' 'message': 'Spec already installed'
} }
suite['testcases'].append(test_case) spec['packages'].append(package)
def gather_info(do_install): def gather_info(do_install):
"""Decorates do_install to gather useful information for """Decorates do_install to gather useful information for
@ -134,7 +136,7 @@ def wrapper(pkg, *args, **kwargs):
# We accounted before for what is already installed # We accounted before for what is already installed
installed_on_entry = pkg.installed installed_on_entry = pkg.installed
test_case = { package = {
'name': pkg.name, 'name': pkg.name,
'id': pkg.spec.dag_hash(), 'id': pkg.spec.dag_hash(),
'elapsed_time': None, 'elapsed_time': None,
@ -147,42 +149,42 @@ def wrapper(pkg, *args, **kwargs):
try: try:
value = do_install(pkg, *args, **kwargs) value = do_install(pkg, *args, **kwargs)
test_case['result'] = 'success' package['result'] = 'success'
if installed_on_entry: if installed_on_entry:
return return
except spack.build_environment.InstallError as e: except spack.build_environment.InstallError as e:
# An InstallError is considered a failure (the recipe # An InstallError is considered a failure (the recipe
# didn't work correctly) # didn't work correctly)
test_case['result'] = 'failure' package['result'] = 'failure'
test_case['stdout'] = fetch_package_log(pkg) package['stdout'] = fetch_package_log(pkg)
test_case['message'] = e.message or 'Installation failure' package['message'] = e.message or 'Installation failure'
test_case['exception'] = e.traceback package['exception'] = e.traceback
except (Exception, BaseException) as e: except (Exception, BaseException) as e:
# Everything else is an error (the installation # Everything else is an error (the installation
# failed outside of the child process) # failed outside of the child process)
test_case['result'] = 'error' package['result'] = 'error'
test_case['stdout'] = fetch_package_log(pkg) package['stdout'] = fetch_package_log(pkg)
test_case['message'] = str(e) or 'Unknown error' package['message'] = str(e) or 'Unknown error'
test_case['exception'] = traceback.format_exc() package['exception'] = traceback.format_exc()
finally: finally:
test_case['elapsed_time'] = time.time() - start_time package['elapsed_time'] = time.time() - start_time
# Append the case to the correct test suites. In some # Append the package to the correct spec report. In some
# cases it may happen that a spec that is asked to be # cases it may happen that a spec that is asked to be
# installed explicitly will also be installed as a # installed explicitly will also be installed as a
# dependency of another spec. In this case append to both # dependency of another spec. In this case append to both
# test suites. # spec reports.
for s in llnl.util.lang.dedupe([pkg.spec.root, pkg.spec]): for s in llnl.util.lang.dedupe([pkg.spec.root, pkg.spec]):
name = name_fmt.format(s.name, s.dag_hash(length=7)) name = name_fmt.format(s.name, s.dag_hash(length=7))
try: try:
item = next(( item = next((
x for x in self.test_suites x for x in self.specs
if x['name'] == name if x['name'] == name
)) ))
item['testcases'].append(test_case) item['packages'].append(package)
except StopIteration: except StopIteration:
pass pass
@ -199,16 +201,16 @@ def __exit__(self, exc_type, exc_val, exc_tb):
# Restore the original method in PackageBase # Restore the original method in PackageBase
spack.package.PackageBase.do_install = InfoCollector._backup_do_install spack.package.PackageBase.do_install = InfoCollector._backup_do_install
for suite in self.test_suites: for spec in self.specs:
suite['ntests'] = len(suite['testcases']) spec['npackages'] = len(spec['packages'])
suite['nfailures'] = len( spec['nfailures'] = len(
[x for x in suite['testcases'] if x['result'] == 'failure'] [x for x in spec['packages'] if x['result'] == 'failure']
) )
suite['nerrors'] = len( spec['nerrors'] = len(
[x for x in suite['testcases'] if x['result'] == 'error'] [x for x in spec['packages'] if x['result'] == 'error']
) )
suite['time'] = sum([ spec['time'] = sum([
float(x['elapsed_time']) for x in suite['testcases'] float(x['elapsed_time']) for x in spec['packages']
]) ])
@ -273,4 +275,4 @@ def __exit__(self, exc_type, exc_val, exc_tb):
with open(self.filename, 'w') as f: with open(self.filename, 'w') as f:
env = spack.tengine.make_environment() env = spack.tengine.make_environment()
t = env.get_template(templates[self.format_name]) t = env.get_template(templates[self.format_name])
f.write(t.render({'test_suites': self.collector.test_suites})) f.write(t.render({'specs': self.collector.specs}))

46
templates/reports/junit.xml
View File

@ -6,41 +6,41 @@
http://help.catchsoftware.com/display/ET/JUnit+Format http://help.catchsoftware.com/display/ET/JUnit+Format
--> -->
<testsuites> <testsuites>
{% for suite in test_suites %} {% for spec in specs %}
<testsuite name="{{ suite.name }}" <testsuite name="{{ spec.name }}"
errors="{{ suite.nerrors }}" errors="{{ spec.nerrors }}"
tests="{{ suite.ntests }}" tests="{{ spec.npackages }}"
failures="{{ suite.nfailures }}" failures="{{ spec.nfailures }}"
time="{{ suite.time }}" time="{{ spec.time }}"
timestamp="{{ suite.timestamp }}" > timestamp="{{ spec.timestamp }}" >
<properties> <properties>
{% for property in suite.properties %} {% for property in spec.properties %}
<property name="{{ property.name }}" value="{{ property.value }}" /> <property name="{{ property.name }}" value="{{ property.value }}" />
{% endfor %} {% endfor %}
</properties> </properties>
{% for test in suite.testcases %} {% for package in spec.packages %}
<testcase classname="{{ test.name }}" <testcase classname="{{ package.name }}"
name="{{ test.id }}" name="{{ package.id }}"
time="{{ test.elapsed_time }}"> time="{{ package.elapsed_time }}">
{% if test.result == 'failure' %} {% if package.result == 'failure' %}
<failure message="{{ test.message }}"> <failure message="{{ package.message }}">
{{ test.exception }} {{ package.exception }}
</failure> </failure>
{% elif test.result == 'error' %} {% elif package.result == 'error' %}
<error message="{{ test.message }}"> <error message="{{ package.message }}">
{{ test.exception }} {{ package.exception }}
</error> </error>
{% elif test.result == 'skipped' %} {% elif package.result == 'skipped' %}
<skipped /> <skipped />
{% endif %} {% endif %}
{% if test.stdout %} {% if package.stdout %}
<system-out> <system-out>
{{ test.stdout }} {{ package.stdout }}
</system-out> </system-out>
{% endif %} {% endif %}
{% if test.stderr %} {% if package.stderr %}
<system-err> <system-err>
{{ test.stderr }} {{ package.stderr }}
</system-err> </system-err>
{% endif %} {% endif %}
</testcase> </testcase>