# Licensed under a 3-clause BSD style license - see LICENSE.rst
import argparse
import os
from asv_runner.console import color_print
from .. import results, util
from ..console import log
from ..repo import NoSuchNameError, get_repo
from . import Command, common_args
from .compare import Compare
from .run import Run
[docs]
class Continuous(Command):
@classmethod
[docs]
def setup_arguments(cls, subparsers):
parser = subparsers.add_parser(
"continuous",
help="Compare two commits directly",
description="""Run a side-by-side comparison of two commits for
continuous integration.""",
)
parser.add_argument(
'base',
nargs='?',
default=None,
help="""The commit/branch to compare against. By default, the
parent of the tested commit.""",
)
parser.add_argument(
'branch',
default=None,
help="""The commit/branch to test. By default, the first configured branch.""",
)
common_args.add_record_samples(parser, record_default=True)
parser.add_argument(
"--quick",
"-q",
action="store_true",
help="""Do a "quick" run, where each benchmark function is
run only once. This is useful to find basic errors in the
benchmark functions faster. The results are unlikely to
be useful, and thus are not saved.""",
)
parser.add_argument(
"--interleave-rounds",
action="store_true",
default=None,
help="""Interleave benchmarks with multiple rounds across
commits. This can avoid measurement biases from commit ordering,
can take longer.""",
)
parser.add_argument(
"--no-interleave-rounds", action="store_false", dest="interleave_rounds"
)
# Backward compatibility for '--(no-)interleave-rounds'
parser.add_argument(
"--interleave-processes",
action="store_true",
default=False,
dest="interleave_rounds",
help=argparse.SUPPRESS,
)
parser.add_argument(
"--no-interleave-processes",
action="store_false",
dest="interleave_rounds",
help=argparse.SUPPRESS,
)
common_args.add_compare(parser, sort_default='ratio', only_changed_default=True)
common_args.add_show_stderr(parser)
common_args.add_bench(parser)
common_args.add_machine(parser)
common_args.add_environment(parser)
common_args.add_launch_method(parser)
parser.set_defaults(func=cls.run_from_args)
return parser
@classmethod
[docs]
def run_from_conf_args(cls, conf, args, **kwargs):
return cls.run(
conf=conf,
branch=args.branch,
base=args.base,
factor=args.factor,
split=args.split,
only_changed=args.only_changed,
sort=args.sort,
use_stats=args.use_stats,
show_stderr=args.show_stderr,
bench=args.bench,
attribute=args.attribute,
machine=args.machine,
env_spec=args.env_spec,
record_samples=args.record_samples,
append_samples=args.append_samples,
quick=args.quick,
interleave_rounds=args.interleave_rounds,
launch_method=args.launch_method,
**kwargs,
)
@classmethod
[docs]
def run(
cls,
conf,
branch=None,
base=None,
factor=None,
split=False,
only_changed=True,
sort='ratio',
use_stats=True,
show_stderr=False,
bench=None,
attribute=None,
machine=None,
env_spec=None,
record_samples=False,
append_samples=False,
quick=False,
interleave_rounds=None,
launch_method=None,
_machine_file=None,
):
repo = get_repo(conf)
repo.pull()
if branch is None:
branch = conf.branches[0]
try:
head = repo.get_hash_from_name(branch)
if base is None:
parent = repo.get_hash_from_parent(head)
else:
parent = repo.get_hash_from_name(base)
except NoSuchNameError as exc:
raise util.UserError(f"Unknown commit {exc}")
commit_hashes = [head, parent]
run_objs = {}
result = Run.run(
conf,
range_spec=commit_hashes,
bench=bench,
attribute=attribute,
show_stderr=show_stderr,
machine=machine,
env_spec=env_spec,
record_samples=record_samples,
append_samples=append_samples,
quick=quick,
interleave_rounds=interleave_rounds,
launch_method=launch_method,
_returns=run_objs,
_machine_file=_machine_file,
)
if result:
return result
log.flush()
def results_iter(commit_hash):
for env in run_objs['environments']:
machine_name = run_objs['machine_params']['machine']
filename = results.get_filename(machine_name, commit_hash, env.name)
filename = os.path.join(conf.results_dir, filename)
try:
result = results.Results.load(filename, machine_name)
except util.UserError as err:
log.warning(str(err))
continue
for name, benchmark in run_objs['benchmarks'].items():
params = benchmark['params']
version = benchmark['version']
value = result.get_result_value(name, params)
stats = result.get_result_stats(name, params)
samples = result.get_result_samples(name, params)
yield name, params, value, stats, samples, version, machine_name, env.name
commit_names = {
parent: repo.get_name_from_hash(parent),
head: repo.get_name_from_hash(head),
}
status = Compare.print_table(
conf,
parent,
head,
resultset_1=results_iter(parent),
resultset_2=results_iter(head),
factor=factor,
split=split,
use_stats=use_stats,
only_changed=only_changed,
sort=sort,
commit_names=commit_names,
)
worsened, improved = status
color_print("")
if worsened:
color_print("SOME BENCHMARKS HAVE CHANGED SIGNIFICANTLY.", 'red')
color_print("PERFORMANCE DECREASED.", 'red')
elif improved:
color_print("SOME BENCHMARKS HAVE CHANGED SIGNIFICANTLY.", 'green')
color_print("PERFORMANCE INCREASED.", 'green')
else:
color_print("BENCHMARKS NOT SIGNIFICANTLY CHANGED.", 'green')
return worsened