tests/run-internalbench.py: Allow running internalbench on hardware.
Signed-off-by: Anson Mansfield <amansfield@mantaro.com>
This commit is contained in:
Anson Mansfield
committed by
Damien George
Damien George
parent
b9d6d6af4b
commit
88cb6bc818
@@ -8,6 +8,10 @@ import re
|
||||
from glob import glob
|
||||
from collections import defaultdict
|
||||
|
||||
run_tests_module = __import__("run-tests")
|
||||
sys.path.append(run_tests_module.base_path("../tools"))
|
||||
import pyboard
|
||||
|
||||
if os.name == "nt":
|
||||
MICROPYTHON = os.getenv(
|
||||
"MICROPY_MICROPYTHON", "../ports/windows/build-standard/micropython.exe"
|
||||
@@ -15,13 +19,39 @@ if os.name == "nt":
|
||||
else:
|
||||
MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/build-standard/micropython")
|
||||
|
||||
injected_bench_code = b"""
|
||||
import time
|
||||
|
||||
def run_tests(pyb, test_dict):
|
||||
class bench_class:
|
||||
ITERS = 20000000
|
||||
|
||||
@staticmethod
|
||||
def run(test):
|
||||
t = time.ticks_us()
|
||||
test(bench_class.ITERS)
|
||||
t = time.ticks_diff(time.ticks_us(), t)
|
||||
s, us = divmod(t, 1_000_000)
|
||||
print("{}.{:06}".format(s, us))
|
||||
|
||||
import sys
|
||||
sys.modules['bench'] = bench_class
|
||||
"""
|
||||
|
||||
|
||||
def execbench(pyb, filename, iters):
|
||||
with open(filename, "rb") as f:
|
||||
pyfile = f.read()
|
||||
code = (injected_bench_code + pyfile).replace(b"20000000", str(iters).encode("utf-8"))
|
||||
return pyb.exec(code).replace(b"\r\n", b"\n")
|
||||
|
||||
|
||||
def run_tests(pyb, test_dict, iters):
|
||||
test_count = 0
|
||||
testcase_count = 0
|
||||
|
||||
for base_test, tests in sorted(test_dict.items()):
|
||||
print(base_test + ":")
|
||||
baseline = None
|
||||
for test_file in tests:
|
||||
# run MicroPython
|
||||
if pyb is None:
|
||||
@@ -36,20 +66,25 @@ def run_tests(pyb, test_dict):
|
||||
# run on pyboard
|
||||
pyb.enter_raw_repl()
|
||||
try:
|
||||
output_mupy = pyb.execfile(test_file).replace(b"\r\n", b"\n")
|
||||
output_mupy = execbench(pyb, test_file[0], iters)
|
||||
except pyboard.PyboardError:
|
||||
output_mupy = b"CRASH"
|
||||
|
||||
output_mupy = float(output_mupy.strip())
|
||||
try:
|
||||
output_mupy = float(output_mupy.strip())
|
||||
except ValueError:
|
||||
output_mupy = -1
|
||||
test_file[1] = output_mupy
|
||||
testcase_count += 1
|
||||
|
||||
test_count += 1
|
||||
baseline = None
|
||||
for t in tests:
|
||||
if baseline is None:
|
||||
baseline = t[1]
|
||||
print(" %.3fs (%+06.2f%%) %s" % (t[1], (t[1] * 100 / baseline) - 100, t[0]))
|
||||
baseline = test_file[1]
|
||||
print(
|
||||
" %.3fs (%+06.2f%%) %s"
|
||||
% (test_file[1], (test_file[1] * 100 / baseline) - 100, test_file[0])
|
||||
)
|
||||
|
||||
test_count += 1
|
||||
|
||||
print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))
|
||||
|
||||
@@ -58,27 +93,47 @@ def run_tests(pyb, test_dict):
|
||||
|
||||
|
||||
def main():
|
||||
cmd_parser = argparse.ArgumentParser(description="Run tests for MicroPython.")
|
||||
cmd_parser.add_argument("--pyboard", action="store_true", help="run the tests on the pyboard")
|
||||
cmd_parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description=f"""Run and manage tests for MicroPython.
|
||||
|
||||
{run_tests_module.test_instance_description}
|
||||
{run_tests_module.test_directory_description}
|
||||
""",
|
||||
epilog=run_tests_module.test_instance_epilog,
|
||||
)
|
||||
cmd_parser.add_argument(
|
||||
"-t", "--test-instance", default="unix", help="the MicroPython instance to test"
|
||||
)
|
||||
cmd_parser.add_argument(
|
||||
"-b", "--baudrate", default=115200, help="the baud rate of the serial device"
|
||||
)
|
||||
cmd_parser.add_argument("-u", "--user", default="micro", help="the telnet login username")
|
||||
cmd_parser.add_argument("-p", "--password", default="python", help="the telnet login password")
|
||||
cmd_parser.add_argument(
|
||||
"-d", "--test-dirs", nargs="*", help="input test directories (if no files given)"
|
||||
)
|
||||
cmd_parser.add_argument(
|
||||
"-I",
|
||||
"--iters",
|
||||
type=int,
|
||||
default=200_000,
|
||||
help="number of test iterations, only for remote instances (default 200,000)",
|
||||
)
|
||||
cmd_parser.add_argument("files", nargs="*", help="input test files")
|
||||
args = cmd_parser.parse_args()
|
||||
|
||||
# Note pyboard support is copied over from run-tests.py, not tests, and likely needs revamping
|
||||
if args.pyboard:
|
||||
import pyboard
|
||||
|
||||
pyb = pyboard.Pyboard("/dev/ttyACM0")
|
||||
pyb.enter_raw_repl()
|
||||
else:
|
||||
pyb = None
|
||||
pyb = run_tests_module.get_test_instance(
|
||||
args.test_instance, args.baudrate, args.user, args.password
|
||||
)
|
||||
|
||||
if len(args.files) == 0:
|
||||
if pyb is None:
|
||||
# run PC tests
|
||||
test_dirs = ("internal_bench",)
|
||||
if args.test_dirs:
|
||||
test_dirs = tuple(args.test_dirs)
|
||||
else:
|
||||
# run pyboard tests
|
||||
test_dirs = ("basics", "float", "pyb")
|
||||
test_dirs = ("internal_bench",)
|
||||
|
||||
tests = sorted(
|
||||
test_file
|
||||
for test_files in (glob("{}/*.py".format(dir)) for dir in test_dirs)
|
||||
@@ -95,7 +150,7 @@ def main():
|
||||
continue
|
||||
test_dict[m.group(1)].append([t, None])
|
||||
|
||||
if not run_tests(pyb, test_dict):
|
||||
if not run_tests(pyb, test_dict, args.iters):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
||||
@@ -1145,29 +1145,11 @@ class append_filter(argparse.Action):
|
||||
args.filters.append((option, re.compile(value)))
|
||||
|
||||
|
||||
def main():
|
||||
global injected_import_hook_code
|
||||
|
||||
cmd_parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description="""Run and manage tests for MicroPython.
|
||||
|
||||
test_instance_description = """\
|
||||
By default the tests are run against the unix port of MicroPython. To run it
|
||||
against something else, use the -t option. See below for details.
|
||||
|
||||
Tests are discovered by scanning test directories for .py files or using the
|
||||
specified test files. If test files nor directories are specified, the script
|
||||
expects to be ran in the tests directory (where this file is located) and the
|
||||
builtin tests suitable for the target platform are ran.
|
||||
|
||||
When running tests, run-tests.py compares the MicroPython output of the test with the output
|
||||
produced by running the test through CPython unless a <test>.exp file is found, in which
|
||||
case it is used as comparison.
|
||||
|
||||
If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result
|
||||
directory with the MicroPython output and the expectations, respectively.
|
||||
""",
|
||||
epilog="""\
|
||||
"""
|
||||
test_instance_epilog = """\
|
||||
The -t option accepts the following for the test instance:
|
||||
- unix - use the unix port of MicroPython, specified by the MICROPY_MICROPYTHON
|
||||
environment variable (which defaults to the standard variant of either the unix
|
||||
@@ -1183,7 +1165,35 @@ The -t option accepts the following for the test instance:
|
||||
- execpty:<command> - execute a command and attach to the printed /dev/pts/<n> device
|
||||
- <a>.<b>.<c>.<d> - connect to the given IPv4 address
|
||||
- anything else specifies a serial port
|
||||
"""
|
||||
|
||||
test_directory_description = """\
|
||||
Tests are discovered by scanning test directories for .py files or using the
|
||||
specified test files. If test files nor directories are specified, the script
|
||||
expects to be ran in the tests directory (where this file is located) and the
|
||||
builtin tests suitable for the target platform are ran.
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
global injected_import_hook_code
|
||||
|
||||
cmd_parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description=f"""Run and manage tests for MicroPython.
|
||||
|
||||
{test_instance_description}
|
||||
{test_directory_description}
|
||||
|
||||
When running tests, run-tests.py compares the MicroPython output of the test with the output
|
||||
produced by running the test through CPython unless a <test>.exp file is found, in which
|
||||
case it is used as comparison.
|
||||
|
||||
If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result
|
||||
directory with the MicroPython output and the expectations, respectively.
|
||||
""",
|
||||
epilog=f"""\
|
||||
{test_instance_epilog}
|
||||
Options -i and -e can be multiple and processed in the order given. Regex
|
||||
"search" (vs "match") operation is used. An action (include/exclude) of
|
||||
the last matching regex is used:
|
||||
|
||||
Reference in New Issue
Block a user