From aaff82afe5a72ec69e05f1e56047d0acfde91d0e Mon Sep 17 00:00:00 2001
From: Paul Sokolovsky <pfalcon@users.sourceforge.net>
Date: Mon, 5 May 2014 01:24:16 +0300
Subject: [PATCH] tests: Add framework for comparative benchmarking.
Motivation is optimizing handling of various constructs as well as
understanding which constructs are more efficient in MicroPython.
More info: http://forum.micropython.org/viewtopic.php?f=3&t=77
Results are wildly unexpected. For example, "optimization" of range
iteration into while loop makes it twice as slow. Generally, the more
bytecodes, the slower the code.
---
tests/bench/bench.py | 10 +++
tests/bench/loop_count-1-range.py | 7 ++
tests/bench/loop_count-2-range_iter.py | 7 ++
tests/bench/loop_count-3-while_up.py | 8 ++
tests/bench/loop_count-4-while_down_gt.py | 7 ++
tests/bench/loop_count-5-while_down_ne.py | 7 ++
tests/bench/var-1-constant.py | 8 ++
tests/bench/var-2-global.py | 10 +++
tests/bench/var-3-local.py | 10 +++
tests/bench/var-4-arg.py | 9 +++
tests/run-bench-tests | 97 +++++++++++++++++++++++
11 files changed, 180 insertions(+)
create mode 100644 tests/bench/bench.py
create mode 100644 tests/bench/loop_count-1-range.py
create mode 100644 tests/bench/loop_count-2-range_iter.py
create mode 100644 tests/bench/loop_count-3-while_up.py
create mode 100644 tests/bench/loop_count-4-while_down_gt.py
create mode 100644 tests/bench/loop_count-5-while_down_ne.py
create mode 100644 tests/bench/var-1-constant.py
create mode 100644 tests/bench/var-2-global.py
create mode 100644 tests/bench/var-3-local.py
create mode 100644 tests/bench/var-4-arg.py
create mode 100755 tests/run-bench-tests
diff --git a/tests/bench/bench.py b/tests/bench/bench.py
new file mode 100644
index 000000000..0cd40a93f
--- /dev/null
+++ b/tests/bench/bench.py
@@ -0,0 +1,10 @@
+import time
+
+
+ITERS = 20000000
+
+def run(f):
+ t = time.time()
+ f(ITERS)
+ t = time.time() - t
+ print(t)
diff --git a/tests/bench/loop_count-1-range.py b/tests/bench/loop_count-1-range.py
new file mode 100644
index 000000000..e22adf6cb
--- /dev/null
+++ b/tests/bench/loop_count-1-range.py
@@ -0,0 +1,7 @@
+import bench
+
+def test(num):
+ for i in range(num):
+ pass
+
+bench.run(test)
diff --git a/tests/bench/loop_count-2-range_iter.py b/tests/bench/loop_count-2-range_iter.py
new file mode 100644
index 000000000..fe4a3857e
--- /dev/null
+++ b/tests/bench/loop_count-2-range_iter.py
@@ -0,0 +1,7 @@
+import bench
+
+def test(num):
+ for i in iter(range(num)):
+ pass
+
+bench.run(test)
diff --git a/tests/bench/loop_count-3-while_up.py b/tests/bench/loop_count-3-while_up.py
new file mode 100644
index 000000000..1ab8054a0
--- /dev/null
+++ b/tests/bench/loop_count-3-while_up.py
@@ -0,0 +1,8 @@
+import bench
+
+def test(num):
+ i = 0
+ while i < num:
+ i += 1
+
+bench.run(test)
diff --git a/tests/bench/loop_count-4-while_down_gt.py b/tests/bench/loop_count-4-while_down_gt.py
new file mode 100644
index 000000000..de8dee2ca
--- /dev/null
+++ b/tests/bench/loop_count-4-while_down_gt.py
@@ -0,0 +1,7 @@
+import bench
+
+def test(num):
+ while num > 0:
+ num -= 1
+
+bench.run(test)
diff --git a/tests/bench/loop_count-5-while_down_ne.py b/tests/bench/loop_count-5-while_down_ne.py
new file mode 100644
index 000000000..b9a1af414
--- /dev/null
+++ b/tests/bench/loop_count-5-while_down_ne.py
@@ -0,0 +1,7 @@
+import bench
+
+def test(num):
+ while num != 0:
+ num -= 1
+
+bench.run(test)
diff --git a/tests/bench/var-1-constant.py b/tests/bench/var-1-constant.py
new file mode 100644
index 000000000..eec977909
--- /dev/null
+++ b/tests/bench/var-1-constant.py
@@ -0,0 +1,8 @@
+import bench
+
+def test(num):
+ i = 0
+ while i < 20000000:
+ i += 1
+
+bench.run(test)
diff --git a/tests/bench/var-2-global.py b/tests/bench/var-2-global.py
new file mode 100644
index 000000000..5758ad61a
--- /dev/null
+++ b/tests/bench/var-2-global.py
@@ -0,0 +1,10 @@
+import bench
+
+ITERS = 20000000
+
+def test(num):
+ i = 0
+ while i < ITERS:
+ i += 1
+
+bench.run(test)
diff --git a/tests/bench/var-3-local.py b/tests/bench/var-3-local.py
new file mode 100644
index 000000000..124b48429
--- /dev/null
+++ b/tests/bench/var-3-local.py
@@ -0,0 +1,10 @@
+import bench
+
+
+def test(num):
+ ITERS = 20000000
+ i = 0
+ while i < ITERS:
+ i += 1
+
+bench.run(test)
diff --git a/tests/bench/var-4-arg.py b/tests/bench/var-4-arg.py
new file mode 100644
index 000000000..cf050c58f
--- /dev/null
+++ b/tests/bench/var-4-arg.py
@@ -0,0 +1,9 @@
+import bench
+
+
+def test(num):
+ i = 0
+ while i < num:
+ i += 1
+
+bench.run(lambda n:test(20000000))
diff --git a/tests/run-bench-tests b/tests/run-bench-tests
new file mode 100755
index 000000000..59074bb87
--- /dev/null
+++ b/tests/run-bench-tests
@@ -0,0 +1,97 @@
+#! /usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import argparse
+import re
+from glob import glob
+from collections import defaultdict
+
+# Tests require at least CPython 3.3. If your default python3 executable
+# is of lower version, you can point MICROPY_CPYTHON3 environment var
+# to the correct executable.
+if os.name == 'nt':
+ CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')
+ MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../windows/micropython.exe')
+else:
+ CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
+ MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../unix/micropython')
+
+def run_tests(pyb, test_dict):
+ test_count = 0
+ testcase_count = 0
+
+ for base_test, tests in test_dict.items():
+ print(base_test + ":")
+ for test_file in tests:
+
+ # run Micro Python
+ if pyb is None:
+ # run on PC
+ try:
+ output_mupy = subprocess.check_output([MICROPYTHON, '-X', 'emit=bytecode', test_file[0]])
+ except subprocess.CalledProcessError:
+ output_mupy = b'CRASH'
+ else:
+ # run on pyboard
+ pyb.enter_raw_repl()
+ try:
+ output_mupy = pyb.execfile(test_file).replace(b'\r\n', b'\n')
+ except pyboard.PyboardError:
+ output_mupy = b'CRASH'
+
+ output_mupy = float(output_mupy.strip())
+ test_file[1] = output_mupy
+ testcase_count += 1
+
+ test_count += 1
+ baseline = None
+ for t in tests:
+ if baseline is None:
+ baseline = t[1]
+ print(" %.3fs (%+06.2f%%) %s" % (t[1], (t[1] * 100 / baseline) - 100, t[0]))
+
+ print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))
+
+ # all tests succeeded
+ return True
+
+def main():
+ cmd_parser = argparse.ArgumentParser(description='Run tests for Micro Python.')
+ cmd_parser.add_argument('--pyboard', action='store_true', help='run the tests on the pyboard')
+ cmd_parser.add_argument('files', nargs='*', help='input test files')
+ args = cmd_parser.parse_args()
+
+ # Note pyboard support is copied over from run-tests, not testes, and likely needs revamping
+ if args.pyboard:
+ import pyboard
+ pyb = pyboard.Pyboard('/dev/ttyACM0')
+ pyb.enter_raw_repl()
+ else:
+ pyb = None
+
+ if len(args.files) == 0:
+ if pyb is None:
+ # run PC tests
+ test_dirs = ('bench',)
+ else:
+ # run pyboard tests
+ test_dirs = ('basics', 'float', 'pyb')
+ tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)
+ else:
+ # tests explicitly given
+ tests = sorted(args.files)
+
+ test_dict = defaultdict(lambda: [])
+ for t in tests:
+ m = re.match(r"(.+?)-(.+)\.py", t)
+ if not m:
+ continue
+ test_dict[m.group(1)].append([t, None])
+
+ if not run_tests(pyb, test_dict):
+ sys.exit(1)
+
+if __name__ == "__main__":
+ main()
--
GitLab