#!/usr/bin/env python3 """JTDX Contest Edition - decoder benchmark over a recorded hour of air. Runs the standalone decoder (jtdxjt9) over one of the published WAV suites, preset by preset, and prints what each preset found and what it cost in time on THIS machine, beside the numbers the same run produced on the reference machine. python3 jtdxbench.py --mode ft8 python3 jtdxbench.py --mode ft4 --presets default,recommended --periods 40 python3 jtdxbench.py --mode ft8 --jt9 /path/to/jtdxjt9 --wav /path/to/ft8_onair_1h --json mine.json Linux, macOS and Windows; Python 3.6 or newer, standard library only. Nothing is written outside a scratch directory that is removed again, and the WAV files are only read. Full options: --help. The suites, the reference numbers and what the columns mean are described in README.md beside this script and at https://ce3tsk.com/download/wav/ CE3TSK 2026-09-06 - GPL v3, as the program it measures. """ import argparse, json, math, os, platform, re, shutil, statistics, subprocess, sys, tempfile, time VERSION = '1.0' # --------------------------------------------------------------------------------------------- # The presets. opts/env are exactly what the GUI's preset sets in file mode; the reference # columns are one run of each over the whole 240-period suite on the machine described in # REFERENCE below. decodes = distinct messages summed over the periods; rx_s = the seconds the # decoder itself measured for the RX phase (the one that has to beat the reply deadline); # bg_s = the background phase that runs on while you transmit (None = the preset has none). # --------------------------------------------------------------------------------------------- PRESETS = { 'ft8': [ # tag name opts env dec rx_m rx_x bg_m bg_x ('classical', "Classical (JTDX's own recipe)", '-C 9 -E 2', {'JTDX_HINT_DEPTH': '1'}, 5249, 0.76, 1.32, None, None), ('maxeff', 'Max efficiency', '-C 5 -E 2', {}, 5417, 0.40, 0.73, None, None), ('maxdec', 'Max decodes', '-C 5 -E 2 -M 1', {}, 5678, 1.14, 2.03, None, None), ('light', 'Pipeline max decodes light', '-C 5 -E 2 -M 1 -B 1 -I 0 -D 6 -U 0 -n 0 -V 2', {}, 6091, 1.13, 1.74, 4.2, 6.7), ('ensemble', 'Ensemble', '-W -K 5 -X -M 3', {}, 5992, 4.52, 8.01, None, None), ('pipeens', 'Pipeline ensemble', '-W -K 4 -M 1 -B 1 -V 5', {}, 6194, 1.29, 2.04, 11.4, 18.8), ('pipefull', 'Pipeline ensemble full', '-W -K 4 -M 1 -B 1', {}, 6208, 1.29, 2.07, 12.7, 21.1), ('piperun', 'Pipeline run', '-C 5 -E 2 -B 1', {}, 6234, 0.40, 0.71, 12.9, 21.0), ], 'ft4': [ ('nohint', 'Without the hint memory', '', {'JTDX_FT4_HINT_DEPTH': '0'}, 1552, 0.09, 0.18, None, None), ('default', 'Default: deep, four passes', '', {}, 1737, 0.10, 0.67, None, None), ('bestpower', 'Best power: background 3', '-B 1 -V 3', {}, 1822, 0.10, 0.19, 0.3, 0.6), ('recommended', 'Recommended (best value)', '-B 1 -V 6', {'JTDX_FT4_BGOSD': '1', 'JTDX_FT4_BGALT': '1', 'JTDX_FT4_BGRESIDUAL': '1'}, 1881, 0.10, 0.19, 2.1, 4.9), ('mostreply', 'Most at reply time', '-M 6 -B 1 -V 6', {'JTDX_FT4_BGOSD': '1', 'JTDX_FT4_BGALT': '1', 'JTDX_FT4_BGRESIDUAL': '1'}, 1878, 0.48, 1.08, 0.5, 1.1), ('maxeffort', 'Max effort', '-M budget -l 13 -B 1 -V 6', {'JTDX_FT4_BGOSD': '1', 'JTDX_FT4_BGALT': '1', 'JTDX_FT4_BGRESIDUAL': '1', 'JTDX_FT4_SENS': '1', 'JTDX_FT4_BGSENS': '1'}, 1888, 0.59, 1.24, 0.6, 1.2), ], } # what a preset must beat for its decodes to be a reply rather than a log entry: FT8 has to # decide about 2.7 s into its 15 s period, FT4 within 1.36 s of its 7.5 s one (FT8_DECODER.md) DEADLINE = {'ft8': 2.7, 'ft4': 1.36} MODE_FLAG = {'ft8': '-8', 'ft4': '-4'} SUITE = {'ft8': 'ft8_onair_1h', 'ft4': 'ft4_onair_1h'} # the crowded-band suite: one synthetic period per mode, every signal in it known. Unlike the # recorded hours this one has a truth set, so it scores what is real and what is invented. CROWDED = {'ft8': ('crowded_band', 'ft8_full_band_16.wav', 'ft8_full_band_truth.tsv'), 'ft4': ('crowded_band', 'ft4_full_band_16.wav', 'ft4_full_band_truth.tsv')} REFERENCE = { 'machine': 'AMD Ryzen 7 5800H, 8 cores / 16 threads, powersave governor, 14 GB', 'os': 'Linux Mint 22.3, kernel 7.0.0-30-generic x86_64', 'toolchain': 'gfortran 13.3.0 -O3, g++ 13.3.0 -O3, fftw3-single 3.3.10', 'engine': 'jtdxjt9 8a3e17ee (3.0.0-rc02 decoder)', 'date': '2026-09-05', 'threads': 12, 'periods': 240, 'mycall': 'CE3TSK', 'mygrid': 'FF46', 'band': '100-3100 Hz', } # the decode lines and the two phase markers. The message is the 26-character field after the # mode separator (~ FT8, : FT4); the marker that may follow it is not part of the message. LINE = re.compile(r'(\d{6})\s+(-?\d+)\s+(-?\d+\.\d)\s+(\d+)\s+[~:+]\s+(.{0,26}?)\s*([^\w\s\-/+<>.]*)\s*$') RXS = re.compile(r'\s*([0-9.]+)') BGS = re.compile(r'\s*([0-9.]+)') def find_decoder(explicit): """The standalone decoder: --jt9, then $JTDXJT9, then the usual places, then PATH.""" exe = 'jtdxjt9.exe' if os.name == 'nt' else 'jtdxjt9' if explicit: if os.path.isfile(explicit) and os.access(explicit, os.X_OK): return os.path.abspath(explicit) sys.exit('--jt9 %s: not an executable file' % explicit) here = os.path.dirname(os.path.abspath(__file__)) cands = [os.environ.get('JTDXJT9'), os.path.join(here, exe), os.path.join(here, '..', exe)] if os.name == 'nt': for pf in (os.environ.get('ProgramFiles', r'C:\Program Files'), os.environ.get('ProgramFiles(x86)', '')): if pf: cands += [os.path.join(pf, d, 'bin', exe) for d in ('JTDX_contest', 'jtdx_contest', 'JTDX')] elif sys.platform == 'darwin': cands += ['/Applications/jtdx_contest.app/Contents/MacOS/' + exe, os.path.expanduser('~/jtdx-prefix/bin/' + exe)] else: cands += [os.path.expanduser('~/jtdx-prefix/bin/' + exe), '/usr/local/bin/' + exe, '/usr/bin/' + exe] for c in cands: if c and os.path.isfile(c) and os.access(c, os.X_OK): return os.path.abspath(c) p = shutil.which(exe) if p: return os.path.abspath(p) sys.exit("no %s found. Point at it with --jt9 (an AppImage: run it once with --appimage-extract,\n" "the decoder is squashfs-root/usr/bin/%s), or set JTDXJT9." % (exe, exe)) def find_suite(mode, explicit): if explicit: d = os.path.abspath(explicit) if not os.path.isdir(d): sys.exit('--wav %s: not a directory' % explicit) else: here = os.path.dirname(os.path.abspath(__file__)) for c in (os.path.join(here, '..', SUITE[mode]), os.path.join(here, SUITE[mode]), os.path.join(os.getcwd(), SUITE[mode])): if os.path.isdir(c): d = os.path.abspath(c); break else: sys.exit('no %s directory found beside this script - unpack the suite, or pass --wav DIR' % SUITE[mode]) files = sorted(f for f in os.listdir(d) if f.lower().endswith('.wav')) if not files: sys.exit('%s holds no .wav files' % d) return d, files def parse(out): """One dict per period: the RX and background message sets and the seconds each phase took. A period without a background phase closes at , one with it at - the same rule the project's own analysis uses.""" lines = out.split('\n') hasbg = any(l.startswith('') for l in lines) periods = []; cur = None; phase = 'rx' new = lambda: {'rx': set(), 'bg': set(), 'rxs': 0.0, 'bgs': 0.0} for l in lines: m = LINE.match(l) if m: if cur is None: cur = new() msg = re.sub(r'\s+', ' ', m.group(5).strip()) if msg: cur[phase].add(msg) elif l.startswith(''): if cur is None: cur = new() r = RXS.search(l); cur['rxs'] = float(r.group(1)) if r else 0.0 if hasbg: phase = 'bg' else: periods.append(cur); cur = None elif l.startswith(''): if cur is None: cur = new() r = BGS.search(l); cur['bgs'] = float(r.group(1)) if r else 0.0 periods.append(cur); cur = None; phase = 'rx' if cur is not None: periods.append(cur) return periods def run_preset(jt9, mode, opts, env, workdir, wavdir, files, args): """One decoder process over the whole suite in time order, as it runs on air: the hint memory and the previous-period feedback carry from period to period inside one process.""" cmd = [jt9, MODE_FLAG[mode], '-d', '3', '-j', str(args.threads), '-L', str(args.low), '-H', str(args.high), '-c', args.mycall, '-G', args.mygrid, '-a', '.', '-t', '.', '-r', '.'] + opts.split() # relative paths where that keeps the command line short (Windows caps it at 32767 chars); # relpath raises when the scratch directory landed on another drive, so fall back to absolute try: rel = os.path.relpath(wavdir, workdir) except ValueError: rel = wavdir prefix = rel if len(rel) < len(wavdir) else wavdir cmd += [os.path.join(prefix, f) for f in files] if os.name == 'nt' and sum(len(a) + 1 for a in cmd) > 30000: sys.exit('the command line would be %d characters, past what Windows accepts. Move the\n' 'suite to a shorter path (C:\\ft8 say), or run fewer periods with --periods.' % sum(len(a) + 1 for a in cmd)) e = dict(os.environ); e.update(env); e['JTDX_STOPHINT'] = '1' # an idle GUI: the DX-call search runs as on air t0 = time.time() p = subprocess.Popen(cmd, cwd=workdir, env=e, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf-8', errors='replace') out, _ = p.communicate() return out, time.time() - t0, p.returncode def summarise(periods, wall, deadline): if not periods: return None msgs = sum(len(p['rx'] | p['bg']) for p in periods) rx = [p['rxs'] for p in periods]; bg = [p['bgs'] for p in periods] hasbg = max(bg) > 0 return { 'periods': len(periods), 'decodes': msgs, 'rx_msgs': sum(len(p['rx']) for p in periods), 'bg_msgs': sum(len(p['bg']) for p in periods) if hasbg else None, 'rx_s_mean': statistics.mean(rx), 'rx_s_max': max(rx), 'bg_s_mean': statistics.mean(bg) if hasbg else None, 'bg_s_max': max(bg) if hasbg else None, 'total_s': statistics.mean(rx) + (statistics.mean(bg) if hasbg else 0.0), 'in_deadline': sum(1 for s in rx if s <= deadline), 'wall_s': wall, } def truth_set(path): """The manifest's messages and the band they occupy. The FT8 file is the reference set of the crowded FT8 period (104 messages, each corroborated across decoder families); the FT4 one is the synthetic manifest, where 'kept' marks a signal actually placed in the file - so for FT4 anything else printed is demonstrably false, while for FT8 it is only 'not in the reference set'. Returns (messages, hard, lowest Hz, highest Hz).""" rows = [l.rstrip('\n').split('\t') for l in open(path) if l.strip() and not l.startswith('#')] head, rows = rows[0], rows[1:] hard = 'status' in head if hard: # FT4: only the signals really synthesised i = head.index('status') rows = [r for r in rows if len(r) > i and r[i] == 'kept'] fi = head.index('ft4_hz') if 'ft4_hz' in head else head.index('freq') hz = [float(r[fi]) for r in rows if len(r) > fi] return {r[0] for r in rows}, hard, min(hz), max(hz) def run_crowded(jt9, mode, args, workdir, header): """Every preset of the mode over the one crowded-band period, scored against the manifest.""" sub, wav, man = CROWDED[mode] here = os.path.dirname(os.path.abspath(__file__)) base = args.wav or next((c for c in (os.path.join(here, '..', sub), os.path.join(here, sub), os.path.join(os.getcwd(), sub)) if os.path.isdir(c)), None) if not base or not os.path.isdir(base): sys.exit('no %s directory found - pass --wav DIR' % sub) base = os.path.abspath(base) if not os.path.isfile(os.path.join(base, wav)): sys.exit('%s: no %s in it' % (base, wav)) kept, hard, lo, hi = truth_set(os.path.join(base, man)) # the FT4 twin reaches to 4.9 kHz: scored over the on-air 100-3100 Hz it would be marked # down for signals the range cannot reach. Unless the range was given, take the manifest's. ceiling = 4969 if mode == 'ft4' else 3650 # what the decoder itself accepts per mode if args.low is None: args.low = max(50, int(lo) - 30) if args.high is None: args.high = min(ceiling, int(hi) + 30) header('crowded_band') print('%s, %d signals in the manifest between %.0f and %.0f Hz; scored over %d-%d Hz' % (wav, len(kept), lo, hi, args.low, args.high)) # --repeat: the same period handed to the decoder N times running, which is what re-opening # the file in the GUI does. The audio never changes, so anything the later periods add came # from the hint memory carrying what the earlier ones decoded. n = max(1, args.repeat) step = 15 if mode == 'ft8' else 8 # the mode's period, so the stamps are plausible stamps = ['%02d%02d%02d' % (i * step // 3600, i * step // 60 % 60, i * step % 60) for i in range(n)] for st in stamps: shutil.copyfile(os.path.join(base, wav), os.path.join(workdir, 'p_%s.wav' % st)) if n > 1: print('each preset over %d consecutive copies of it; "best" is the best single period' % n) print() w = (14, 9, 9, 9, 11, 9) if n == 1 else (14, 9, 7, 9, 9, 11, 9) hdr = (('preset', 'found', 'of', 'missed', 'false' if hard else 'not in set', 'RX s') if n == 1 else ('preset', 'found', 'best', 'of', 'missed', 'false' if hard else 'not in set', 'RX s')) print(''.join(h.rjust(x) if i else h.ljust(x) for i, (h, x) in enumerate(zip(hdr, w)))) print('-' * sum(w)) out_rows = {} for tag, name, opts, env, *_ in [r for r in PRESETS[mode] if r[0] in args._want]: out, wall, rc = run_preset(jt9, mode, opts, env, workdir, workdir, ['p_%s.wav' % st for st in stamps], args) per = parse(out) sets = [p['rx'] | p['bg'] for p in per] or [set()] found = [len(g & kept) for g in sets]; false = [len(g - kept) for g in sets] t, f, best = found[0], max(false), max(found) rxs = per[0]['rxs'] if per else 0.0 cells = ((tag, str(t), str(len(kept)), str(len(kept) - t), str(f), '%.2f' % rxs) if n == 1 else (tag, str(t), str(best), str(len(kept)), str(len(kept) - best), str(f), '%.2f' % rxs)) print(''.join(c.rjust(x) if i else c.ljust(x) for i, (c, x) in enumerate(zip(cells, w)))) out_rows[tag] = {'found': t, 'best': best, 'per_period_found': found, 'of': len(kept), 'missed': len(kept) - best, 'false' if hard else 'not_in_set': f, 'rx_s': rxs, 'wall_s': wall, 'name': name, 'opts': opts, 'repeat': n} if n > 1: gain = [(r['best'] - r['found']) for r in out_rows.values()] print('\nThe audio was identical in all %d periods, so "best" - "found" (%+d to %+d here) is what' '\nthe hint memory carried from one period into the next.' % (n, min(gain), max(gain))) print('\n%s' % ('Anything outside the manifest is a false decode: the file was synthesised from it.' if hard else 'The FT8 reference set is a union over many decoders, so a message outside it is\n' 'unconfirmed rather than proven false.')) return out_rows def machine(): try: import multiprocessing; cpus = multiprocessing.cpu_count() except Exception: cpus = 0 name = platform.processor() or platform.machine() try: # a readable CPU name where the OS offers one if sys.platform.startswith('linux'): for l in open('/proc/cpuinfo'): if l.startswith('model name'): name = l.split(':', 1)[1].strip(); break elif sys.platform == 'darwin': name = subprocess.check_output(['sysctl', '-n', 'machdep.cpu.brand_string'], universal_newlines=True).strip() elif os.name == 'nt': name = os.environ.get('PROCESSOR_IDENTIFIER', name) except Exception: pass return {'cpu': name, 'logical_cpus': cpus, 'os': platform.platform(), 'python': platform.python_version(), 'machine': platform.machine()} def main(): ap = argparse.ArgumentParser(description='JTDX Contest Edition decoder benchmark', formatter_class=argparse.RawDescriptionHelpFormatter, epilog='Reference machine: %s, %s (%s).' % (REFERENCE['machine'], REFERENCE['engine'], REFERENCE['date'])) ap.add_argument('--mode', choices=('ft8', 'ft4'), required=True, help='which suite to run') ap.add_argument('--jt9', help='the standalone decoder (default: found beside this script, $JTDXJT9, or PATH)') ap.add_argument('--wav', help='the suite directory (default: the mode\'s suite found beside this script)') # 'main' rather than 'default' as the name of the selection: FT4 has a preset TAGGED # default, and --presets default must mean that one preset, not a selection containing it ap.add_argument('--presets', default='main', metavar='LIST', help="comma-separated tags, 'all', or 'main' (a representative few, the default); --list shows them") ap.add_argument('--periods', type=int, default=0, metavar='N', help='use only the first N periods (default: all)') ap.add_argument('--threads', type=int, default=12, help='decoder threads (default 12, the reference setting)') ap.add_argument('--mycall', default=REFERENCE['mycall'], help='changing it changes the numbers: AP and the hint memory follow it') ap.add_argument('--mygrid', default=REFERENCE['mygrid']) ap.add_argument('--low', type=int, help='decode range low edge, Hz (default 100; with --crowded, the manifest\'s)') ap.add_argument('--high', type=int, help='decode range high edge, Hz (default 3100; with --crowded, the manifest\'s)') ap.add_argument('--json', metavar='FILE', help='also write the results as JSON') ap.add_argument('--keep', metavar='DIR', help='keep the raw decoder output in DIR') ap.add_argument('--crowded', action='store_true', help='run the crowded-band period instead of the hour, scored against its truth manifest') ap.add_argument('--repeat', type=int, default=1, metavar='N', help='with --crowded: hand the same period to the decoder N times running, as re-opening ' 'the file does, and report the best single period - what the hint memory adds') ap.add_argument('--list', action='store_true', help='list the presets of --mode and exit') ap.add_argument('--version', action='version', version='jtdxbench %s' % VERSION) args = ap.parse_args() table = PRESETS[args.mode] if args.list: print('%-13s %-38s %s' % ('tag', 'preset', 'options')) for tag, name, opts, env, *_ in table: print('%-13s %-38s %s' % (tag, name, ' '.join([opts] + ['%s=%s' % kv for kv in sorted(env.items())]).strip())) return 0 # 'main': the presets worth a first look - the cheapest, the recommended and the deepest MAIN = {'ft8': ['classical', 'maxeff', 'maxdec', 'light'], 'ft4': ['default', 'bestpower', 'recommended', 'mostreply', 'maxeffort']} if args.presets == 'all': want = [t for t, *_ in table] elif args.presets == 'main': want = MAIN[args.mode] else: want = [t.strip() for t in args.presets.split(',') if t.strip()] known = {t for t, *_ in table} bad = [t for t in want if t not in known] if bad: sys.exit('unknown preset(s) for %s: %s (--list shows them)' % (args.mode, ', '.join(bad))) rows = [r for r in table if r[0] in want] jt9 = find_decoder(args.jt9) args._want = want if not args.crowded: # the recorded hours were measured over the on-air range if args.low is None: args.low = 100 if args.high is None: args.high = 3100 mi = machine() if mi['logical_cpus'] and args.threads > mi['logical_cpus']: print('note: --threads %d on a machine with %d logical CPUs - the reference used 12 threads on 16.' % (args.threads, mi['logical_cpus']), file=sys.stderr) def header(suite): print('JTDX Contest Edition decoder benchmark %s' % VERSION) print(' decoder : %s' % jt9) print(' suite : %s (%s)' % (suite, args.mode.upper())) print(' machine : %s, %d logical CPUs, %s' % (mi['cpu'], mi['logical_cpus'], mi['os'])) print(' settings: %d threads, %d-%d Hz, mycall %s %s' % (args.threads, args.low, args.high, args.mycall, args.mygrid)) print(' presets : %s' % ', '.join(want)) print() if args.crowded: workdir = tempfile.mkdtemp(prefix='jtdxbench_') try: scored = run_crowded(jt9, args.mode, args, workdir, header) finally: shutil.rmtree(workdir, ignore_errors=True) if args.json: with open(args.json, 'w') as f: json.dump({'jtdxbench': VERSION, 'mode': args.mode, 'suite': 'crowded_band', 'threads': args.threads, 'decoder': jt9, 'machine': mi, 'results': scored}, f, indent=1, sort_keys=True, default=str) print('\nwritten: %s' % args.json) return 0 wavdir, files = find_suite(args.mode, args.wav) if args.periods: files = files[:args.periods] header('%s, %d periods' % (os.path.basename(wavdir), len(files))) if args.keep: os.makedirs(args.keep, exist_ok=True) # scratch beside the suite where that is writable (it keeps the command line short), else /tmp try: workdir = tempfile.mkdtemp(prefix='.jtdxbench_', dir=os.path.dirname(wavdir)) except OSError: workdir = tempfile.mkdtemp(prefix='jtdxbench_') results = {} try: for tag, name, opts, env, rdec, rrx, rrxx, rbg, rbgx in rows: sys.stdout.write(' %-13s running ... ' % tag); sys.stdout.flush() out, wall, rc = run_preset(jt9, args.mode, opts, env, workdir, wavdir, files, args) if args.keep: with open(os.path.join(args.keep, tag + '.out'), 'w') as f: f.write(out) s = summarise(parse(out), wall, DEADLINE[args.mode]) if s is None or s['periods'] == 0: print('NO OUTPUT (exit %d)' % rc) print(' %s' % (out.strip().split('\n')[-1] if out.strip() else '(decoder printed nothing)')) continue if s['periods'] != len(files): print('%d of %d periods closed - ' % (s['periods'], len(files)), end='') s.update(tag=tag, name=name, opts=opts, env=env, exit=rc, ref={'decodes': rdec, 'rx_s_mean': rrx, 'rx_s_max': rrxx, 'bg_s_mean': rbg, 'bg_s_max': rbgx, 'periods': REFERENCE['periods'], 'total_s': rrx + (rbg or 0.0)}) results[tag] = s print('%d msgs, RX %.2f s, %.0f s wall' % (s['decodes'], s['rx_s_mean'], wall)) finally: shutil.rmtree(workdir, ignore_errors=True) if not results: sys.exit('nothing ran') report(args, mi, jt9, wavdir, files, results) if args.json: with open(args.json, 'w') as f: json.dump({'jtdxbench': VERSION, 'mode': args.mode, 'periods': len(files), 'threads': args.threads, 'decoder': jt9, 'machine': mi, 'reference': REFERENCE, 'results': results}, f, indent=1, sort_keys=True, default=str) print('\nwritten: %s' % args.json) return 0 def report(args, mi, jt9, wavdir, files, results): full = len(files) == REFERENCE['periods'] dl = DEADLINE[args.mode] print('\n%s: this machine over %d periods%s\n' % (args.mode.upper(), len(files), '' if full else ' (a subset - the decode counts are not comparable with the reference)')) hdr = ('preset', 'decodes', 'RX s mean', 'RX s max', 'bg s mean', 'total s/period', 'wall s', 'in %.2gs' % dl) w = (14, 9, 10, 9, 10, 15, 8, 9) print(''.join(h.rjust(x) if i else h.ljust(x) for i, (h, x) in enumerate(zip(hdr, w)))) print('-' * sum(w)) for tag, s in results.items(): cells = (tag, str(s['decodes']), '%.2f' % s['rx_s_mean'], '%.2f' % s['rx_s_max'], '-' if s['bg_s_mean'] is None else '%.2f' % s['bg_s_mean'], '%.2f' % s['total_s'], '%.0f' % s['wall_s'], '%d/%d' % (s['in_deadline'], s['periods'])) print(''.join(c.rjust(x) if i else c.ljust(x) for i, (c, x) in enumerate(zip(cells, w)))) print('\nAgainst the reference machine (%s,\n%s%s, %s):\n' % (REFERENCE['machine'], ' ' * 4, REFERENCE['engine'], REFERENCE['date'])) hdr = ('preset', 'decodes', 'reference', 'delta', 'RX s', 'reference', 'speed') w = (14, 9, 11, 9, 9, 11, 9) print(''.join(h.rjust(x) if i else h.ljust(x) for i, (h, x) in enumerate(zip(hdr, w)))) print('-' * sum(w)) ratios = [] for tag, s in results.items(): r = s['ref']; ratio = r['total_s'] / s['total_s'] if s['total_s'] else 0 ratios.append(ratio) d = ('%+.1f %%' % (100.0 * (s['decodes'] - r['decodes']) / r['decodes'])) if full else 'n/a' cells = (tag, str(s['decodes']), str(r['decodes']), d, '%.2f' % s['rx_s_mean'], '%.2f' % r['rx_s_mean'], '%.2fx' % ratio) print(''.join(c.rjust(x) if i else c.ljust(x) for i, (c, x) in enumerate(zip(cells, w)))) good = [r for r in ratios if r > 0] speed = math.exp(sum(math.log(r) for r in good) / len(good)) if good else 0 print('\nSpeed index: %.2fx the reference machine (geometric mean over %d preset%s of the' '\n reference seconds per period divided by this machine\'s). Above 1 is faster.' % (speed, len(good), '' if len(good) == 1 else 's')) late = [(t, s) for t, s in results.items() if s['in_deadline'] < s['periods']] if late: print('\nPresets that missed the %.2g s %s reply deadline in some periods on this machine:' % (dl, args.mode.upper())) for t, s in late: print(' %-13s %d of %d periods late, worst %.2f s' % (t, s['periods'] - s['in_deadline'], s['periods'], s['rx_s_max'])) print(' (a late RX phase still decodes - it decides the reply after the answer was due)') else: print('\nEvery preset run decided inside the %.2g s %s reply deadline in every period.' % (dl, args.mode.upper())) if not full: print('\nThe decode counts above come from %d of the suite\'s %d periods; run without' '\n--periods for numbers comparable with the reference.' % (len(files), REFERENCE['periods'])) if __name__ == '__main__': try: sys.exit(main()) except KeyboardInterrupt: sys.exit(130)