scripts/simplebench/img_bench_templater.py

   1 #!/usr/bin/env python3
   2 #
   3 # Process img-bench test templates
   4 #
   5 # Copyright (c) 2021 Virtuozzo International GmbH.
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20
  21
  22 import sys
  23 import subprocess
  24 import re
  25 import json
  26
  27 import simplebench
  28 from results_to_text import results_to_text
  29 from table_templater import Templater
  30
  31
  32 def bench_func(env, case):
  33     test = templater.gen(env['data'], case['data'])
  34
  35     p = subprocess.run(test, shell=True, stdout=subprocess.PIPE,
  36                        stderr=subprocess.STDOUT, universal_newlines=True)
  37
  38     if p.returncode == 0:
  39         try:
  40             m = re.search(r'Run completed in (\d+.\d+) seconds.', p.stdout)
  41             return {'seconds': float(m.group(1))}
  42         except Exception:
  43             return {'error': f'failed to parse qemu-img output: {p.stdout}'}
  44     else:
  45         return {'error': f'qemu-img failed: {p.returncode}: {p.stdout}'}
  46
  47
  48 if __name__ == '__main__':
  49     if len(sys.argv) > 1:
  50         print("""
  51 Usage: img_bench_templater.py < path/to/test-template.sh
  52
  53 This script generates performance tests from a test template (example below),
  54 runs them, and displays the results in a table. The template is read from
  55 stdin.  It must be written in bash and end with a `qemu-img bench` invocation
  56 (whose result is parsed to get the test instance’s result).
  57
  58 Use the following syntax in the template to create the various different test
  59 instances:
  60
  61   column templating: {var1|var2|...} - test will use different values in
  62   different columns. You may use several {} constructions in the test, in this
  63   case product of all choice-sets will be used.
  64
  65   row templating: [var1|var2|...] - similar thing to define rows (test-cases)
  66
  67 Test template example:
  68
  69 Assume you want to compare two qemu-img binaries, called qemu-img-old and
  70 qemu-img-new in your build directory in two test-cases with 4K writes and 64K
  71 writes. The template may look like this:
  72
  73 qemu_img=/path/to/qemu/build/qemu-img-{old|new}
  74 $qemu_img create -f qcow2 /ssd/x.qcow2 1G
  75 $qemu_img bench -c 100 -d 8 [-s 4K|-s 64K] -w -t none -n /ssd/x.qcow2
  76
  77 When passing this to stdin of img_bench_templater.py, the resulting comparison
  78 table will contain two columns (for two binaries) and two rows (for two
  79 test-cases).
  80
  81 In addition to displaying the results, script also stores results in JSON
  82 format into results.json file in current directory.
  83 """)
  84         sys.exit()
  85
  86     templater = Templater(sys.stdin.read())
  87
  88     envs = [{'id': ' / '.join(x), 'data': x} for x in templater.columns]
  89     cases = [{'id': ' / '.join(x), 'data': x} for x in templater.rows]
  90
  91     result = simplebench.bench(bench_func, envs, cases, count=5,
  92                                initial_run=False)
  93     print(results_to_text(result))
  94     with open('results.json', 'w') as f:
  95         json.dump(result, f, indent=4)