scripts/simplebench/results_to_text.py

   1 #!/usr/bin/env python3
   2 #
   3 # Simple benchmarking framework
   4 #
   5 # Copyright (c) 2019 Virtuozzo International GmbH.
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20
  21 import math
  22 import tabulate
  23
  24 # We want leading whitespace for difference row cells (see below)
  25 tabulate.PRESERVE_WHITESPACE = True
  26
  27
  28 def format_value(x, stdev):
  29     stdev_pr = stdev / x * 100
  30     if stdev_pr < 1.5:
  31         # don't care too much
  32         return f'{x:.2g}'
  33     else:
  34         return f'{x:.2g} ± {math.ceil(stdev_pr)}%'
  35
  36
  37 def result_to_text(result):
  38     """Return text representation of bench_one() returned dict."""
  39     if 'average' in result:
  40         s = format_value(result['average'], result['stdev'])
  41         if 'n-failed' in result:
  42             s += '\n({} failed)'.format(result['n-failed'])
  43         return s
  44     else:
  45         return 'FAILED'
  46
  47
  48 def results_dimension(results):
  49     dim = None
  50     for case in results['cases']:
  51         for env in results['envs']:
  52             res = results['tab'][case['id']][env['id']]
  53             if dim is None:
  54                 dim = res['dimension']
  55             else:
  56                 assert dim == res['dimension']
  57
  58     assert dim in ('iops', 'seconds')
  59
  60     return dim
  61
  62
  63 def results_to_text(results):
  64     """Return text representation of bench() returned dict."""
  65     n_columns = len(results['envs'])
  66     named_columns = n_columns > 2
  67     dim = results_dimension(results)
  68     tab = []
  69
  70     if named_columns:
  71         # Environment columns are named A, B, ...
  72         tab.append([''] + [chr(ord('A') + i) for i in range(n_columns)])
  73
  74     tab.append([''] + [c['id'] for c in results['envs']])
  75
  76     for case in results['cases']:
  77         row = [case['id']]
  78         case_results = results['tab'][case['id']]
  79         for env in results['envs']:
  80             res = case_results[env['id']]
  81             row.append(result_to_text(res))
  82         tab.append(row)
  83
  84         # Add row of difference between columns. For each column starting from
  85         # B we calculate difference with all previous columns.
  86         row = ['', '']  # case name and first column
  87         for i in range(1, n_columns):
  88             cell = ''
  89             env = results['envs'][i]
  90             res = case_results[env['id']]
  91
  92             if 'average' not in res:
  93                 # Failed result
  94                 row.append(cell)
  95                 continue
  96
  97             for j in range(0, i):
  98                 env_j = results['envs'][j]
  99                 res_j = case_results[env_j['id']]
 100                 cell += ' '
 101
 102                 if 'average' not in res_j:
 103                     # Failed result
 104                     cell += '--'
 105                     continue
 106
 107                 col_j = tab[0][j + 1] if named_columns else ''
 108                 diff_pr = round((res['average'] - res_j['average']) /
 109                                 res_j['average'] * 100)
 110                 cell += f' {col_j}{diff_pr:+}%'
 111             row.append(cell)
 112         tab.append(row)
 113
 114     return f'All results are in {dim}\n\n' + tabulate.tabulate(tab)
 115
 116
 117 if __name__ == '__main__':
 118     import sys
 119     import json
 120
 121     if len(sys.argv) < 2:
 122         print(f'USAGE: {sys.argv[0]} results.json')
 123         exit(1)
 124
 125     with open(sys.argv[1]) as f:
 126         print(results_to_text(json.load(f)))