2 * Copyright (c) 2018 Jiri Svoboda
3 * Copyright (c) 2018 Vojtech Horky
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * - Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * - Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * - The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 /** @addtogroup hbench
46 #include <str_error.h>
48 #include <types/casting.h>
51 #define MIN_DURATION_SECS 10
52 #define NUM_SAMPLES 10
53 #define MAX_ERROR_STR_LENGTH 1024
55 static void short_report(bench_run_t
*info
, int run_index
,
56 benchmark_t
*bench
, uint64_t workload_size
)
58 csv_report_add_entry(info
, run_index
, bench
, workload_size
);
60 usec_t duration_usec
= NSEC2USEC(stopwatch_get_nanos(&info
->stopwatch
));
62 printf("Completed %" PRIu64
" operations in %llu us",
63 workload_size
, duration_usec
);
64 if (duration_usec
> 0) {
65 double nanos
= stopwatch_get_nanos(&info
->stopwatch
);
66 double thruput
= (double) workload_size
/ (nanos
/ 1000000000.0l);
67 printf(", %.0f ops/s.\n", thruput
);
73 /** Estimate square root value.
75 * @param value The value to compute square root of.
76 * @param precision Required precision (e.g. 0.00001).
80 * This is a temporary solution until we have proper sqrt() implementation
83 * The algorithm uses Babylonian method [1].
85 * [1] https://en.wikipedia.org/wiki/Methods_of_computing_square_roots#Babylonian_method
87 static double estimate_square_root(double value
, double precision
)
90 double prev_estimate
= estimate
+ 10 * precision
;
92 while (fabs(estimate
- prev_estimate
) > precision
) {
93 prev_estimate
= estimate
;
94 estimate
= (prev_estimate
+ value
/ prev_estimate
) / 2.;
100 /** Compute available statistics from given stopwatches.
102 * We compute normal mean for average duration of the workload and geometric
103 * mean for average thruput. Note that geometric mean is necessary to compute
104 * average throughput correctly - consider the following example:
105 * - we run always 60 operations,
106 * - first run executes in 30 s (i.e. 2 ops/s)
107 * - and second one in 10 s (6 ops/s).
108 * Then, naively, average throughput would be (2+6)/2 = 4 [ops/s]. However, we
109 * actually executed 60 + 60 ops in 30 + 10 seconds. So the actual average
110 * throughput is 3 ops/s (which is exactly what geometric mean means).
113 static void compute_stats(bench_run_t
*runs
, size_t run_count
,
114 uint64_t workload_size
, double precision
, double *out_duration_avg
,
115 double *out_duration_sigma
, double *out_thruput_avg
)
117 double inv_thruput_sum
= 0.0;
118 double nanos_sum
= 0.0;
119 double nanos_sum2
= 0.0;
121 for (size_t i
= 0; i
< run_count
; i
++) {
122 double nanos
= stopwatch_get_nanos(&runs
[i
].stopwatch
);
123 double thruput
= (double) workload_size
/ nanos
;
125 inv_thruput_sum
+= 1.0 / thruput
;
127 nanos_sum2
+= nanos
* nanos
;
129 *out_duration_avg
= nanos_sum
/ run_count
;
130 double sigma2
= (nanos_sum2
- nanos_sum
* (*out_duration_avg
)) /
131 ((double) run_count
- 1);
132 // FIXME: implement sqrt properly
133 *out_duration_sigma
= estimate_square_root(sigma2
, precision
);
134 *out_thruput_avg
= 1.0 / (inv_thruput_sum
/ run_count
);
137 static void summary_stats(bench_run_t
*runs
, size_t run_count
,
138 benchmark_t
*bench
, uint64_t workload_size
)
140 double duration_avg
, duration_sigma
, thruput_avg
;
141 compute_stats(runs
, run_count
, workload_size
, 0.001,
142 &duration_avg
, &duration_sigma
, &thruput_avg
);
144 printf("Average: %" PRIu64
" ops in %.0f us (sd %.0f us); "
145 "%.0f ops/s; Samples: %zu\n",
146 workload_size
, duration_avg
/ 1000.0, duration_sigma
/ 1000.0,
147 thruput_avg
* 1000000000.0, run_count
);
150 static bool run_benchmark(benchmark_t
*bench
)
152 printf("Warm up and determine workload size...\n");
155 * We share this buffer across all runs as we know that it is
156 * used only on failure (and we abort after first error).
158 char *error_msg
= malloc(MAX_ERROR_STR_LENGTH
+ 1);
159 if (error_msg
== NULL
) {
160 printf("Out of memory!\n");
163 str_cpy(error_msg
, MAX_ERROR_STR_LENGTH
, "");
165 bench_run_t helper_run
;
166 bench_run_init(&helper_run
, error_msg
, MAX_ERROR_STR_LENGTH
);
170 if (bench
->setup
!= NULL
) {
171 ret
= bench
->setup(&helper_run
);
178 * Find workload size that is big enough to last few seconds.
179 * We also check that uint64_t is big enough.
181 uint64_t workload_size
= 0;
182 for (size_t bits
= 0; bits
<= 64; bits
++) {
184 str_cpy(error_msg
, MAX_ERROR_STR_LENGTH
, "Workload too small even for 1 << 63");
187 workload_size
= ((uint64_t) 1) << bits
;
190 bench_run_init(&run
, error_msg
, MAX_ERROR_STR_LENGTH
);
192 bool ok
= bench
->entry(&run
, workload_size
);
196 short_report(&run
, -1, bench
, workload_size
);
198 nsec_t duration
= stopwatch_get_nanos(&run
.stopwatch
);
199 if (duration
> SEC2NSEC(MIN_DURATION_SECS
)) {
204 printf("Workload size set to %" PRIu64
", measuring %d samples.\n", workload_size
, NUM_SAMPLES
);
206 bench_run_t
*runs
= calloc(NUM_SAMPLES
, sizeof(bench_run_t
));
208 snprintf(error_msg
, MAX_ERROR_STR_LENGTH
, "failed allocating memory");
211 for (int i
= 0; i
< NUM_SAMPLES
; i
++) {
212 bench_run_init(&runs
[i
], error_msg
, MAX_ERROR_STR_LENGTH
);
214 bool ok
= bench
->entry(&runs
[i
], workload_size
);
219 short_report(&runs
[i
], i
, bench
, workload_size
);
222 summary_stats(runs
, NUM_SAMPLES
, bench
, workload_size
);
223 printf("\nBenchmark completed\n");
230 printf("Error: %s\n", error_msg
);
234 if (bench
->teardown
!= NULL
) {
235 bool ok
= bench
->teardown(&helper_run
);
237 printf("Error: %s\n", error_msg
);
247 static int run_benchmarks(void)
249 unsigned int count_ok
= 0;
250 unsigned int count_fail
= 0;
252 char *failed_names
= NULL
;
254 printf("\n*** Running all benchmarks ***\n\n");
256 for (size_t it
= 0; it
< benchmark_count
; it
++) {
257 printf("%s (%s)\n", benchmarks
[it
]->name
, benchmarks
[it
]->desc
);
258 if (run_benchmark(benchmarks
[it
])) {
264 failed_names
= str_dup(benchmarks
[it
]->name
);
267 asprintf(&f
, "%s, %s", failed_names
, benchmarks
[it
]->name
);
269 printf("Out of memory.\n");
278 printf("\nCompleted, %u benchmarks run, %u succeeded.\n",
279 count_ok
+ count_fail
, count_ok
);
281 printf("Failed benchmarks: %s\n", failed_names
);
286 static void list_benchmarks(void)
289 for (size_t i
= 0; i
< benchmark_count
; i
++) {
290 size_t len_now
= str_length(benchmarks
[i
]->name
);
295 assert(can_cast_size_t_to_int(len
) && "benchmark name length overflow");
297 for (size_t i
= 0; i
< benchmark_count
; i
++)
298 printf(" %-*s %s\n", (int) len
, benchmarks
[i
]->name
, benchmarks
[i
]->desc
);
300 printf(" %-*s Run all benchmarks\n", (int) len
, "*");
303 static void print_usage(const char *progname
)
305 printf("Usage: %s [options] <benchmark>\n", progname
);
307 "Print this help and exit\n");
308 printf("-o, --output filename.csv "
309 "Store machine-readable data in filename.csv\n");
310 printf("-p, --param KEY=VALUE "
311 "Additional parameters for the benchmark\n");
312 printf("<benchmark> is one of the following:\n");
316 static void handle_param_arg(char *arg
)
319 char *key
= str_tok(arg
, "=", &value
);
320 bench_param_set(key
, value
);
323 int main(int argc
, char *argv
[])
325 errno_t rc
= bench_param_init();
327 fprintf(stderr
, "Failed to initialize internal params structure: %s\n",
332 const char *short_options
= "ho:p:";
333 struct option long_options
[] = {
334 { "help", optional_argument
, NULL
, 'h' },
335 { "param", required_argument
, NULL
, 'p' },
336 { "output", required_argument
, NULL
, 'o' },
340 char *csv_output_filename
= NULL
;
343 while ((opt
= getopt_long(argc
, argv
, short_options
, long_options
, NULL
)) > 0) {
349 csv_output_filename
= optarg
;
352 handle_param_arg(optarg
);
360 if (optind
+ 1 != argc
) {
362 fprintf(stderr
, "Error: specify one benchmark to run or * for all.\n");
366 const char *benchmark
= argv
[optind
];
368 if (csv_output_filename
!= NULL
) {
369 errno_t rc
= csv_report_open(csv_output_filename
);
371 fprintf(stderr
, "Failed to open CSV report '%s': %s\n",
372 csv_output_filename
, str_error(rc
));
379 if (str_cmp(benchmark
, "*") == 0) {
380 exit_code
= run_benchmarks();
382 bool benchmark_exists
= false;
383 for (size_t i
= 0; i
< benchmark_count
; i
++) {
384 if (str_cmp(benchmark
, benchmarks
[i
]->name
) == 0) {
385 benchmark_exists
= true;
386 exit_code
= run_benchmark(benchmarks
[i
]) ? 0 : -1;
390 if (!benchmark_exists
) {
391 printf("Unknown benchmark \"%s\"\n", benchmark
);
397 bench_param_cleanup();