From 3bd747587973aa824a1405a1ff9753641ff0fedc Mon Sep 17 00:00:00 2001 From: Vojtech Horky Date: Fri, 28 Dec 2018 10:32:11 +0100 Subject: [PATCH] perf: huge refactoring The overall aim of this refactoring was to remove duplicate code in all benchmarks that was responsible (a) for computing proper workload size and (b) for computing final statistics. After the refactoring, the actual benchmark code is quite short and takes care of the actual work only. The harness code was factored out into perf.c that is now responsible for computing the workload size and then runs the actual benchmark. As an extra feature, we pass stopwatch_t into the benchmark code that is only responsible for starting/stopping. Duration is then queried outside the benchmark code. --- uspace/app/perf/{perf.h => ipc/ns_ping.c} | 57 ++++++------ uspace/app/perf/ipc/ns_ping.def | 3 +- uspace/app/perf/{perf.h => ipc/ping_pong.c} | 64 ++++++++----- uspace/app/perf/ipc/ping_pong.def | 3 +- uspace/app/perf/{perf.h => malloc/malloc1.c} | 53 +++++------ uspace/app/perf/malloc/malloc1.def | 3 +- uspace/app/perf/{perf.h => malloc/malloc2.c} | 56 +++++++----- uspace/app/perf/malloc/malloc2.def | 3 +- uspace/app/perf/perf.c | 130 +++++++++++++++++++++++++-- uspace/app/perf/perf.h | 17 ++-- 10 files changed, 266 insertions(+), 123 deletions(-) copy uspace/app/perf/{perf.h => ipc/ns_ping.c} (74%) copy uspace/app/perf/{perf.h => ipc/ping_pong.c} (59%) copy uspace/app/perf/{perf.h => malloc/malloc1.c} (75%) copy uspace/app/perf/{perf.h => malloc/malloc2.c} (63%) diff --git a/uspace/app/perf/perf.h b/uspace/app/perf/ipc/ns_ping.c similarity index 74% copy from uspace/app/perf/perf.h copy to uspace/app/perf/ipc/ns_ping.c index 3a4d940a8..a9aa525ac 100644 --- a/uspace/app/perf/perf.h +++ b/uspace/app/perf/ipc/ns_ping.c @@ -26,33 +26,30 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/** @addtogroup perf - * @{ - */ -/** @file - */ - -#ifndef PERF_H_ -#define PERF_H_ - -#include - -typedef const char *(*benchmark_entry_t)(void); - -typedef struct { - const char *name; - const char *desc; - benchmark_entry_t entry; -} benchmark_t; - -extern const char *bench_malloc1(void); -extern const char *bench_malloc2(void); -extern const char *bench_ns_ping(void); -extern const char *bench_ping_pong(void); - -extern benchmark_t benchmarks[]; - -#endif - -/** @} - */ +#include +#include +#include +#include +#include +#include "../perf.h" + +bool bench_ns_ping(stopwatch_t *stopwatch, uint64_t niter, + char *error, size_t error_size) +{ + stopwatch_start(stopwatch); + + for (uint64_t count = 0; count < niter; count++) { + errno_t rc = ns_ping(); + + if (rc != EOK) { + snprintf(error, error_size, + "failed sending ping message: %s (%d)", + str_error(rc), rc); + return false; + } + } + + stopwatch_stop(stopwatch); + + return true; +} diff --git a/uspace/app/perf/ipc/ns_ping.def b/uspace/app/perf/ipc/ns_ping.def index 71e048e6e..e9e76b84e 100644 --- a/uspace/app/perf/ipc/ns_ping.def +++ b/uspace/app/perf/ipc/ns_ping.def @@ -1,5 +1,6 @@ { "ns_ping", "Name service IPC ping-pong benchmark", - &bench_ns_ping + &bench_ns_ping, + NULL, NULL }, diff --git a/uspace/app/perf/perf.h b/uspace/app/perf/ipc/ping_pong.c similarity index 59% copy from uspace/app/perf/perf.h copy to uspace/app/perf/ipc/ping_pong.c index 3a4d940a8..1653f875d 100644 --- a/uspace/app/perf/perf.h +++ b/uspace/app/perf/ipc/ping_pong.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Jiri Svoboda + * Copyright (c) 2009 Jiri Svoboda * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,33 +26,51 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/** @addtogroup perf - * @{ - */ -/** @file - */ +#include +#include +#include +#include +#include +#include "../perf.h" -#ifndef PERF_H_ -#define PERF_H_ +static ipc_test_t *test = NULL; -#include +bool bench_ping_pong_setup(char *error, size_t error_size) +{ + errno_t rc = ipc_test_create(&test); + if (rc != EOK) { + snprintf(error, error_size, + "failed contacting IPC test server: %s (%d)", + str_error(rc), rc); + return false; + } -typedef const char *(*benchmark_entry_t)(void); + return true; +} -typedef struct { - const char *name; - const char *desc; - benchmark_entry_t entry; -} benchmark_t; +bool bench_ping_pong_teardown(char *error, size_t error_size) +{ + ipc_test_destroy(test); + return true; +} -extern const char *bench_malloc1(void); -extern const char *bench_malloc2(void); -extern const char *bench_ns_ping(void); -extern const char *bench_ping_pong(void); +bool bench_ping_pong(stopwatch_t *stopwatch, uint64_t niter, + char *error, size_t error_size) +{ + stopwatch_start(stopwatch); -extern benchmark_t benchmarks[]; + for (uint64_t count = 0; count < niter; count++) { + errno_t rc = ipc_test_ping(test); -#endif + if (rc != EOK) { + snprintf(error, error_size, + "failed sending ping message: %s (%d)", + str_error(rc), rc); + return false; + } + } -/** @} - */ + stopwatch_stop(stopwatch); + + return true; +} diff --git a/uspace/app/perf/ipc/ping_pong.def b/uspace/app/perf/ipc/ping_pong.def index 19bf35607..c5ebb95fa 100644 --- a/uspace/app/perf/ipc/ping_pong.def +++ b/uspace/app/perf/ipc/ping_pong.def @@ -1,5 +1,6 @@ { "ping_pong", "IPC ping-pong benchmark", - &bench_ping_pong + &bench_ping_pong, + &bench_ping_pong_setup, &bench_ping_pong_teardown }, diff --git a/uspace/app/perf/perf.h b/uspace/app/perf/malloc/malloc1.c similarity index 75% copy from uspace/app/perf/perf.h copy to uspace/app/perf/malloc/malloc1.c index 3a4d940a8..1f0e8edac 100644 --- a/uspace/app/perf/perf.h +++ b/uspace/app/perf/malloc/malloc1.c @@ -26,33 +26,26 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/** @addtogroup perf - * @{ - */ -/** @file - */ - -#ifndef PERF_H_ -#define PERF_H_ - -#include - -typedef const char *(*benchmark_entry_t)(void); - -typedef struct { - const char *name; - const char *desc; - benchmark_entry_t entry; -} benchmark_t; - -extern const char *bench_malloc1(void); -extern const char *bench_malloc2(void); -extern const char *bench_ns_ping(void); -extern const char *bench_ping_pong(void); - -extern benchmark_t benchmarks[]; - -#endif - -/** @} - */ +#include +#include +#include +#include "../perf.h" + +bool bench_malloc1(stopwatch_t *stopwatch, uint64_t size, + char *error, size_t error_size) +{ + stopwatch_start(stopwatch); + for (uint64_t i = 0; i < size; i++) { + void *p = malloc(1); + if (p == NULL) { + snprintf(error, error_size, + "failed to allocate 1B in run %" PRIu64 " (out of %" PRIu64 ")", + i, size); + return false; + } + free(p); + } + stopwatch_stop(stopwatch); + + return true; +} diff --git a/uspace/app/perf/malloc/malloc1.def b/uspace/app/perf/malloc/malloc1.def index 13c8754c4..d3264e8dc 100644 --- a/uspace/app/perf/malloc/malloc1.def +++ b/uspace/app/perf/malloc/malloc1.def @@ -1,5 +1,6 @@ { "malloc1", "User-space memory allocator benchmark, repeatedly allocate one block", - &bench_malloc1 + &bench_malloc1, + NULL, NULL }, diff --git a/uspace/app/perf/perf.h b/uspace/app/perf/malloc/malloc2.c similarity index 63% copy from uspace/app/perf/perf.h copy to uspace/app/perf/malloc/malloc2.c index 3a4d940a8..483517e96 100644 --- a/uspace/app/perf/perf.h +++ b/uspace/app/perf/malloc/malloc2.c @@ -26,33 +26,43 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/** @addtogroup perf - * @{ - */ -/** @file - */ +#include +#include +#include "../perf.h" -#ifndef PERF_H_ -#define PERF_H_ +bool bench_malloc2(stopwatch_t *stopwatch, uint64_t niter, + char *error, size_t error_size) +{ + stopwatch_start(stopwatch); -#include + void **p = malloc(niter * sizeof(void *)); + if (p == NULL) { + snprintf(error, error_size, + "failed to allocate backend array (%" PRIu64 "B)", + niter * sizeof(void *)); + return false; + } -typedef const char *(*benchmark_entry_t)(void); + for (uint64_t count = 0; count < niter; count++) { + p[count] = malloc(1); + if (p[count] == NULL) { + snprintf(error, error_size, + "failed to allocate 1B in run %" PRIu64 " (out of %" PRIu64 ")", + count, niter); + for (uint64_t j = 0; j < count; j++) { + free(p[j]); + } + free(p); + return false; + } + } -typedef struct { - const char *name; - const char *desc; - benchmark_entry_t entry; -} benchmark_t; + for (uint64_t count = 0; count < niter; count++) + free(p[count]); -extern const char *bench_malloc1(void); -extern const char *bench_malloc2(void); -extern const char *bench_ns_ping(void); -extern const char *bench_ping_pong(void); + free(p); -extern benchmark_t benchmarks[]; + stopwatch_stop(stopwatch); -#endif - -/** @} - */ + return true; +} diff --git a/uspace/app/perf/malloc/malloc2.def b/uspace/app/perf/malloc/malloc2.def index 0b73dc1b8..35182e9e5 100644 --- a/uspace/app/perf/malloc/malloc2.def +++ b/uspace/app/perf/malloc/malloc2.def @@ -1,5 +1,6 @@ { "malloc2", "User-space memory allocator benchmark, allocate many small blocks", - &bench_malloc2 + &bench_malloc2, + NULL, NULL }, diff --git a/uspace/app/perf/perf.c b/uspace/app/perf/perf.c index bcd9de866..cab8789ad 100644 --- a/uspace/app/perf/perf.c +++ b/uspace/app/perf/perf.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2018 Jiri Svoboda + * Copyright (c) 2018 Vojtech Horky * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,28 +38,141 @@ #include #include #include +#include +#include +#include #include "perf.h" +#define MIN_DURATION_SECS 10 +#define NUM_SAMPLES 10 +#define MAX_ERROR_STR_LENGTH 1024 + benchmark_t benchmarks[] = { #include "ipc/ns_ping.def" #include "ipc/ping_pong.def" #include "malloc/malloc1.def" #include "malloc/malloc2.def" - { NULL, NULL, NULL } + { NULL, NULL, NULL, NULL, NULL } }; +static void short_report(stopwatch_t *stopwatch, int run_index, + benchmark_t *bench, size_t workload_size) +{ + usec_t duration_usec = NSEC2USEC(stopwatch_get_nanos(stopwatch)); + + printf("Completed %zu operations in %llu us", + workload_size, duration_usec); + if (duration_usec > 0) { + double cycles = workload_size * 1000 * 1000 / duration_usec; + printf(", %.0f cycles/s.\n", cycles); + } else { + printf(".\n"); + } +} + +static void summary_stats(stopwatch_t *stopwatch, size_t stopwatch_count, + benchmark_t *bench, size_t workload_size) +{ + double sum = 0.0; + double sum_square = 0.0; + + for (size_t i = 0; i < stopwatch_count; i++) { + double nanos = stopwatch_get_nanos(&stopwatch[i]); + double thruput = (double) workload_size / (nanos / 1000000000.0l); + sum += thruput; + sum_square += thruput * thruput; + } + + double avg = sum / stopwatch_count; + + double sd_numer = sum_square + stopwatch_count * avg * avg - 2 * sum * avg; + double sd_square = sd_numer / ((double) stopwatch_count - 1); + + printf("Average: %.0f cycles/s Std.dev^2: %.0f cycles/s Samples: %zu\n", + avg, sd_square, stopwatch_count); +} + static bool run_benchmark(benchmark_t *bench) { - /* Execute the benchmarl */ - const char *ret = bench->entry(); + printf("Warm up and determine workload size...\n"); + + char *error_msg = malloc(MAX_ERROR_STR_LENGTH + 1); + if (error_msg == NULL) { + printf("Out of memory!\n"); + return false; + } + str_cpy(error_msg, MAX_ERROR_STR_LENGTH, ""); + + bool ret = true; - if (ret == NULL) { - printf("\nBenchmark completed\n"); - return true; + if (bench->setup != NULL) { + ret = bench->setup(error_msg, MAX_ERROR_STR_LENGTH); + if (!ret) { + goto leave_error; + } } - printf("\n%s\n", ret); - return false; + size_t workload_size = 1; + + while (true) { + stopwatch_t stopwatch = STOPWATCH_INITIALIZE_STATIC; + + bool ok = bench->entry(&stopwatch, workload_size, + error_msg, MAX_ERROR_STR_LENGTH); + if (!ok) { + goto leave_error; + } + short_report(&stopwatch, -1, bench, workload_size); + + nsec_t duration = stopwatch_get_nanos(&stopwatch); + if (duration > SEC2NSEC(MIN_DURATION_SECS)) { + break; + } + workload_size *= 2; + } + + printf("Workload size set to %zu, measuring %d samples.\n", workload_size, NUM_SAMPLES); + + stopwatch_t *stopwatch = calloc(NUM_SAMPLES, sizeof(stopwatch_t)); + if (stopwatch == NULL) { + snprintf(error_msg, MAX_ERROR_STR_LENGTH, "failed allocating memory"); + goto leave_error; + } + for (int i = 0; i < NUM_SAMPLES; i++) { + stopwatch_init(&stopwatch[i]); + + bool ok = bench->entry(&stopwatch[i], workload_size, + error_msg, MAX_ERROR_STR_LENGTH); + if (!ok) { + free(stopwatch); + goto leave_error; + } + short_report(&stopwatch[i], i, bench, workload_size); + } + + summary_stats(stopwatch, NUM_SAMPLES, bench, workload_size); + printf("\nBenchmark completed\n"); + + free(stopwatch); + + goto leave; + +leave_error: + printf("Error: %s\n", error_msg); + ret = false; + +leave: + if (bench->teardown != NULL) { + bool ok = bench->teardown(error_msg, MAX_ERROR_STR_LENGTH); + if (!ok) { + printf("Error: %s\n", error_msg); + ret = false; + } + } + + free(error_msg); + + return ret; } static int run_benchmarks(void) diff --git a/uspace/app/perf/perf.h b/uspace/app/perf/perf.h index 3a4d940a8..4fb70243f 100644 --- a/uspace/app/perf/perf.h +++ b/uspace/app/perf/perf.h @@ -36,19 +36,26 @@ #define PERF_H_ #include +#include -typedef const char *(*benchmark_entry_t)(void); +typedef bool (*benchmark_entry_t)(stopwatch_t *, uint64_t, + char *, size_t); +typedef bool (*benchmark_helper_t)(char *, size_t); typedef struct { const char *name; const char *desc; benchmark_entry_t entry; + benchmark_helper_t setup; + benchmark_helper_t teardown; } benchmark_t; -extern const char *bench_malloc1(void); -extern const char *bench_malloc2(void); -extern const char *bench_ns_ping(void); -extern const char *bench_ping_pong(void); +extern bool bench_malloc1(stopwatch_t *, uint64_t, char *, size_t); +extern bool bench_malloc2(stopwatch_t *, uint64_t, char *, size_t); +extern bool bench_ns_ping(stopwatch_t *, uint64_t, char *, size_t); +extern bool bench_ping_pong(stopwatch_t *, uint64_t, char *, size_t); +extern bool bench_ping_pong_setup(char *, size_t); +extern bool bench_ping_pong_teardown(char *, size_t); extern benchmark_t benchmarks[]; -- 2.11.4.GIT