softmmu/dirtylimit.c

   1 /*
   2  * Dirty page rate limit implementation code
   3  *
   4  * Copyright (c) 2022 CHINA TELECOM CO.,LTD.
   5  *
   6  * Authors:
   7  *  Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  10  * See the COPYING file in the top-level directory.
  11  */
  12
  13 #include "qemu/osdep.h"
  14 #include "qemu/main-loop.h"
  15 #include "qapi/qapi-commands-migration.h"
  16 #include "qapi/qmp/qdict.h"
  17 #include "qapi/error.h"
  18 #include "sysemu/dirtyrate.h"
  19 #include "sysemu/dirtylimit.h"
  20 #include "monitor/hmp.h"
  21 #include "monitor/monitor.h"
  22 #include "exec/memory.h"
  23 #include "exec/target_page.h"
  24 #include "hw/boards.h"
  25 #include "sysemu/kvm.h"
  26 #include "trace.h"
  27
  28 /*
  29  * Dirtylimit stop working if dirty page rate error
  30  * value less than DIRTYLIMIT_TOLERANCE_RANGE
  31  */
  32 #define DIRTYLIMIT_TOLERANCE_RANGE  25  /* MB/s */
  33 /*
  34  * Plus or minus vcpu sleep time linearly if dirty
  35  * page rate error value percentage over
  36  * DIRTYLIMIT_LINEAR_ADJUSTMENT_PCT.
  37  * Otherwise, plus or minus a fixed vcpu sleep time.
  38  */
  39 #define DIRTYLIMIT_LINEAR_ADJUSTMENT_PCT     50
  40 /*
  41  * Max vcpu sleep time percentage during a cycle
  42  * composed of dirty ring full and sleep time.
  43  */
  44 #define DIRTYLIMIT_THROTTLE_PCT_MAX 99
  45
  46 struct {
  47     VcpuStat stat;
  48     bool running;
  49     QemuThread thread;
  50 } *vcpu_dirty_rate_stat;
  51
  52 typedef struct VcpuDirtyLimitState {
  53     int cpu_index;
  54     bool enabled;
  55     /*
  56      * Quota dirty page rate, unit is MB/s
  57      * zero if not enabled.
  58      */
  59     uint64_t quota;
  60 } VcpuDirtyLimitState;
  61
  62 struct {
  63     VcpuDirtyLimitState *states;
  64     /* Max cpus number configured by user */
  65     int max_cpus;
  66     /* Number of vcpu under dirtylimit */
  67     int limited_nvcpu;
  68 } *dirtylimit_state;
  69
  70 /* protect dirtylimit_state */
  71 static QemuMutex dirtylimit_mutex;
  72
  73 /* dirtylimit thread quit if dirtylimit_quit is true */
  74 static bool dirtylimit_quit;
  75
  76 static void vcpu_dirty_rate_stat_collect(void)
  77 {
  78     VcpuStat stat;
  79     int i = 0;
  80
  81     /* calculate vcpu dirtyrate */
  82     vcpu_calculate_dirtyrate(DIRTYLIMIT_CALC_TIME_MS,
  83                              &stat,
  84                              GLOBAL_DIRTY_LIMIT,
  85                              false);
  86
  87     for (i = 0; i < stat.nvcpu; i++) {
  88         vcpu_dirty_rate_stat->stat.rates[i].id = i;
  89         vcpu_dirty_rate_stat->stat.rates[i].dirty_rate =
  90             stat.rates[i].dirty_rate;
  91     }
  92
  93     free(stat.rates);
  94 }
  95
  96 static void *vcpu_dirty_rate_stat_thread(void *opaque)
  97 {
  98     rcu_register_thread();
  99
 100     /* start log sync */
 101     global_dirty_log_change(GLOBAL_DIRTY_LIMIT, true);
 102
 103     while (qatomic_read(&vcpu_dirty_rate_stat->running)) {
 104         vcpu_dirty_rate_stat_collect();
 105         if (dirtylimit_in_service()) {
 106             dirtylimit_process();
 107         }
 108     }
 109
 110     /* stop log sync */
 111     global_dirty_log_change(GLOBAL_DIRTY_LIMIT, false);
 112
 113     rcu_unregister_thread();
 114     return NULL;
 115 }
 116
 117 int64_t vcpu_dirty_rate_get(int cpu_index)
 118 {
 119     DirtyRateVcpu *rates = vcpu_dirty_rate_stat->stat.rates;
 120     return qatomic_read_i64(&rates[cpu_index].dirty_rate);
 121 }
 122
 123 void vcpu_dirty_rate_stat_start(void)
 124 {
 125     if (qatomic_read(&vcpu_dirty_rate_stat->running)) {
 126         return;
 127     }
 128
 129     qatomic_set(&vcpu_dirty_rate_stat->running, 1);
 130     qemu_thread_create(&vcpu_dirty_rate_stat->thread,
 131                        "dirtyrate-stat",
 132                        vcpu_dirty_rate_stat_thread,
 133                        NULL,
 134                        QEMU_THREAD_JOINABLE);
 135 }
 136
 137 void vcpu_dirty_rate_stat_stop(void)
 138 {
 139     qatomic_set(&vcpu_dirty_rate_stat->running, 0);
 140     dirtylimit_state_unlock();
 141     qemu_mutex_unlock_iothread();
 142     qemu_thread_join(&vcpu_dirty_rate_stat->thread);
 143     qemu_mutex_lock_iothread();
 144     dirtylimit_state_lock();
 145 }
 146
 147 void vcpu_dirty_rate_stat_initialize(void)
 148 {
 149     MachineState *ms = MACHINE(qdev_get_machine());
 150     int max_cpus = ms->smp.max_cpus;
 151
 152     vcpu_dirty_rate_stat =
 153         g_malloc0(sizeof(*vcpu_dirty_rate_stat));
 154
 155     vcpu_dirty_rate_stat->stat.nvcpu = max_cpus;
 156     vcpu_dirty_rate_stat->stat.rates =
 157         g_new0(DirtyRateVcpu, max_cpus);
 158
 159     vcpu_dirty_rate_stat->running = false;
 160 }
 161
 162 void vcpu_dirty_rate_stat_finalize(void)
 163 {
 164     free(vcpu_dirty_rate_stat->stat.rates);
 165     vcpu_dirty_rate_stat->stat.rates = NULL;
 166
 167     free(vcpu_dirty_rate_stat);
 168     vcpu_dirty_rate_stat = NULL;
 169 }
 170
 171 void dirtylimit_state_lock(void)
 172 {
 173     qemu_mutex_lock(&dirtylimit_mutex);
 174 }
 175
 176 void dirtylimit_state_unlock(void)
 177 {
 178     qemu_mutex_unlock(&dirtylimit_mutex);
 179 }
 180
 181 static void
 182 __attribute__((__constructor__)) dirtylimit_mutex_init(void)
 183 {
 184     qemu_mutex_init(&dirtylimit_mutex);
 185 }
 186
 187 static inline VcpuDirtyLimitState *dirtylimit_vcpu_get_state(int cpu_index)
 188 {
 189     return &dirtylimit_state->states[cpu_index];
 190 }
 191
 192 void dirtylimit_state_initialize(void)
 193 {
 194     MachineState *ms = MACHINE(qdev_get_machine());
 195     int max_cpus = ms->smp.max_cpus;
 196     int i;
 197
 198     dirtylimit_state = g_malloc0(sizeof(*dirtylimit_state));
 199
 200     dirtylimit_state->states =
 201             g_new0(VcpuDirtyLimitState, max_cpus);
 202
 203     for (i = 0; i < max_cpus; i++) {
 204         dirtylimit_state->states[i].cpu_index = i;
 205     }
 206
 207     dirtylimit_state->max_cpus = max_cpus;
 208     trace_dirtylimit_state_initialize(max_cpus);
 209 }
 210
 211 void dirtylimit_state_finalize(void)
 212 {
 213     free(dirtylimit_state->states);
 214     dirtylimit_state->states = NULL;
 215
 216     free(dirtylimit_state);
 217     dirtylimit_state = NULL;
 218
 219     trace_dirtylimit_state_finalize();
 220 }
 221
 222 bool dirtylimit_in_service(void)
 223 {
 224     return !!dirtylimit_state;
 225 }
 226
 227 bool dirtylimit_vcpu_index_valid(int cpu_index)
 228 {
 229     MachineState *ms = MACHINE(qdev_get_machine());
 230
 231     return !(cpu_index < 0 ||
 232              cpu_index >= ms->smp.max_cpus);
 233 }
 234
 235 static uint64_t dirtylimit_dirty_ring_full_time(uint64_t dirtyrate)
 236 {
 237     static uint64_t max_dirtyrate;
 238     unsigned target_page_bits = qemu_target_page_bits();
 239     uint64_t dirty_ring_size_MB;
 240
 241     /* So far, the largest (non-huge) page size is 64k, i.e. 16 bits. */
 242     assert(target_page_bits < 20);
 243
 244     /* Convert ring size (pages) to MiB (2**20). */
 245     dirty_ring_size_MB = kvm_dirty_ring_size() >> (20 - target_page_bits);
 246
 247     if (max_dirtyrate < dirtyrate) {
 248         max_dirtyrate = dirtyrate;
 249     }
 250
 251     return dirty_ring_size_MB * 1000000 / max_dirtyrate;
 252 }
 253
 254 static inline bool dirtylimit_done(uint64_t quota,
 255                                    uint64_t current)
 256 {
 257     uint64_t min, max;
 258
 259     min = MIN(quota, current);
 260     max = MAX(quota, current);
 261
 262     return ((max - min) <= DIRTYLIMIT_TOLERANCE_RANGE) ? true : false;
 263 }
 264
 265 static inline bool
 266 dirtylimit_need_linear_adjustment(uint64_t quota,
 267                                   uint64_t current)
 268 {
 269     uint64_t min, max;
 270
 271     min = MIN(quota, current);
 272     max = MAX(quota, current);
 273
 274     return ((max - min) * 100 / max) > DIRTYLIMIT_LINEAR_ADJUSTMENT_PCT;
 275 }
 276
 277 static void dirtylimit_set_throttle(CPUState *cpu,
 278                                     uint64_t quota,
 279                                     uint64_t current)
 280 {
 281     int64_t ring_full_time_us = 0;
 282     uint64_t sleep_pct = 0;
 283     uint64_t throttle_us = 0;
 284
 285     if (current == 0) {
 286         cpu->throttle_us_per_full = 0;
 287         return;
 288     }
 289
 290     ring_full_time_us = dirtylimit_dirty_ring_full_time(current);
 291
 292     if (dirtylimit_need_linear_adjustment(quota, current)) {
 293         if (quota < current) {
 294             sleep_pct = (current - quota) * 100 / current;
 295             throttle_us =
 296                 ring_full_time_us * sleep_pct / (double)(100 - sleep_pct);
 297             cpu->throttle_us_per_full += throttle_us;
 298         } else {
 299             sleep_pct = (quota - current) * 100 / quota;
 300             throttle_us =
 301                 ring_full_time_us * sleep_pct / (double)(100 - sleep_pct);
 302             cpu->throttle_us_per_full -= throttle_us;
 303         }
 304
 305         trace_dirtylimit_throttle_pct(cpu->cpu_index,
 306                                       sleep_pct,
 307                                       throttle_us);
 308     } else {
 309         if (quota < current) {
 310             cpu->throttle_us_per_full += ring_full_time_us / 10;
 311         } else {
 312             cpu->throttle_us_per_full -= ring_full_time_us / 10;
 313         }
 314     }
 315
 316     /*
 317      * TODO: in the big kvm_dirty_ring_size case (eg: 65536, or other scenario),
 318      *       current dirty page rate may never reach the quota, we should stop
 319      *       increasing sleep time?
 320      */
 321     cpu->throttle_us_per_full = MIN(cpu->throttle_us_per_full,
 322         ring_full_time_us * DIRTYLIMIT_THROTTLE_PCT_MAX);
 323
 324     cpu->throttle_us_per_full = MAX(cpu->throttle_us_per_full, 0);
 325 }
 326
 327 static void dirtylimit_adjust_throttle(CPUState *cpu)
 328 {
 329     uint64_t quota = 0;
 330     uint64_t current = 0;
 331     int cpu_index = cpu->cpu_index;
 332
 333     quota = dirtylimit_vcpu_get_state(cpu_index)->quota;
 334     current = vcpu_dirty_rate_get(cpu_index);
 335
 336     if (!dirtylimit_done(quota, current)) {
 337         dirtylimit_set_throttle(cpu, quota, current);
 338     }
 339
 340     return;
 341 }
 342
 343 void dirtylimit_process(void)
 344 {
 345     CPUState *cpu;
 346
 347     if (!qatomic_read(&dirtylimit_quit)) {
 348         dirtylimit_state_lock();
 349
 350         if (!dirtylimit_in_service()) {
 351             dirtylimit_state_unlock();
 352             return;
 353         }
 354
 355         CPU_FOREACH(cpu) {
 356             if (!dirtylimit_vcpu_get_state(cpu->cpu_index)->enabled) {
 357                 continue;
 358             }
 359             dirtylimit_adjust_throttle(cpu);
 360         }
 361         dirtylimit_state_unlock();
 362     }
 363 }
 364
 365 void dirtylimit_change(bool start)
 366 {
 367     if (start) {
 368         qatomic_set(&dirtylimit_quit, 0);
 369     } else {
 370         qatomic_set(&dirtylimit_quit, 1);
 371     }
 372 }
 373
 374 void dirtylimit_set_vcpu(int cpu_index,
 375                          uint64_t quota,
 376                          bool enable)
 377 {
 378     trace_dirtylimit_set_vcpu(cpu_index, quota);
 379
 380     if (enable) {
 381         dirtylimit_state->states[cpu_index].quota = quota;
 382         if (!dirtylimit_vcpu_get_state(cpu_index)->enabled) {
 383             dirtylimit_state->limited_nvcpu++;
 384         }
 385     } else {
 386         dirtylimit_state->states[cpu_index].quota = 0;
 387         if (dirtylimit_state->states[cpu_index].enabled) {
 388             dirtylimit_state->limited_nvcpu--;
 389         }
 390     }
 391
 392     dirtylimit_state->states[cpu_index].enabled = enable;
 393 }
 394
 395 void dirtylimit_set_all(uint64_t quota,
 396                         bool enable)
 397 {
 398     MachineState *ms = MACHINE(qdev_get_machine());
 399     int max_cpus = ms->smp.max_cpus;
 400     int i;
 401
 402     for (i = 0; i < max_cpus; i++) {
 403         dirtylimit_set_vcpu(i, quota, enable);
 404     }
 405 }
 406
 407 void dirtylimit_vcpu_execute(CPUState *cpu)
 408 {
 409     if (dirtylimit_in_service() &&
 410         dirtylimit_vcpu_get_state(cpu->cpu_index)->enabled &&
 411         cpu->throttle_us_per_full) {
 412         trace_dirtylimit_vcpu_execute(cpu->cpu_index,
 413                 cpu->throttle_us_per_full);
 414         usleep(cpu->throttle_us_per_full);
 415     }
 416 }
 417
 418 static void dirtylimit_init(void)
 419 {
 420     dirtylimit_state_initialize();
 421     dirtylimit_change(true);
 422     vcpu_dirty_rate_stat_initialize();
 423     vcpu_dirty_rate_stat_start();
 424 }
 425
 426 static void dirtylimit_cleanup(void)
 427 {
 428     vcpu_dirty_rate_stat_stop();
 429     vcpu_dirty_rate_stat_finalize();
 430     dirtylimit_change(false);
 431     dirtylimit_state_finalize();
 432 }
 433
 434 void qmp_cancel_vcpu_dirty_limit(bool has_cpu_index,
 435                                  int64_t cpu_index,
 436                                  Error **errp)
 437 {
 438     if (!kvm_enabled() || !kvm_dirty_ring_enabled()) {
 439         return;
 440     }
 441
 442     if (has_cpu_index && !dirtylimit_vcpu_index_valid(cpu_index)) {
 443         error_setg(errp, "incorrect cpu index specified");
 444         return;
 445     }
 446
 447     if (!dirtylimit_in_service()) {
 448         return;
 449     }
 450
 451     dirtylimit_state_lock();
 452
 453     if (has_cpu_index) {
 454         dirtylimit_set_vcpu(cpu_index, 0, false);
 455     } else {
 456         dirtylimit_set_all(0, false);
 457     }
 458
 459     if (!dirtylimit_state->limited_nvcpu) {
 460         dirtylimit_cleanup();
 461     }
 462
 463     dirtylimit_state_unlock();
 464 }
 465
 466 void hmp_cancel_vcpu_dirty_limit(Monitor *mon, const QDict *qdict)
 467 {
 468     int64_t cpu_index = qdict_get_try_int(qdict, "cpu_index", -1);
 469     Error *err = NULL;
 470
 471     qmp_cancel_vcpu_dirty_limit(!!(cpu_index != -1), cpu_index, &err);
 472     if (err) {
 473         hmp_handle_error(mon, err);
 474         return;
 475     }
 476
 477     monitor_printf(mon, "[Please use 'info vcpu_dirty_limit' to query "
 478                    "dirty limit for virtual CPU]\n");
 479 }
 480
 481 void qmp_set_vcpu_dirty_limit(bool has_cpu_index,
 482                               int64_t cpu_index,
 483                               uint64_t dirty_rate,
 484                               Error **errp)
 485 {
 486     if (!kvm_enabled() || !kvm_dirty_ring_enabled()) {
 487         error_setg(errp, "dirty page limit feature requires KVM with"
 488                    " accelerator property 'dirty-ring-size' set'");
 489         return;
 490     }
 491
 492     if (has_cpu_index && !dirtylimit_vcpu_index_valid(cpu_index)) {
 493         error_setg(errp, "incorrect cpu index specified");
 494         return;
 495     }
 496
 497     if (!dirty_rate) {
 498         qmp_cancel_vcpu_dirty_limit(has_cpu_index, cpu_index, errp);
 499         return;
 500     }
 501
 502     dirtylimit_state_lock();
 503
 504     if (!dirtylimit_in_service()) {
 505         dirtylimit_init();
 506     }
 507
 508     if (has_cpu_index) {
 509         dirtylimit_set_vcpu(cpu_index, dirty_rate, true);
 510     } else {
 511         dirtylimit_set_all(dirty_rate, true);
 512     }
 513
 514     dirtylimit_state_unlock();
 515 }
 516
 517 void hmp_set_vcpu_dirty_limit(Monitor *mon, const QDict *qdict)
 518 {
 519     int64_t dirty_rate = qdict_get_int(qdict, "dirty_rate");
 520     int64_t cpu_index = qdict_get_try_int(qdict, "cpu_index", -1);
 521     Error *err = NULL;
 522
 523     qmp_set_vcpu_dirty_limit(!!(cpu_index != -1), cpu_index, dirty_rate, &err);
 524     if (err) {
 525         hmp_handle_error(mon, err);
 526         return;
 527     }
 528
 529     monitor_printf(mon, "[Please use 'info vcpu_dirty_limit' to query "
 530                    "dirty limit for virtual CPU]\n");
 531 }
 532
 533 static struct DirtyLimitInfo *dirtylimit_query_vcpu(int cpu_index)
 534 {
 535     DirtyLimitInfo *info = NULL;
 536
 537     info = g_malloc0(sizeof(*info));
 538     info->cpu_index = cpu_index;
 539     info->limit_rate = dirtylimit_vcpu_get_state(cpu_index)->quota;
 540     info->current_rate = vcpu_dirty_rate_get(cpu_index);
 541
 542     return info;
 543 }
 544
 545 static struct DirtyLimitInfoList *dirtylimit_query_all(void)
 546 {
 547     int i, index;
 548     DirtyLimitInfo *info = NULL;
 549     DirtyLimitInfoList *head = NULL, **tail = &head;
 550
 551     dirtylimit_state_lock();
 552
 553     if (!dirtylimit_in_service()) {
 554         dirtylimit_state_unlock();
 555         return NULL;
 556     }
 557
 558     for (i = 0; i < dirtylimit_state->max_cpus; i++) {
 559         index = dirtylimit_state->states[i].cpu_index;
 560         if (dirtylimit_vcpu_get_state(index)->enabled) {
 561             info = dirtylimit_query_vcpu(index);
 562             QAPI_LIST_APPEND(tail, info);
 563         }
 564     }
 565
 566     dirtylimit_state_unlock();
 567
 568     return head;
 569 }
 570
 571 struct DirtyLimitInfoList *qmp_query_vcpu_dirty_limit(Error **errp)
 572 {
 573     if (!dirtylimit_in_service()) {
 574         return NULL;
 575     }
 576
 577     return dirtylimit_query_all();
 578 }
 579
 580 void hmp_info_vcpu_dirty_limit(Monitor *mon, const QDict *qdict)
 581 {
 582     DirtyLimitInfoList *limit, *head, *info = NULL;
 583     Error *err = NULL;
 584
 585     if (!dirtylimit_in_service()) {
 586         monitor_printf(mon, "Dirty page limit not enabled!\n");
 587         return;
 588     }
 589
 590     info = qmp_query_vcpu_dirty_limit(&err);
 591     if (err) {
 592         hmp_handle_error(mon, err);
 593         return;
 594     }
 595
 596     head = info;
 597     for (limit = head; limit != NULL; limit = limit->next) {
 598         monitor_printf(mon, "vcpu[%"PRIi64"], limit rate %"PRIi64 " (MB/s),"
 599                             " current rate %"PRIi64 " (MB/s)\n",
 600                             limit->value->cpu_index,
 601                             limit->value->limit_rate,
 602                             limit->value->current_rate);
 603     }
 604
 605     g_free(info);
 606 }