block/dirty-bitmap.c

   1 /*
   2  * Block Dirty Bitmap
   3  *
   4  * Copyright (c) 2016-2017 Red Hat. Inc
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include "qemu/osdep.h"
  25 #include "qapi/error.h"
  26 #include "qemu-common.h"
  27 #include "trace.h"
  28 #include "block/block_int.h"
  29 #include "block/blockjob.h"
  30
  31 /**
  32  * A BdrvDirtyBitmap can be in three possible states:
  33  * (1) successor is NULL and disabled is false: full r/w mode
  34  * (2) successor is NULL and disabled is true: read only mode ("disabled")
  35  * (3) successor is set: frozen mode.
  36  *     A frozen bitmap cannot be renamed, deleted, anonymized, cleared, set,
  37  *     or enabled. A frozen bitmap can only abdicate() or reclaim().
  38  */
  39 struct BdrvDirtyBitmap {
  40     QemuMutex *mutex;
  41     HBitmap *bitmap;            /* Dirty bitmap implementation */
  42     HBitmap *meta;              /* Meta dirty bitmap */
  43     bool qmp_locked;            /* Bitmap is locked, it can't be modified
  44                                    through QMP */
  45     BdrvDirtyBitmap *successor; /* Anonymous child; implies frozen status */
  46     char *name;                 /* Optional non-empty unique ID */
  47     int64_t size;               /* Size of the bitmap, in bytes */
  48     bool disabled;              /* Bitmap is disabled. It ignores all writes to
  49                                    the device */
  50     int active_iterators;       /* How many iterators are active */
  51     bool readonly;              /* Bitmap is read-only. This field also
  52                                    prevents the respective image from being
  53                                    modified (i.e. blocks writes and discards).
  54                                    Such operations must fail and both the image
  55                                    and this bitmap must remain unchanged while
  56                                    this flag is set. */
  57     bool persistent;            /* bitmap must be saved to owner disk image */
  58     QLIST_ENTRY(BdrvDirtyBitmap) list;
  59 };
  60
  61 struct BdrvDirtyBitmapIter {
  62     HBitmapIter hbi;
  63     BdrvDirtyBitmap *bitmap;
  64 };
  65
  66 static inline void bdrv_dirty_bitmaps_lock(BlockDriverState *bs)
  67 {
  68     qemu_mutex_lock(&bs->dirty_bitmap_mutex);
  69 }
  70
  71 static inline void bdrv_dirty_bitmaps_unlock(BlockDriverState *bs)
  72 {
  73     qemu_mutex_unlock(&bs->dirty_bitmap_mutex);
  74 }
  75
  76 void bdrv_dirty_bitmap_lock(BdrvDirtyBitmap *bitmap)
  77 {
  78     qemu_mutex_lock(bitmap->mutex);
  79 }
  80
  81 void bdrv_dirty_bitmap_unlock(BdrvDirtyBitmap *bitmap)
  82 {
  83     qemu_mutex_unlock(bitmap->mutex);
  84 }
  85
  86 /* Called with BQL or dirty_bitmap lock taken.  */
  87 BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs, const char *name)
  88 {
  89     BdrvDirtyBitmap *bm;
  90
  91     assert(name);
  92     QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
  93         if (bm->name && !strcmp(name, bm->name)) {
  94             return bm;
  95         }
  96     }
  97     return NULL;
  98 }
  99
 100 /* Called with BQL taken.  */
 101 BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
 102                                           uint32_t granularity,
 103                                           const char *name,
 104                                           Error **errp)
 105 {
 106     int64_t bitmap_size;
 107     BdrvDirtyBitmap *bitmap;
 108
 109     assert(is_power_of_2(granularity) && granularity >= BDRV_SECTOR_SIZE);
 110
 111     if (name && bdrv_find_dirty_bitmap(bs, name)) {
 112         error_setg(errp, "Bitmap already exists: %s", name);
 113         return NULL;
 114     }
 115     bitmap_size = bdrv_getlength(bs);
 116     if (bitmap_size < 0) {
 117         error_setg_errno(errp, -bitmap_size, "could not get length of device");
 118         errno = -bitmap_size;
 119         return NULL;
 120     }
 121     bitmap = g_new0(BdrvDirtyBitmap, 1);
 122     bitmap->mutex = &bs->dirty_bitmap_mutex;
 123     bitmap->bitmap = hbitmap_alloc(bitmap_size, ctz32(granularity));
 124     bitmap->size = bitmap_size;
 125     bitmap->name = g_strdup(name);
 126     bitmap->disabled = false;
 127     bdrv_dirty_bitmaps_lock(bs);
 128     QLIST_INSERT_HEAD(&bs->dirty_bitmaps, bitmap, list);
 129     bdrv_dirty_bitmaps_unlock(bs);
 130     return bitmap;
 131 }
 132
 133 /* bdrv_create_meta_dirty_bitmap
 134  *
 135  * Create a meta dirty bitmap that tracks the changes of bits in @bitmap. I.e.
 136  * when a dirty status bit in @bitmap is changed (either from reset to set or
 137  * the other way around), its respective meta dirty bitmap bit will be marked
 138  * dirty as well.
 139  *
 140  * @bitmap: the block dirty bitmap for which to create a meta dirty bitmap.
 141  * @chunk_size: how many bytes of bitmap data does each bit in the meta bitmap
 142  * track.
 143  */
 144 void bdrv_create_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap,
 145                                    int chunk_size)
 146 {
 147     assert(!bitmap->meta);
 148     qemu_mutex_lock(bitmap->mutex);
 149     bitmap->meta = hbitmap_create_meta(bitmap->bitmap,
 150                                        chunk_size * BITS_PER_BYTE);
 151     qemu_mutex_unlock(bitmap->mutex);
 152 }
 153
 154 void bdrv_release_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 155 {
 156     assert(bitmap->meta);
 157     qemu_mutex_lock(bitmap->mutex);
 158     hbitmap_free_meta(bitmap->bitmap);
 159     bitmap->meta = NULL;
 160     qemu_mutex_unlock(bitmap->mutex);
 161 }
 162
 163 int64_t bdrv_dirty_bitmap_size(const BdrvDirtyBitmap *bitmap)
 164 {
 165     return bitmap->size;
 166 }
 167
 168 const char *bdrv_dirty_bitmap_name(const BdrvDirtyBitmap *bitmap)
 169 {
 170     return bitmap->name;
 171 }
 172
 173 /* Called with BQL taken.  */
 174 bool bdrv_dirty_bitmap_frozen(BdrvDirtyBitmap *bitmap)
 175 {
 176     return bitmap->successor;
 177 }
 178
 179 /* Both conditions disallow user-modification via QMP. */
 180 bool bdrv_dirty_bitmap_user_locked(BdrvDirtyBitmap *bitmap) {
 181     return bdrv_dirty_bitmap_frozen(bitmap) ||
 182            bdrv_dirty_bitmap_qmp_locked(bitmap);
 183 }
 184
 185 void bdrv_dirty_bitmap_set_qmp_locked(BdrvDirtyBitmap *bitmap, bool qmp_locked)
 186 {
 187     qemu_mutex_lock(bitmap->mutex);
 188     bitmap->qmp_locked = qmp_locked;
 189     qemu_mutex_unlock(bitmap->mutex);
 190 }
 191
 192 bool bdrv_dirty_bitmap_qmp_locked(BdrvDirtyBitmap *bitmap)
 193 {
 194     return bitmap->qmp_locked;
 195 }
 196
 197 /* Called with BQL taken.  */
 198 bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap)
 199 {
 200     return !(bitmap->disabled || bitmap->successor);
 201 }
 202
 203 /* Called with BQL taken.  */
 204 DirtyBitmapStatus bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap)
 205 {
 206     if (bdrv_dirty_bitmap_frozen(bitmap)) {
 207         return DIRTY_BITMAP_STATUS_FROZEN;
 208     } else if (bdrv_dirty_bitmap_qmp_locked(bitmap)) {
 209         return DIRTY_BITMAP_STATUS_LOCKED;
 210     } else if (!bdrv_dirty_bitmap_enabled(bitmap)) {
 211         return DIRTY_BITMAP_STATUS_DISABLED;
 212     } else {
 213         return DIRTY_BITMAP_STATUS_ACTIVE;
 214     }
 215 }
 216
 217 /**
 218  * Create a successor bitmap destined to replace this bitmap after an operation.
 219  * Requires that the bitmap is not frozen and has no successor.
 220  * Called with BQL taken.
 221  */
 222 int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
 223                                        BdrvDirtyBitmap *bitmap, Error **errp)
 224 {
 225     uint64_t granularity;
 226     BdrvDirtyBitmap *child;
 227
 228     if (bdrv_dirty_bitmap_frozen(bitmap)) {
 229         error_setg(errp, "Cannot create a successor for a bitmap that is "
 230                    "currently frozen");
 231         return -1;
 232     }
 233     assert(!bitmap->successor);
 234
 235     /* Create an anonymous successor */
 236     granularity = bdrv_dirty_bitmap_granularity(bitmap);
 237     child = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
 238     if (!child) {
 239         return -1;
 240     }
 241
 242     /* Successor will be on or off based on our current state. */
 243     child->disabled = bitmap->disabled;
 244
 245     /* Install the successor and freeze the parent */
 246     bitmap->successor = child;
 247     return 0;
 248 }
 249
 250 void bdrv_enable_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)
 251 {
 252     assert(!bdrv_dirty_bitmap_frozen(bitmap));
 253     bitmap->disabled = false;
 254 }
 255
 256 /* Called with BQL taken. */
 257 void bdrv_dirty_bitmap_enable_successor(BdrvDirtyBitmap *bitmap)
 258 {
 259     assert(bitmap->mutex == bitmap->successor->mutex);
 260     qemu_mutex_lock(bitmap->mutex);
 261     bdrv_enable_dirty_bitmap_locked(bitmap->successor);
 262     qemu_mutex_unlock(bitmap->mutex);
 263 }
 264
 265 /* Called within bdrv_dirty_bitmap_lock..unlock and with BQL taken.  */
 266 static void bdrv_release_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)
 267 {
 268     assert(!bitmap->active_iterators);
 269     assert(!bdrv_dirty_bitmap_frozen(bitmap));
 270     assert(!bitmap->meta);
 271     QLIST_REMOVE(bitmap, list);
 272     hbitmap_free(bitmap->bitmap);
 273     g_free(bitmap->name);
 274     g_free(bitmap);
 275 }
 276
 277 /**
 278  * For a bitmap with a successor, yield our name to the successor,
 279  * delete the old bitmap, and return a handle to the new bitmap.
 280  * Called with BQL taken.
 281  */
 282 BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
 283                                             BdrvDirtyBitmap *bitmap,
 284                                             Error **errp)
 285 {
 286     char *name;
 287     BdrvDirtyBitmap *successor = bitmap->successor;
 288
 289     if (successor == NULL) {
 290         error_setg(errp, "Cannot relinquish control if "
 291                    "there's no successor present");
 292         return NULL;
 293     }
 294
 295     name = bitmap->name;
 296     bitmap->name = NULL;
 297     successor->name = name;
 298     bitmap->successor = NULL;
 299     successor->persistent = bitmap->persistent;
 300     bitmap->persistent = false;
 301     bdrv_release_dirty_bitmap(bs, bitmap);
 302
 303     return successor;
 304 }
 305
 306 /**
 307  * In cases of failure where we can no longer safely delete the parent,
 308  * we may wish to re-join the parent and child/successor.
 309  * The merged parent will be un-frozen, but not explicitly re-enabled.
 310  * Called within bdrv_dirty_bitmap_lock..unlock and with BQL taken.
 311  */
 312 BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BlockDriverState *bs,
 313                                                   BdrvDirtyBitmap *parent,
 314                                                   Error **errp)
 315 {
 316     BdrvDirtyBitmap *successor = parent->successor;
 317
 318     if (!successor) {
 319         error_setg(errp, "Cannot reclaim a successor when none is present");
 320         return NULL;
 321     }
 322
 323     if (!hbitmap_merge(parent->bitmap, successor->bitmap, parent->bitmap)) {
 324         error_setg(errp, "Merging of parent and successor bitmap failed");
 325         return NULL;
 326     }
 327     bdrv_release_dirty_bitmap_locked(successor);
 328     parent->successor = NULL;
 329
 330     return parent;
 331 }
 332
 333 /* Called with BQL taken. */
 334 BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
 335                                            BdrvDirtyBitmap *parent,
 336                                            Error **errp)
 337 {
 338     BdrvDirtyBitmap *ret;
 339
 340     qemu_mutex_lock(parent->mutex);
 341     ret = bdrv_reclaim_dirty_bitmap_locked(bs, parent, errp);
 342     qemu_mutex_unlock(parent->mutex);
 343
 344     return ret;
 345 }
 346
 347 /**
 348  * Truncates _all_ bitmaps attached to a BDS.
 349  * Called with BQL taken.
 350  */
 351 void bdrv_dirty_bitmap_truncate(BlockDriverState *bs, int64_t bytes)
 352 {
 353     BdrvDirtyBitmap *bitmap;
 354
 355     bdrv_dirty_bitmaps_lock(bs);
 356     QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
 357         assert(!bdrv_dirty_bitmap_frozen(bitmap));
 358         assert(!bitmap->active_iterators);
 359         hbitmap_truncate(bitmap->bitmap, bytes);
 360         bitmap->size = bytes;
 361     }
 362     bdrv_dirty_bitmaps_unlock(bs);
 363 }
 364
 365 /* Called with BQL taken.  */
 366 void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap)
 367 {
 368     bdrv_dirty_bitmaps_lock(bs);
 369     bdrv_release_dirty_bitmap_locked(bitmap);
 370     bdrv_dirty_bitmaps_unlock(bs);
 371 }
 372
 373 /**
 374  * Release all named dirty bitmaps attached to a BDS (for use in bdrv_close()).
 375  * There must not be any frozen bitmaps attached.
 376  * This function does not remove persistent bitmaps from the storage.
 377  * Called with BQL taken.
 378  */
 379 void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs)
 380 {
 381     BdrvDirtyBitmap *bm, *next;
 382
 383     bdrv_dirty_bitmaps_lock(bs);
 384     QLIST_FOREACH_SAFE(bm, &bs->dirty_bitmaps, list, next) {
 385         if (bdrv_dirty_bitmap_name(bm)) {
 386             bdrv_release_dirty_bitmap_locked(bm);
 387         }
 388     }
 389     bdrv_dirty_bitmaps_unlock(bs);
 390 }
 391
 392 /**
 393  * Release all persistent dirty bitmaps attached to a BDS (for use in
 394  * bdrv_inactivate_recurse()).
 395  * There must not be any frozen bitmaps attached.
 396  * This function does not remove persistent bitmaps from the storage.
 397  * Called with BQL taken.
 398  */
 399 void bdrv_release_persistent_dirty_bitmaps(BlockDriverState *bs)
 400 {
 401     BdrvDirtyBitmap *bm, *next;
 402
 403     bdrv_dirty_bitmaps_lock(bs);
 404     QLIST_FOREACH_SAFE(bm, &bs->dirty_bitmaps, list, next) {
 405         if (bdrv_dirty_bitmap_get_persistance(bm)) {
 406             bdrv_release_dirty_bitmap_locked(bm);
 407         }
 408     }
 409     bdrv_dirty_bitmaps_unlock(bs);
 410 }
 411
 412 /**
 413  * Remove persistent dirty bitmap from the storage if it exists.
 414  * Absence of bitmap is not an error, because we have the following scenario:
 415  * BdrvDirtyBitmap can have .persistent = true but not yet saved and have no
 416  * stored version. For such bitmap bdrv_remove_persistent_dirty_bitmap() should
 417  * not fail.
 418  * This function doesn't release corresponding BdrvDirtyBitmap.
 419  */
 420 void bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs,
 421                                          const char *name,
 422                                          Error **errp)
 423 {
 424     if (bs->drv && bs->drv->bdrv_remove_persistent_dirty_bitmap) {
 425         bs->drv->bdrv_remove_persistent_dirty_bitmap(bs, name, errp);
 426     }
 427 }
 428
 429 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 430 {
 431     bdrv_dirty_bitmap_lock(bitmap);
 432     assert(!bdrv_dirty_bitmap_frozen(bitmap));
 433     bitmap->disabled = true;
 434     bdrv_dirty_bitmap_unlock(bitmap);
 435 }
 436
 437 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 438 {
 439     bdrv_dirty_bitmap_lock(bitmap);
 440     bdrv_enable_dirty_bitmap_locked(bitmap);
 441     bdrv_dirty_bitmap_unlock(bitmap);
 442 }
 443
 444 BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
 445 {
 446     BdrvDirtyBitmap *bm;
 447     BlockDirtyInfoList *list = NULL;
 448     BlockDirtyInfoList **plist = &list;
 449
 450     bdrv_dirty_bitmaps_lock(bs);
 451     QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
 452         BlockDirtyInfo *info = g_new0(BlockDirtyInfo, 1);
 453         BlockDirtyInfoList *entry = g_new0(BlockDirtyInfoList, 1);
 454         info->count = bdrv_get_dirty_count(bm);
 455         info->granularity = bdrv_dirty_bitmap_granularity(bm);
 456         info->has_name = !!bm->name;
 457         info->name = g_strdup(bm->name);
 458         info->status = bdrv_dirty_bitmap_status(bm);
 459         entry->value = info;
 460         *plist = entry;
 461         plist = &entry->next;
 462     }
 463     bdrv_dirty_bitmaps_unlock(bs);
 464
 465     return list;
 466 }
 467
 468 /* Called within bdrv_dirty_bitmap_lock..unlock */
 469 bool bdrv_get_dirty_locked(BlockDriverState *bs, BdrvDirtyBitmap *bitmap,
 470                            int64_t offset)
 471 {
 472     if (bitmap) {
 473         return hbitmap_get(bitmap->bitmap, offset);
 474     } else {
 475         return false;
 476     }
 477 }
 478
 479 /**
 480  * Chooses a default granularity based on the existing cluster size,
 481  * but clamped between [4K, 64K]. Defaults to 64K in the case that there
 482  * is no cluster size information available.
 483  */
 484 uint32_t bdrv_get_default_bitmap_granularity(BlockDriverState *bs)
 485 {
 486     BlockDriverInfo bdi;
 487     uint32_t granularity;
 488
 489     if (bdrv_get_info(bs, &bdi) >= 0 && bdi.cluster_size > 0) {
 490         granularity = MAX(4096, bdi.cluster_size);
 491         granularity = MIN(65536, granularity);
 492     } else {
 493         granularity = 65536;
 494     }
 495
 496     return granularity;
 497 }
 498
 499 uint32_t bdrv_dirty_bitmap_granularity(const BdrvDirtyBitmap *bitmap)
 500 {
 501     return 1U << hbitmap_granularity(bitmap->bitmap);
 502 }
 503
 504 BdrvDirtyBitmapIter *bdrv_dirty_iter_new(BdrvDirtyBitmap *bitmap)
 505 {
 506     BdrvDirtyBitmapIter *iter = g_new(BdrvDirtyBitmapIter, 1);
 507     hbitmap_iter_init(&iter->hbi, bitmap->bitmap, 0);
 508     iter->bitmap = bitmap;
 509     bitmap->active_iterators++;
 510     return iter;
 511 }
 512
 513 BdrvDirtyBitmapIter *bdrv_dirty_meta_iter_new(BdrvDirtyBitmap *bitmap)
 514 {
 515     BdrvDirtyBitmapIter *iter = g_new(BdrvDirtyBitmapIter, 1);
 516     hbitmap_iter_init(&iter->hbi, bitmap->meta, 0);
 517     iter->bitmap = bitmap;
 518     bitmap->active_iterators++;
 519     return iter;
 520 }
 521
 522 void bdrv_dirty_iter_free(BdrvDirtyBitmapIter *iter)
 523 {
 524     if (!iter) {
 525         return;
 526     }
 527     assert(iter->bitmap->active_iterators > 0);
 528     iter->bitmap->active_iterators--;
 529     g_free(iter);
 530 }
 531
 532 int64_t bdrv_dirty_iter_next(BdrvDirtyBitmapIter *iter)
 533 {
 534     return hbitmap_iter_next(&iter->hbi, true);
 535 }
 536
 537 /**
 538  * Return the next consecutively dirty area in the dirty bitmap
 539  * belonging to the given iterator @iter.
 540  *
 541  * @max_offset: Maximum value that may be returned for
 542  *              *offset + *bytes
 543  * @offset:     Will contain the start offset of the next dirty area
 544  * @bytes:      Will contain the length of the next dirty area
 545  *
 546  * Returns: True if a dirty area could be found before max_offset
 547  *          (which means that *offset and *bytes then contain valid
 548  *          values), false otherwise.
 549  *
 550  * Note that @iter is never advanced if false is returned.  If an area
 551  * is found (which means that true is returned), it will be advanced
 552  * past that area.
 553  */
 554 bool bdrv_dirty_iter_next_area(BdrvDirtyBitmapIter *iter, uint64_t max_offset,
 555                                uint64_t *offset, int *bytes)
 556 {
 557     uint32_t granularity = bdrv_dirty_bitmap_granularity(iter->bitmap);
 558     uint64_t gran_max_offset;
 559     int64_t ret;
 560     int size;
 561
 562     if (max_offset == iter->bitmap->size) {
 563         /* If max_offset points to the image end, round it up by the
 564          * bitmap granularity */
 565         gran_max_offset = ROUND_UP(max_offset, granularity);
 566     } else {
 567         gran_max_offset = max_offset;
 568     }
 569
 570     ret = hbitmap_iter_next(&iter->hbi, false);
 571     if (ret < 0 || ret + granularity > gran_max_offset) {
 572         return false;
 573     }
 574
 575     *offset = ret;
 576     size = 0;
 577
 578     assert(granularity <= INT_MAX);
 579
 580     do {
 581         /* Advance iterator */
 582         ret = hbitmap_iter_next(&iter->hbi, true);
 583         size += granularity;
 584     } while (ret + granularity <= gran_max_offset &&
 585              hbitmap_iter_next(&iter->hbi, false) == ret + granularity &&
 586              size <= INT_MAX - granularity);
 587
 588     *bytes = MIN(size, max_offset - *offset);
 589     return true;
 590 }
 591
 592 /* Called within bdrv_dirty_bitmap_lock..unlock */
 593 void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
 594                                   int64_t offset, int64_t bytes)
 595 {
 596     assert(bdrv_dirty_bitmap_enabled(bitmap));
 597     assert(!bdrv_dirty_bitmap_readonly(bitmap));
 598     hbitmap_set(bitmap->bitmap, offset, bytes);
 599 }
 600
 601 void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
 602                            int64_t offset, int64_t bytes)
 603 {
 604     bdrv_dirty_bitmap_lock(bitmap);
 605     bdrv_set_dirty_bitmap_locked(bitmap, offset, bytes);
 606     bdrv_dirty_bitmap_unlock(bitmap);
 607 }
 608
 609 /* Called within bdrv_dirty_bitmap_lock..unlock */
 610 void bdrv_reset_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
 611                                     int64_t offset, int64_t bytes)
 612 {
 613     assert(bdrv_dirty_bitmap_enabled(bitmap));
 614     assert(!bdrv_dirty_bitmap_readonly(bitmap));
 615     hbitmap_reset(bitmap->bitmap, offset, bytes);
 616 }
 617
 618 void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
 619                              int64_t offset, int64_t bytes)
 620 {
 621     bdrv_dirty_bitmap_lock(bitmap);
 622     bdrv_reset_dirty_bitmap_locked(bitmap, offset, bytes);
 623     bdrv_dirty_bitmap_unlock(bitmap);
 624 }
 625
 626 void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out)
 627 {
 628     assert(bdrv_dirty_bitmap_enabled(bitmap));
 629     assert(!bdrv_dirty_bitmap_readonly(bitmap));
 630     bdrv_dirty_bitmap_lock(bitmap);
 631     if (!out) {
 632         hbitmap_reset_all(bitmap->bitmap);
 633     } else {
 634         HBitmap *backup = bitmap->bitmap;
 635         bitmap->bitmap = hbitmap_alloc(bitmap->size,
 636                                        hbitmap_granularity(backup));
 637         *out = backup;
 638     }
 639     bdrv_dirty_bitmap_unlock(bitmap);
 640 }
 641
 642 void bdrv_restore_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *backup)
 643 {
 644     HBitmap *tmp = bitmap->bitmap;
 645     assert(bdrv_dirty_bitmap_enabled(bitmap));
 646     assert(!bdrv_dirty_bitmap_readonly(bitmap));
 647     bitmap->bitmap = backup;
 648     hbitmap_free(tmp);
 649 }
 650
 651 uint64_t bdrv_dirty_bitmap_serialization_size(const BdrvDirtyBitmap *bitmap,
 652                                               uint64_t offset, uint64_t bytes)
 653 {
 654     return hbitmap_serialization_size(bitmap->bitmap, offset, bytes);
 655 }
 656
 657 uint64_t bdrv_dirty_bitmap_serialization_align(const BdrvDirtyBitmap *bitmap)
 658 {
 659     return hbitmap_serialization_align(bitmap->bitmap);
 660 }
 661
 662 void bdrv_dirty_bitmap_serialize_part(const BdrvDirtyBitmap *bitmap,
 663                                       uint8_t *buf, uint64_t offset,
 664                                       uint64_t bytes)
 665 {
 666     hbitmap_serialize_part(bitmap->bitmap, buf, offset, bytes);
 667 }
 668
 669 void bdrv_dirty_bitmap_deserialize_part(BdrvDirtyBitmap *bitmap,
 670                                         uint8_t *buf, uint64_t offset,
 671                                         uint64_t bytes, bool finish)
 672 {
 673     hbitmap_deserialize_part(bitmap->bitmap, buf, offset, bytes, finish);
 674 }
 675
 676 void bdrv_dirty_bitmap_deserialize_zeroes(BdrvDirtyBitmap *bitmap,
 677                                           uint64_t offset, uint64_t bytes,
 678                                           bool finish)
 679 {
 680     hbitmap_deserialize_zeroes(bitmap->bitmap, offset, bytes, finish);
 681 }
 682
 683 void bdrv_dirty_bitmap_deserialize_ones(BdrvDirtyBitmap *bitmap,
 684                                         uint64_t offset, uint64_t bytes,
 685                                         bool finish)
 686 {
 687     hbitmap_deserialize_ones(bitmap->bitmap, offset, bytes, finish);
 688 }
 689
 690 void bdrv_dirty_bitmap_deserialize_finish(BdrvDirtyBitmap *bitmap)
 691 {
 692     hbitmap_deserialize_finish(bitmap->bitmap);
 693 }
 694
 695 void bdrv_set_dirty(BlockDriverState *bs, int64_t offset, int64_t bytes)
 696 {
 697     BdrvDirtyBitmap *bitmap;
 698
 699     if (QLIST_EMPTY(&bs->dirty_bitmaps)) {
 700         return;
 701     }
 702
 703     bdrv_dirty_bitmaps_lock(bs);
 704     QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
 705         if (!bdrv_dirty_bitmap_enabled(bitmap)) {
 706             continue;
 707         }
 708         assert(!bdrv_dirty_bitmap_readonly(bitmap));
 709         hbitmap_set(bitmap->bitmap, offset, bytes);
 710     }
 711     bdrv_dirty_bitmaps_unlock(bs);
 712 }
 713
 714 /**
 715  * Advance a BdrvDirtyBitmapIter to an arbitrary offset.
 716  */
 717 void bdrv_set_dirty_iter(BdrvDirtyBitmapIter *iter, int64_t offset)
 718 {
 719     hbitmap_iter_init(&iter->hbi, iter->hbi.hb, offset);
 720 }
 721
 722 int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap)
 723 {
 724     return hbitmap_count(bitmap->bitmap);
 725 }
 726
 727 int64_t bdrv_get_meta_dirty_count(BdrvDirtyBitmap *bitmap)
 728 {
 729     return hbitmap_count(bitmap->meta);
 730 }
 731
 732 bool bdrv_dirty_bitmap_readonly(const BdrvDirtyBitmap *bitmap)
 733 {
 734     return bitmap->readonly;
 735 }
 736
 737 /* Called with BQL taken. */
 738 void bdrv_dirty_bitmap_set_readonly(BdrvDirtyBitmap *bitmap, bool value)
 739 {
 740     qemu_mutex_lock(bitmap->mutex);
 741     bitmap->readonly = value;
 742     qemu_mutex_unlock(bitmap->mutex);
 743 }
 744
 745 bool bdrv_has_readonly_bitmaps(BlockDriverState *bs)
 746 {
 747     BdrvDirtyBitmap *bm;
 748     QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
 749         if (bm->readonly) {
 750             return true;
 751         }
 752     }
 753
 754     return false;
 755 }
 756
 757 /* Called with BQL taken. */
 758 void bdrv_dirty_bitmap_set_persistance(BdrvDirtyBitmap *bitmap, bool persistent)
 759 {
 760     qemu_mutex_lock(bitmap->mutex);
 761     bitmap->persistent = persistent;
 762     qemu_mutex_unlock(bitmap->mutex);
 763 }
 764
 765 bool bdrv_dirty_bitmap_get_persistance(BdrvDirtyBitmap *bitmap)
 766 {
 767     return bitmap->persistent;
 768 }
 769
 770 bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs)
 771 {
 772     BdrvDirtyBitmap *bm;
 773     QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
 774         if (bm->persistent && !bm->readonly) {
 775             return true;
 776         }
 777     }
 778
 779     return false;
 780 }
 781
 782 BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs,
 783                                         BdrvDirtyBitmap *bitmap)
 784 {
 785     return bitmap == NULL ? QLIST_FIRST(&bs->dirty_bitmaps) :
 786                             QLIST_NEXT(bitmap, list);
 787 }
 788
 789 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp)
 790 {
 791     return hbitmap_sha256(bitmap->bitmap, errp);
 792 }
 793
 794 int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t offset)
 795 {
 796     return hbitmap_next_zero(bitmap->bitmap, offset);
 797 }
 798
 799 void bdrv_merge_dirty_bitmap(BdrvDirtyBitmap *dest, const BdrvDirtyBitmap *src,
 800                              HBitmap **backup, Error **errp)
 801 {
 802     bool ret;
 803
 804     /* only bitmaps from one bds are supported */
 805     assert(dest->mutex == src->mutex);
 806
 807     qemu_mutex_lock(dest->mutex);
 808
 809     if (bdrv_dirty_bitmap_frozen(dest)) {
 810         error_setg(errp, "Bitmap '%s' is frozen and cannot be modified",
 811                    dest->name);
 812         goto out;
 813     }
 814
 815     if (bdrv_dirty_bitmap_readonly(dest)) {
 816         error_setg(errp, "Bitmap '%s' is readonly and cannot be modified",
 817                    dest->name);
 818         goto out;
 819     }
 820
 821     if (!hbitmap_can_merge(dest->bitmap, src->bitmap)) {
 822         error_setg(errp, "Bitmaps are incompatible and can't be merged");
 823         goto out;
 824     }
 825
 826     if (backup) {
 827         *backup = dest->bitmap;
 828         dest->bitmap = hbitmap_alloc(dest->size, hbitmap_granularity(*backup));
 829         ret = hbitmap_merge(*backup, src->bitmap, dest->bitmap);
 830     } else {
 831         ret = hbitmap_merge(dest->bitmap, src->bitmap, dest->bitmap);
 832     }
 833     assert(ret);
 834
 835 out:
 836     qemu_mutex_unlock(dest->mutex);
 837 }