tests/qemu-iotests/257

   1 #!/usr/bin/env python3
   2 # group: rw
   3 #
   4 # Test bitmap-sync backups (incremental, differential, and partials)
   5 #
   6 # Copyright (c) 2019 John Snow for Red Hat, Inc.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20 #
  21 # owner=jsnow@redhat.com
  22
  23 import math
  24 import os
  25
  26 import iotests
  27 from iotests import log, qemu_img
  28
  29 SIZE = 64 * 1024 * 1024
  30 GRANULARITY = 64 * 1024
  31
  32
  33 class Pattern:
  34     def __init__(self, byte, offset, size=GRANULARITY):
  35         self.byte = byte
  36         self.offset = offset
  37         self.size = size
  38
  39     def bits(self, granularity):
  40         lower = self.offset // granularity
  41         upper = (self.offset + self.size - 1) // granularity
  42         return set(range(lower, upper + 1))
  43
  44
  45 class PatternGroup:
  46     """Grouping of Pattern objects. Initialize with an iterable of Patterns."""
  47     def __init__(self, patterns):
  48         self.patterns = patterns
  49
  50     def bits(self, granularity):
  51         """Calculate the unique bits dirtied by this pattern grouping"""
  52         res = set()
  53         for pattern in self.patterns:
  54             res |= pattern.bits(granularity)
  55         return res
  56
  57
  58 GROUPS = [
  59     PatternGroup([
  60         # Batch 0: 4 clusters
  61         Pattern('0x49', 0x0000000),
  62         Pattern('0x6c', 0x0100000),   # 1M
  63         Pattern('0x6f', 0x2000000),   # 32M
  64         Pattern('0x76', 0x3ff0000)]), # 64M - 64K
  65     PatternGroup([
  66         # Batch 1: 6 clusters (3 new)
  67         Pattern('0x65', 0x0000000),   # Full overwrite
  68         Pattern('0x77', 0x00f8000),   # Partial-left (1M-32K)
  69         Pattern('0x72', 0x2008000),   # Partial-right (32M+32K)
  70         Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K)
  71     PatternGroup([
  72         # Batch 2: 7 clusters (3 new)
  73         Pattern('0x74', 0x0010000),   # Adjacent-right
  74         Pattern('0x69', 0x00e8000),   # Partial-left  (1M-96K)
  75         Pattern('0x6e', 0x2018000),   # Partial-right (32M+96K)
  76         Pattern('0x67', 0x3fe0000,
  77                 2*GRANULARITY)]),     # Overwrite [(64M-128K)-64M)
  78     PatternGroup([
  79         # Batch 3: 8 clusters (5 new)
  80         # Carefully chosen such that nothing re-dirties the one cluster
  81         # that copies out successfully before failure in Group #1.
  82         Pattern('0xaa', 0x0010000,
  83                 3*GRANULARITY),       # Overwrite and 2x Adjacent-right
  84         Pattern('0xbb', 0x00d8000),   # Partial-left (1M-160K)
  85         Pattern('0xcc', 0x2028000),   # Partial-right (32M+160K)
  86         Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
  87 ]
  88
  89
  90 class EmulatedBitmap:
  91     def __init__(self, granularity=GRANULARITY):
  92         self._bits = set()
  93         self.granularity = granularity
  94
  95     def dirty_bits(self, bits):
  96         self._bits |= set(bits)
  97
  98     def dirty_group(self, n):
  99         self.dirty_bits(GROUPS[n].bits(self.granularity))
 100
 101     def clear(self):
 102         self._bits = set()
 103
 104     def clear_bits(self, bits):
 105         self._bits -= set(bits)
 106
 107     def clear_bit(self, bit):
 108         self.clear_bits({bit})
 109
 110     def clear_group(self, n):
 111         self.clear_bits(GROUPS[n].bits(self.granularity))
 112
 113     @property
 114     def first_bit(self):
 115         return sorted(self.bits)[0]
 116
 117     @property
 118     def bits(self):
 119         return self._bits
 120
 121     @property
 122     def count(self):
 123         return len(self.bits)
 124
 125     def compare(self, qmp_bitmap):
 126         """
 127         Print a nice human-readable message checking that a bitmap as reported
 128         by the QMP interface has as many bits set as we expect it to.
 129         """
 130
 131         name = qmp_bitmap.get('name', '(anonymous)')
 132         log("= Checking Bitmap {:s} =".format(name))
 133
 134         want = self.count
 135         have = qmp_bitmap['count'] // qmp_bitmap['granularity']
 136
 137         log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
 138             want, have, "OK!" if want == have else "ERROR!"))
 139         log('')
 140
 141
 142 class Drive:
 143     """Represents, vaguely, a drive attached to a VM.
 144     Includes format, graph, and device information."""
 145
 146     def __init__(self, path, vm=None):
 147         self.path = path
 148         self.vm = vm
 149         self.fmt = None
 150         self.size = None
 151         self.node = None
 152
 153     def img_create(self, fmt, size):
 154         self.fmt = fmt
 155         self.size = size
 156         iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size))
 157
 158     def create_target(self, name, fmt, size):
 159         basename = os.path.basename(self.path)
 160         file_node_name = "file_{}".format(basename)
 161         vm = self.vm
 162
 163         log(vm.command('blockdev-create', job_id='bdc-file-job',
 164                        options={
 165                            'driver': 'file',
 166                            'filename': self.path,
 167                            'size': 0,
 168                        }))
 169         vm.run_job('bdc-file-job')
 170         log(vm.command('blockdev-add', driver='file',
 171                        node_name=file_node_name, filename=self.path))
 172
 173         log(vm.command('blockdev-create', job_id='bdc-fmt-job',
 174                        options={
 175                            'driver': fmt,
 176                            'file': file_node_name,
 177                            'size': size,
 178                        }))
 179         vm.run_job('bdc-fmt-job')
 180         log(vm.command('blockdev-add', driver=fmt,
 181                        node_name=name,
 182                        file=file_node_name))
 183         self.fmt = fmt
 184         self.size = size
 185         self.node = name
 186
 187 def blockdev_backup(vm, device, target, sync, **kwargs):
 188     # Strip any arguments explicitly nulled by the caller:
 189     kwargs = {key: val for key, val in kwargs.items() if val is not None}
 190     result = vm.qmp_log('blockdev-backup',
 191                         device=device,
 192                         target=target,
 193                         sync=sync,
 194                         filter_node_name='backup-top',
 195                         x_perf={'max-workers': 1},
 196                         **kwargs)
 197     return result
 198
 199 def blockdev_backup_mktarget(drive, target_id, filepath, sync, **kwargs):
 200     target_drive = Drive(filepath, vm=drive.vm)
 201     target_drive.create_target(target_id, drive.fmt, drive.size)
 202     blockdev_backup(drive.vm, drive.node, target_id, sync, **kwargs)
 203
 204 def reference_backup(drive, n, filepath):
 205     log("--- Reference Backup #{:d} ---\n".format(n))
 206     target_id = "ref_target_{:d}".format(n)
 207     job_id = "ref_backup_{:d}".format(n)
 208     blockdev_backup_mktarget(drive, target_id, filepath, "full",
 209                              job_id=job_id)
 210     drive.vm.run_job(job_id, auto_dismiss=True)
 211     log('')
 212
 213 def backup(drive, n, filepath, sync, **kwargs):
 214     log("--- Test Backup #{:d} ---\n".format(n))
 215     target_id = "backup_target_{:d}".format(n)
 216     job_id = "backup_{:d}".format(n)
 217     kwargs.setdefault('auto-finalize', False)
 218     blockdev_backup_mktarget(drive, target_id, filepath, sync,
 219                              job_id=job_id, **kwargs)
 220     return job_id
 221
 222 def perform_writes(drive, n, filter_node_name=None):
 223     log("--- Write #{:d} ---\n".format(n))
 224     for pattern in GROUPS[n].patterns:
 225         cmd = "write -P{:s} 0x{:07x} 0x{:x}".format(
 226             pattern.byte,
 227             pattern.offset,
 228             pattern.size)
 229         log(cmd)
 230         log(drive.vm.hmp_qemu_io(filter_node_name or drive.node, cmd))
 231     bitmaps = drive.vm.query_bitmaps()
 232     log({'bitmaps': bitmaps}, indent=2)
 233     log('')
 234     return bitmaps
 235
 236
 237 def compare_images(image, reference, baseimg=None, expected_match=True):
 238     """
 239     Print a nice human-readable message comparing these images.
 240     """
 241     expected_ret = 0 if expected_match else 1
 242     if baseimg:
 243         qemu_img("rebase", "-u", "-b", baseimg, '-F', iotests.imgfmt, image)
 244
 245     sub = qemu_img("compare", image, reference, check=False)
 246
 247     log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format(
 248         image, reference,
 249         "Identical" if sub.returncode == 0 else "Mismatch",
 250         "OK!" if sub.returncode == expected_ret else "ERROR!"),
 251         filters=[iotests.filter_testfiles])
 252
 253 def test_bitmap_sync(bsync_mode, msync_mode='bitmap', failure=None):
 254     """
 255     Test bitmap backup routines.
 256
 257     :param bsync_mode: Is the Bitmap Sync mode, and can be any of:
 258         - on-success: This is the "incremental" style mode. Bitmaps are
 259                       synchronized to what was copied out only on success.
 260                       (Partial images must be discarded.)
 261         - never:      This is the "differential" style mode.
 262                       Bitmaps are never synchronized.
 263         - always:     This is a "best effort" style mode.
 264                       Bitmaps are always synchronized, regardless of failure.
 265                       (Partial images must be kept.)
 266
 267     :param msync_mode: The mirror sync mode to use for the first backup.
 268                        Can be any one of:
 269         - bitmap: Backups based on bitmap manifest.
 270         - full:   Full backups.
 271         - top:    Full backups of the top layer only.
 272
 273     :param failure: Is the (optional) failure mode, and can be any of:
 274         - None:         No failure. Test the normative path. Default.
 275         - simulated:    Cancel the job right before it completes.
 276                         This also tests writes "during" the job.
 277         - intermediate: This tests a job that fails mid-process and produces
 278                         an incomplete backup. Testing limitations prevent
 279                         testing competing writes.
 280     """
 281     with iotests.FilePath(
 282             'img', 'bsync1', 'bsync2', 'fbackup0', 'fbackup1', 'fbackup2') as \
 283             (img_path, bsync1, bsync2, fbackup0, fbackup1, fbackup2), \
 284          iotests.VM() as vm:
 285
 286         mode = "Mode {:s}; Bitmap Sync {:s}".format(msync_mode, bsync_mode)
 287         preposition = "with" if failure else "without"
 288         cond = "{:s} {:s}".format(preposition,
 289                                   "{:s} failure".format(failure) if failure
 290                                   else "failure")
 291         log("\n=== {:s} {:s} ===\n".format(mode, cond))
 292
 293         log('--- Preparing image & VM ---\n')
 294         drive0 = Drive(img_path, vm=vm)
 295         drive0.img_create(iotests.imgfmt, SIZE)
 296         vm.add_device("{},id=scsi0".format('virtio-scsi'))
 297         vm.launch()
 298
 299         file_config = {
 300             'driver': 'file',
 301             'filename': drive0.path
 302         }
 303
 304         if failure == 'intermediate':
 305             file_config = {
 306                 'driver': 'blkdebug',
 307                 'image': file_config,
 308                 'set-state': [{
 309                     'event': 'flush_to_disk',
 310                     'state': 1,
 311                     'new_state': 2
 312                 }, {
 313                     'event': 'read_aio',
 314                     'state': 2,
 315                     'new_state': 3
 316                 }],
 317                 'inject-error': [{
 318                     'event': 'read_aio',
 319                     'errno': 5,
 320                     'state': 3,
 321                     'immediately': False,
 322                     'once': True
 323                 }]
 324             }
 325
 326         drive0.node = 'drive0'
 327         vm.qmp_log('blockdev-add',
 328                    filters=[iotests.filter_qmp_testfiles],
 329                    node_name=drive0.node,
 330                    driver=drive0.fmt,
 331                    file=file_config)
 332         log('')
 333
 334         # 0 - Writes and Reference Backup
 335         perform_writes(drive0, 0)
 336         reference_backup(drive0, 0, fbackup0)
 337         log('--- Add Bitmap ---\n')
 338         vm.qmp_log("block-dirty-bitmap-add", node=drive0.node,
 339                    name="bitmap0", granularity=GRANULARITY)
 340         log('')
 341         ebitmap = EmulatedBitmap()
 342
 343         # 1 - Writes and Reference Backup
 344         bitmaps = perform_writes(drive0, 1)
 345         ebitmap.dirty_group(1)
 346         bitmap = vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)
 347         ebitmap.compare(bitmap)
 348         reference_backup(drive0, 1, fbackup1)
 349
 350         # 1 - Test Backup (w/ Optional induced failure)
 351         if failure == 'intermediate':
 352             # Activate blkdebug induced failure for second-to-next read
 353             log(vm.hmp_qemu_io(drive0.node, 'flush'))
 354             log('')
 355         job = backup(drive0, 1, bsync1, msync_mode,
 356                      bitmap="bitmap0", bitmap_mode=bsync_mode)
 357
 358         def _callback():
 359             """Issue writes while the job is open to test bitmap divergence."""
 360             # Note: when `failure` is 'intermediate', this isn't called.
 361             log('')
 362             bitmaps = perform_writes(drive0, 2, filter_node_name='backup-top')
 363             # Named bitmap (static, should be unchanged)
 364             ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0',
 365                                           bitmaps=bitmaps))
 366             # Anonymous bitmap (dynamic, shows new writes)
 367             anonymous = EmulatedBitmap()
 368             anonymous.dirty_group(2)
 369             anonymous.compare(vm.get_bitmap(drive0.node, '', recording=True,
 370                                             bitmaps=bitmaps))
 371
 372             # Simulate the order in which this will happen:
 373             # group 1 gets cleared first, then group two gets written.
 374             if ((bsync_mode == 'on-success' and not failure) or
 375                 (bsync_mode == 'always')):
 376                 ebitmap.clear()
 377             ebitmap.dirty_group(2)
 378
 379         vm.run_job(job, auto_dismiss=True, auto_finalize=False,
 380                    pre_finalize=_callback,
 381                    cancel=(failure == 'simulated'))
 382         bitmaps = vm.query_bitmaps()
 383         log({'bitmaps': bitmaps}, indent=2)
 384         log('')
 385
 386         if bsync_mode == 'always' and failure == 'intermediate':
 387             # TOP treats anything allocated as dirty, expect to see:
 388             if msync_mode == 'top':
 389                 ebitmap.dirty_group(0)
 390
 391             # We manage to copy one sector (one bit) before the error.
 392             ebitmap.clear_bit(ebitmap.first_bit)
 393
 394             # Full returns all bits set except what was copied/skipped
 395             if msync_mode == 'full':
 396                 fail_bit = ebitmap.first_bit
 397                 ebitmap.clear()
 398                 ebitmap.dirty_bits(range(fail_bit, SIZE // GRANULARITY))
 399
 400         ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
 401
 402         # 2 - Writes and Reference Backup
 403         bitmaps = perform_writes(drive0, 3)
 404         ebitmap.dirty_group(3)
 405         ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
 406         reference_backup(drive0, 2, fbackup2)
 407
 408         # 2 - Bitmap Backup (In failure modes, this is a recovery.)
 409         job = backup(drive0, 2, bsync2, "bitmap",
 410                      bitmap="bitmap0", bitmap_mode=bsync_mode)
 411         vm.run_job(job, auto_dismiss=True, auto_finalize=False)
 412         bitmaps = vm.query_bitmaps()
 413         log({'bitmaps': bitmaps}, indent=2)
 414         log('')
 415         if bsync_mode != 'never':
 416             ebitmap.clear()
 417         ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
 418
 419         log('--- Cleanup ---\n')
 420         vm.qmp_log("block-dirty-bitmap-remove",
 421                    node=drive0.node, name="bitmap0")
 422         bitmaps = vm.query_bitmaps()
 423         log({'bitmaps': bitmaps}, indent=2)
 424         vm.shutdown()
 425         log('')
 426
 427         log('--- Verification ---\n')
 428         # 'simulated' failures will actually all pass here because we canceled
 429         # while "pending". This is actually undefined behavior,
 430         # don't rely on this to be true!
 431         compare_images(bsync1, fbackup1, baseimg=fbackup0,
 432                        expected_match=failure != 'intermediate')
 433         if not failure or bsync_mode == 'always':
 434             # Always keep the last backup on success or when using 'always'
 435             base = bsync1
 436         else:
 437             base = fbackup0
 438         compare_images(bsync2, fbackup2, baseimg=base)
 439         compare_images(img_path, fbackup2)
 440         log('')
 441
 442 def test_backup_api():
 443     """
 444     Test malformed and prohibited invocations of the backup API.
 445     """
 446     with iotests.FilePath('img', 'bsync1') as (img_path, backup_path), \
 447          iotests.VM() as vm:
 448
 449         log("\n=== API failure tests ===\n")
 450         log('--- Preparing image & VM ---\n')
 451         drive0 = Drive(img_path, vm=vm)
 452         drive0.img_create(iotests.imgfmt, SIZE)
 453         vm.add_device("{},id=scsi0".format('virtio-scsi'))
 454         vm.launch()
 455
 456         file_config = {
 457             'driver': 'file',
 458             'filename': drive0.path
 459         }
 460
 461         drive0.node = 'drive0'
 462         vm.qmp_log('blockdev-add',
 463                    filters=[iotests.filter_qmp_testfiles],
 464                    node_name=drive0.node,
 465                    driver=drive0.fmt,
 466                    file=file_config)
 467         log('')
 468
 469         target0 = Drive(backup_path, vm=vm)
 470         target0.create_target("backup_target", drive0.fmt, drive0.size)
 471         log('')
 472
 473         vm.qmp_log("block-dirty-bitmap-add", node=drive0.node,
 474                    name="bitmap0", granularity=GRANULARITY)
 475         log('')
 476
 477         log('-- Testing invalid QMP commands --\n')
 478
 479         error_cases = {
 480             'incremental': {
 481                 None:        ['on-success', 'always', 'never', None],
 482                 'bitmap404': ['on-success', 'always', 'never', None],
 483                 'bitmap0':   ['always', 'never']
 484             },
 485             'bitmap': {
 486                 None:        ['on-success', 'always', 'never', None],
 487                 'bitmap404': ['on-success', 'always', 'never', None],
 488                 'bitmap0':   [None],
 489             },
 490             'full': {
 491                 None:        ['on-success', 'always', 'never'],
 492                 'bitmap404': ['on-success', 'always', 'never', None],
 493                 'bitmap0':   ['never', None],
 494             },
 495             'top': {
 496                 None:        ['on-success', 'always', 'never'],
 497                 'bitmap404': ['on-success', 'always', 'never', None],
 498                 'bitmap0':   ['never', None],
 499             },
 500             'none': {
 501                 None:        ['on-success', 'always', 'never'],
 502                 'bitmap404': ['on-success', 'always', 'never', None],
 503                 'bitmap0':   ['on-success', 'always', 'never', None],
 504             }
 505         }
 506
 507         # Dicts, as always, are not stably-ordered prior to 3.7, so use tuples:
 508         for sync_mode in ('incremental', 'bitmap', 'full', 'top', 'none'):
 509             log("-- Sync mode {:s} tests --\n".format(sync_mode))
 510             for bitmap in (None, 'bitmap404', 'bitmap0'):
 511                 for policy in error_cases[sync_mode][bitmap]:
 512                     blockdev_backup(drive0.vm, drive0.node, "backup_target",
 513                                     sync_mode, job_id='api_job',
 514                                     bitmap=bitmap, bitmap_mode=policy)
 515                     log('')
 516
 517
 518 def main():
 519     for bsync_mode in ("never", "on-success", "always"):
 520         for failure in ("simulated", "intermediate", None):
 521             test_bitmap_sync(bsync_mode, "bitmap", failure)
 522
 523     for sync_mode in ('full', 'top'):
 524         for bsync_mode in ('on-success', 'always'):
 525             for failure in ('simulated', 'intermediate', None):
 526                 test_bitmap_sync(bsync_mode, sync_mode, failure)
 527
 528     test_backup_api()
 529
 530 if __name__ == '__main__':
 531     iotests.script_main(main, supported_fmts=['qcow2'],
 532                         supported_protocols=['file'])