tests/qemu-iotests/108

   1 #!/usr/bin/env bash
   2 # group: rw auto quick
   3 #
   4 # Test case for repairing qcow2 images which cannot be repaired using
   5 # the on-disk refcount structures
   6 #
   7 # Copyright (C) 2014 Red Hat, Inc.
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU General Public License as published by
  11 # the Free Software Foundation; either version 2 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21 #
  22
  23 # creator
  24 owner=hreitz@redhat.com
  25
  26 seq="$(basename $0)"
  27 echo "QA output created by $seq"
  28
  29 status=1        # failure is the default!
  30
  31 _cleanup()
  32 {
  33     _cleanup_test_img
  34     if [ -f "$TEST_DIR/qsd.pid" ]; then
  35         qsd_pid=$(cat "$TEST_DIR/qsd.pid")
  36         kill -KILL "$qsd_pid"
  37         fusermount -u "$TEST_DIR/fuse-export" &>/dev/null
  38     fi
  39     rm -f "$TEST_DIR/fuse-export"
  40 }
  41 trap "_cleanup; exit \$status" 0 1 2 3 15
  42
  43 # get standard environment, filters and checks
  44 . ./common.rc
  45 . ./common.filter
  46 . ./common.qemu
  47
  48 # This tests qcow2-specific low-level functionality
  49 _supported_fmt qcow2
  50 _supported_proto file fuse
  51 _supported_os Linux
  52 # This test directly modifies a refblock so it relies on refcount_bits being 16;
  53 # and the low-level modification it performs are not tuned for external data
  54 # files
  55 _unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file
  56
  57 # This test either needs sudo -n losetup or FUSE exports to work
  58 if sudo -n losetup &>/dev/null; then
  59     loopdev=true
  60 else
  61     loopdev=false
  62
  63     # QSD --export fuse will either yield "Parameter 'id' is missing"
  64     # or "Invalid parameter 'fuse'", depending on whether there is
  65     # FUSE support or not.
  66     error=$($QSD --export fuse 2>&1)
  67     if [[ $error = *"'fuse'"* ]]; then
  68         _notrun 'Passwordless sudo for losetup or FUSE support required, but' \
  69                 'neither is available'
  70     fi
  71 fi
  72
  73 echo
  74 echo '=== Repairing an image without any refcount table ==='
  75 echo
  76
  77 _make_test_img 64M
  78 # just write some data
  79 $QEMU_IO -c 'write -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
  80
  81 # refcount_table_offset
  82 poke_file "$TEST_IMG" $((0x30)) "\x00\x00\x00\x00\x00\x00\x00\x00"
  83 # refcount_table_clusters
  84 poke_file "$TEST_IMG" $((0x38)) "\x00\x00\x00\x00"
  85
  86 _check_test_img -r all
  87
  88 $QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
  89
  90 echo
  91 echo '=== Repairing unreferenced data cluster in new refblock area ==='
  92 echo
  93
  94 _make_test_img -o 'cluster_size=512' 64M
  95 # Allocate the first 128 kB in the image (first refblock)
  96 $QEMU_IO -c 'write 0 0x1b200' "$TEST_IMG" | _filter_qemu_io
  97 # should be 131072 == 0x20000
  98 stat -c '%s' "$TEST_IMG"
  99
 100 # Enter a cluster at 128 kB (0x20000)
 101 # XXX: This should be the first free entry in the last L2 table, but we cannot
 102 # be certain
 103 poke_file "$TEST_IMG" $((0x1ccc8)) "\x80\x00\x00\x00\x00\x02\x00\x00"
 104
 105 # Fill the cluster
 106 truncate -s $((0x20200)) "$TEST_IMG"
 107 $QEMU_IO -c "open -o driver=raw $TEST_IMG" -c 'write -P 42 128k 512' \
 108     | _filter_qemu_io
 109
 110 # The data should now appear at this guest offset
 111 $QEMU_IO -c 'read -P 42 0x1b200 512' "$TEST_IMG" | _filter_qemu_io
 112
 113 # This cluster is unallocated; fix it
 114 _check_test_img -r all
 115
 116 # This repair operation must have allocated a new refblock; and that refblock
 117 # should not overlap with the unallocated data cluster. If it does, the data
 118 # will be damaged, so check it.
 119 $QEMU_IO -c 'read -P 42 0x1b200 512' "$TEST_IMG" | _filter_qemu_io
 120
 121 echo
 122 echo '=== Repairing refblock beyond the image end ==='
 123 echo
 124
 125 echo
 126 echo '--- Otherwise clean ---'
 127 echo
 128
 129 _make_test_img 64M
 130 # Normally, qemu doesn't create empty refblocks, so we just have to do it by
 131 # hand
 132 # XXX: This should be the entry for the second refblock
 133 poke_file "$TEST_IMG" $((0x10008)) "\x00\x00\x00\x00\x00\x10\x00\x00"
 134 # Mark that refblock as used
 135 # XXX: This should be the 17th entry (cluster 16) of the first
 136 # refblock
 137 poke_file "$TEST_IMG" $((0x20020)) "\x00\x01"
 138 _check_test_img -r all
 139
 140 echo
 141 echo '--- Refblock is unallocated ---'
 142 echo
 143
 144 _make_test_img 64M
 145 poke_file "$TEST_IMG" $((0x10008)) "\x00\x00\x00\x00\x00\x10\x00\x00"
 146 _check_test_img -r all
 147
 148 echo
 149 echo '--- Signed overflow after the refblock ---'
 150 echo
 151
 152 _make_test_img 64M
 153 poke_file "$TEST_IMG" $((0x10008)) "\x7f\xff\xff\xff\xff\xff\x00\x00"
 154 _check_test_img -r all
 155
 156 echo
 157 echo '--- Unsigned overflow after the refblock ---'
 158 echo
 159
 160 _make_test_img 64M
 161 poke_file "$TEST_IMG" $((0x10008)) "\xff\xff\xff\xff\xff\xff\x00\x00"
 162 _check_test_img -r all
 163
 164 echo
 165 echo '=== Check rebuilt reftable location ==='
 166
 167 # In an earlier version of the refcount rebuild algorithm, the
 168 # reftable was generally placed at the image end (unless something was
 169 # allocated in the area covered by the refblock right before the image
 170 # file end, then we would try to place the reftable in that refblock).
 171 # This was later changed so the reftable would be placed in the
 172 # earliest possible location.  Test this.
 173
 174 echo
 175 echo '--- Does the image size increase? ---'
 176 echo
 177
 178 # First test: Just create some image, write some data to it, and
 179 # resize it so there is free space at the end of the image (enough
 180 # that it spans at least one full refblock, which for cluster_size=512
 181 # images, spans 128k).  With the old algorithm, the reftable would
 182 # have then been placed at the end of the image file, but with the new
 183 # one, it will be put in that free space.
 184 # We want to check whether the size of the image file increases due to
 185 # rebuilding the refcount structures (it should not).
 186
 187 _make_test_img -o 'cluster_size=512' 1M
 188 # Write something
 189 $QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
 190
 191 # Add free space
 192 file_len=$(stat -c '%s' "$TEST_IMG")
 193 truncate -s $((file_len + 256 * 1024)) "$TEST_IMG"
 194
 195 # Corrupt the image by saying the image header was not allocated
 196 rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
 197 rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8)
 198 poke_file "$TEST_IMG" $rb_offset "\x00\x00"
 199
 200 # Check whether rebuilding the refcount structures increases the image
 201 # file size
 202 file_len=$(stat -c '%s' "$TEST_IMG")
 203 echo
 204 # The only leaks there can be are the old refcount structures that are
 205 # leaked during rebuilding, no need to clutter the output with them
 206 _check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0'
 207 echo
 208 post_repair_file_len=$(stat -c '%s' "$TEST_IMG")
 209
 210 if [[ $file_len -eq $post_repair_file_len ]]; then
 211     echo 'OK: Image size did not change'
 212 else
 213     echo 'ERROR: Image size differs' \
 214         "($file_len before, $post_repair_file_len after)"
 215 fi
 216
 217 echo
 218 echo '--- Will the reftable occupy a hole specifically left for it?  ---'
 219 echo
 220
 221 # Note: With cluster_size=512, every refblock covers 128k.
 222 # The reftable covers 8M per reftable cluster.
 223
 224 # Create an image that requires two reftable clusters (just because
 225 # this is more interesting than a single-clustered reftable).
 226 _make_test_img -o 'cluster_size=512' 9M
 227 $QEMU_IO -c 'write 0 8M' "$TEST_IMG" | _filter_qemu_io
 228
 229 # Writing 8M will have resized the reftable.  Unfortunately, doing so
 230 # will leave holes in the file, so we need to fill them up so we can
 231 # be sure the whole file is allocated.  Do that by writing
 232 # consecutively smaller chunks starting from 8 MB, until the file
 233 # length increases even with a chunk size of 512.  Then we must have
 234 # filled all holes.
 235 ofs=$((8 * 1024 * 1024))
 236 block_len=$((16 * 1024))
 237 while [[ $block_len -ge 512 ]]; do
 238     file_len=$(stat -c '%s' "$TEST_IMG")
 239     while [[ $(stat -c '%s' "$TEST_IMG") -eq $file_len ]]; do
 240         # Do not include this in the reference output, it does not
 241         # really matter which qemu-io calls we do here exactly
 242         $QEMU_IO -c "write $ofs $block_len" "$TEST_IMG" >/dev/null
 243         ofs=$((ofs + block_len))
 244     done
 245     block_len=$((block_len / 2))
 246 done
 247
 248 # Fill up to 9M (do not include this in the reference output either,
 249 # $ofs is random for all we know)
 250 $QEMU_IO -c "write $ofs $((9 * 1024 * 1024 - ofs))" "$TEST_IMG" >/dev/null
 251
 252 # Make space as follows:
 253 # - For the first refblock: Right at the beginning of the image (this
 254 #   refblock is placed in the first place possible),
 255 # - For the reftable somewhere soon afterwards, still near the
 256 #   beginning of the image (i.e. covered by the first refblock); the
 257 #   reftable too is placed in the first place possible, but only after
 258 #   all refblocks have been placed)
 259 # No space is needed for the other refblocks, because no refblock is
 260 # put before the space it covers.  In this test case, we do not mind
 261 # if they are placed at the image file's end.
 262
 263 # Before we make that space, we have to find out the host offset of
 264 # the area that belonged to the two data clusters at guest offset 4k,
 265 # because we expect the reftable to be placed there, and we will have
 266 # to verify that it is.
 267
 268 l1_offset=$(peek_file_be "$TEST_IMG" 40 8)
 269 l2_offset=$(peek_file_be "$TEST_IMG" $l1_offset 8)
 270 l2_offset=$((l2_offset & 0x00fffffffffffe00))
 271 data_4k_offset=$(peek_file_be "$TEST_IMG" \
 272                  $((l2_offset + 4096 / 512 * 8)) 8)
 273 data_4k_offset=$((data_4k_offset & 0x00fffffffffffe00))
 274
 275 $QEMU_IO -c "discard 0 512" -c "discard 4k 1k" "$TEST_IMG" | _filter_qemu_io
 276
 277 # Corrupt the image by saying the image header was not allocated
 278 rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
 279 rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8)
 280 poke_file "$TEST_IMG" $rb_offset "\x00\x00"
 281
 282 echo
 283 # The only leaks there can be are the old refcount structures that are
 284 # leaked during rebuilding, no need to clutter the output with them
 285 _check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0'
 286 echo
 287
 288 # Check whether the reftable was put where we expected
 289 rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
 290 if [[ $rt_offset -eq $data_4k_offset ]]; then
 291     echo 'OK: Reftable is where we expect it'
 292 else
 293     echo "ERROR: Reftable is at $rt_offset, but was expected at $data_4k_offset"
 294 fi
 295
 296 echo
 297 echo '--- Rebuilding refcount structures on block devices ---'
 298 echo
 299
 300 # A block device cannot really grow, at least not during qemu-img
 301 # check.  As mentioned in the above cases, rebuilding the refcount
 302 # structure may lead to new refcount structures being written after
 303 # the end of the image, and in the past that happened even if there
 304 # was more than sufficient space in the image.  Such post-EOF writes
 305 # will not work on block devices, so test that the new algorithm
 306 # avoids it.
 307
 308 # If we have passwordless sudo and losetup, we can use those to create
 309 # a block device.  Otherwise, we can resort to qemu's FUSE export to
 310 # create a file that isn't growable, which effectively tests the same
 311 # thing.
 312
 313 _cleanup_test_img
 314 truncate -s $((64 * 1024 * 1024)) "$TEST_IMG"
 315
 316 if $loopdev; then
 317     export_mp=$(sudo -n losetup --show -f "$TEST_IMG")
 318     export_mp_driver=host_device
 319     sudo -n chmod go+rw "$export_mp"
 320 else
 321     # Create non-growable FUSE export that is a bit like an empty
 322     # block device
 323     export_mp="$TEST_DIR/fuse-export"
 324     export_mp_driver=file
 325     touch "$export_mp"
 326
 327     $QSD \
 328         --blockdev file,node-name=export-node,filename="$TEST_IMG" \
 329         --export fuse,id=fuse-export,node-name=export-node,mountpoint="$export_mp",writable=on,growable=off,allow-other=off \
 330         --pidfile "$TEST_DIR/qsd.pid" \
 331         --daemonize
 332 fi
 333
 334 # Now create a qcow2 image on the device -- unfortunately, qemu-img
 335 # create force-creates the file, so we have to resort to the
 336 # blockdev-create job.
 337 _launch_qemu \
 338     --blockdev $export_mp_driver,node-name=file,filename="$export_mp"
 339
 340 _send_qemu_cmd \
 341     $QEMU_HANDLE \
 342     '{ "execute": "qmp_capabilities" }' \
 343     'return'
 344
 345 # Small cluster size again, so the image needs multiple refblocks
 346 _send_qemu_cmd \
 347     $QEMU_HANDLE \
 348     '{ "execute": "blockdev-create",
 349        "arguments": {
 350            "job-id": "create",
 351            "options": {
 352                "driver": "qcow2",
 353                "file": "file",
 354                "size": '$((64 * 1024 * 1024))',
 355                "cluster-size": 512
 356            } } }' \
 357     '"concluded"'
 358
 359 _send_qemu_cmd \
 360     $QEMU_HANDLE \
 361     '{ "execute": "job-dismiss", "arguments": { "id": "create" } }' \
 362     'return'
 363
 364 _send_qemu_cmd \
 365     $QEMU_HANDLE \
 366     '{ "execute": "quit" }' \
 367     'return'
 368
 369 wait=y _cleanup_qemu
 370 echo
 371
 372 # Write some data
 373 $QEMU_IO -c 'write 0 64k' "$export_mp" | _filter_qemu_io
 374
 375 # Corrupt the image by saying the image header was not allocated
 376 rt_offset=$(peek_file_be "$export_mp" 48 8)
 377 rb_offset=$(peek_file_be "$export_mp" $rt_offset 8)
 378 poke_file "$export_mp" $rb_offset "\x00\x00"
 379
 380 # Repairing such a simple case should just work
 381 # (We used to put the reftable at the end of the image file, which can
 382 # never work for non-growable devices.)
 383 echo
 384 TEST_IMG="$export_mp" _check_test_img -r all \
 385     | grep -v '^Repairing cluster.*refcount=1 reference=0'
 386
 387 if $loopdev; then
 388     sudo -n losetup -d "$export_mp"
 389 else
 390     qsd_pid=$(cat "$TEST_DIR/qsd.pid")
 391     kill -TERM "$qsd_pid"
 392     # Wait for process to exit (cannot `wait` because the QSD is daemonized)
 393     while [ -f "$TEST_DIR/qsd.pid" ]; do
 394         true
 395     done
 396 fi
 397
 398 # success, all done
 399 echo '*** done'
 400 rm -f $seq.full
 401 status=0