block/fvd-utils.c

   1 /*
   2  * Copyright (c) 2010-2011 IBM
   3  *
   4  * Authors:
   5  *         Chunqiang Tang <ctang@us.ibm.com>
   6  *
   7  * This work is licensed under the terms of the GNU GPL, version 2.
   8  * See the COPYING file in the top-level directory.
   9  */
  10
  11 /*==============================================================================
  12  *  A short description: this module implements basic utility functions for
  13  *  the Fast Virtual Disk (FVD) format.
  14  *============================================================================*/
  15
  16 static inline int stale_bitmap_show_sector_in_base_img (int64_t sector_num,
  17                                                 const BDRVFvdState * s)
  18 {
  19     if (sector_num >= s->nb_sectors_in_base_img) {
  20         return FALSE;
  21     }
  22
  23     int64_t block_num = sector_num / s->block_size;
  24     int64_t bitmap_byte_offset = block_num / 8;
  25     uint8_t bitmap_bit_offset = block_num % 8;
  26     uint8_t b = s->stale_bitmap[bitmap_byte_offset];
  27     return 0 == (int) ((b >> bitmap_bit_offset) & 0x01);
  28 }
  29
  30 static inline int
  31 fresh_bitmap_show_sector_in_base_img (int64_t sector_num,
  32                                               const BDRVFvdState * s)
  33 {
  34     if (sector_num >= s->nb_sectors_in_base_img) {
  35         return FALSE;
  36     }
  37
  38     int64_t block_num = sector_num / s->block_size;
  39     int64_t bitmap_byte_offset = block_num / 8;
  40     uint8_t bitmap_bit_offset = block_num % 8;
  41     uint8_t b = s->fresh_bitmap[bitmap_byte_offset];
  42     return 0 == (int) ((b >> bitmap_bit_offset) & 0x01);
  43 }
  44
  45 static inline void update_fresh_bitmap (int64_t sector_num, int nb_sectors,
  46                                            const BDRVFvdState * s)
  47 {
  48     if (sector_num >= s->nb_sectors_in_base_img) {
  49         return;
  50     }
  51
  52     int64_t end = sector_num + nb_sectors;
  53     if (end > s->nb_sectors_in_base_img) {
  54         end = s->nb_sectors_in_base_img;
  55     }
  56
  57     int64_t block_num = sector_num / s->block_size;
  58     int64_t block_end = (end - 1) / s->block_size;
  59
  60     for (; block_num <= block_end; block_num++) {
  61         int64_t bitmap_byte_offset = block_num / 8;
  62         uint8_t bitmap_bit_offset = block_num % 8;
  63         uint8_t mask = (uint8_t) (0x01 << bitmap_bit_offset);
  64         uint8_t b = s->fresh_bitmap[bitmap_byte_offset];
  65         if (!(b & mask)) {
  66             b |= mask;
  67             s->fresh_bitmap[bitmap_byte_offset] = b;
  68         }
  69     }
  70 }
  71
  72 static void update_stale_bitmap (BDRVFvdState * s, int64_t sector_num,
  73                                  int nb_sectors)
  74 {
  75     if (sector_num >= s->nb_sectors_in_base_img) {
  76         return;
  77     }
  78
  79     int64_t end = sector_num + nb_sectors;
  80     if (end > s->nb_sectors_in_base_img) {
  81         end = s->nb_sectors_in_base_img;
  82     }
  83
  84     int64_t block_num = sector_num / s->block_size;
  85     const int64_t block_end = (end - 1) / s->block_size;
  86
  87     for (; block_num <= block_end; block_num++) {
  88         int64_t bitmap_byte_offset = block_num / 8;
  89         uint8_t bitmap_bit_offset = block_num % 8;
  90         uint8_t mask = (uint8_t) (0x01 << bitmap_bit_offset);
  91         uint8_t b = s->stale_bitmap[bitmap_byte_offset];
  92         if (!(b & mask)) {
  93             ASSERT (s->stale_bitmap == s->fresh_bitmap ||
  94                     (s->fresh_bitmap[bitmap_byte_offset] & mask));
  95             b |= mask;
  96             s->stale_bitmap[bitmap_byte_offset] = b;
  97         }
  98     }
  99 }
 100
 101 static void update_both_bitmaps (BDRVFvdState * s, int64_t sector_num,
 102                                  int nb_sectors)
 103 {
 104     if (sector_num >= s->nb_sectors_in_base_img) {
 105         return;
 106     }
 107
 108     int64_t end = sector_num + nb_sectors;
 109     if (end > s->nb_sectors_in_base_img) {
 110         end = s->nb_sectors_in_base_img;
 111     }
 112
 113     int64_t block_num = sector_num / s->block_size;
 114     const int64_t block_end = (end - 1) / s->block_size;
 115
 116     for (; block_num <= block_end; block_num++) {
 117         int64_t bitmap_byte_offset = block_num / 8;
 118         uint8_t bitmap_bit_offset = block_num % 8;
 119         uint8_t mask = (uint8_t) (0x01 << bitmap_bit_offset);
 120         uint8_t b = s->fresh_bitmap[bitmap_byte_offset];
 121         if (!(b & mask)) {
 122             b |= mask;
 123             s->fresh_bitmap[bitmap_byte_offset] =
 124                 s->stale_bitmap[bitmap_byte_offset] = b;
 125         }
 126     }
 127 }
 128
 129 /* Return TRUE if a valid region is found. */
 130 static int find_region_in_base_img (BDRVFvdState * s, int64_t * from,
 131                                     int64_t * to)
 132 {
 133     int64_t sec = *from;
 134     int64_t last_sec = *to;
 135
 136     if (last_sec > s->nb_sectors_in_base_img) {
 137         last_sec = s->nb_sectors_in_base_img;
 138     }
 139
 140     if (sec >= last_sec) {
 141         return FALSE;
 142     }
 143
 144     if (!fresh_bitmap_show_sector_in_base_img (sec, s)) {
 145         /* Find the first sector in the base image. */
 146
 147         sec = ROUND_UP (sec + 1, s->block_size); /* Begin of next block. */
 148         while (1) {
 149             if (sec >= last_sec) {
 150                 return FALSE;
 151             }
 152             if (fresh_bitmap_show_sector_in_base_img (sec, s)) {
 153                 break;
 154             }
 155             sec += s->block_size;        /* Begin of the next block. */
 156         }
 157     }
 158
 159     /* Find the end of the region in the base image. */
 160     int64_t first_sec = sec;
 161     sec = ROUND_UP (sec + 1, s->block_size);        /* Begin of next block. */
 162     while (1) {
 163         if (sec >= last_sec) {
 164             sec = last_sec;
 165             break;
 166         }
 167         if (!fresh_bitmap_show_sector_in_base_img (sec, s)) {
 168             break;
 169         }
 170         sec += s->block_size;        /* Begin of the next block. */
 171     }
 172     last_sec = sec;
 173
 174     /* Check conflicting copy-on-reads. */
 175     FvdAIOCB *old;
 176     QLIST_FOREACH (old, &s->copy_locks, copy_lock.next) {
 177         if (old->copy_lock.begin <= first_sec
 178                 && first_sec < old->copy_lock.end) {
 179             first_sec = old->copy_lock.end;
 180         }
 181         if (old->copy_lock.begin < last_sec && last_sec <= old->copy_lock.end) {
 182             last_sec = old->copy_lock.begin;
 183         }
 184     }
 185
 186     if (first_sec >= last_sec) {
 187         return FALSE;        /* The entire region is already covered. */
 188     }
 189
 190      /* This loop cannot be merged with the loop above. Otherwise, the logic
 191       * would be incorrect.  This loop covers the case that an old request
 192       * spans over a subset of the region being checked. */
 193     QLIST_FOREACH (old, &s->copy_locks, copy_lock.next) {
 194         if (first_sec <= old->copy_lock.begin
 195             && old->copy_lock.begin < last_sec) {
 196             last_sec = old->copy_lock.begin;
 197         }
 198     }
 199
 200     /* Check conflicting writes. */
 201     QLIST_FOREACH (old, &s->write_locks, write.next_write_lock) {
 202         int64_t old_end = old->sector_num + old->nb_sectors;
 203         if (old->sector_num <= first_sec && first_sec < old_end) {
 204             first_sec = old_end;
 205         }
 206         if (old->sector_num < last_sec && last_sec <= old_end) {
 207             last_sec = old->sector_num;
 208         }
 209     }
 210
 211     if (first_sec >= last_sec) {
 212         return FALSE;        /* The entire region is already covered. */
 213     }
 214
 215      /* This loop cannot be merged with the loop above. Otherwise, the logic
 216       * would be incorrect.  This loop covers the case that an old request
 217       * spans over a subset of the region being checked. */
 218     QLIST_FOREACH (old, &s->write_locks, write.next_write_lock) {
 219         if (first_sec <= old->sector_num && old->sector_num < last_sec) {
 220             last_sec = old->sector_num;
 221         }
 222     }
 223
 224     ASSERT (first_sec % s->block_size == 0 && (last_sec % s->block_size == 0
 225                 || last_sec == s->nb_sectors_in_base_img));
 226
 227     *from = first_sec;
 228     *to = last_sec;
 229     return TRUE;
 230 }
 231
 232 static inline int bitmap_show_sector_in_base_img (int64_t sector_num,
 233                                                        const BDRVFvdState * s,
 234                                                        int bitmap_offset,
 235                                                        uint8_t * bitmap)
 236 {
 237     if (sector_num >= s->nb_sectors_in_base_img) {
 238         return FALSE;
 239     }
 240
 241     int64_t block_num = sector_num / s->block_size;
 242     int64_t bitmap_byte_offset = block_num / 8 - bitmap_offset;
 243     uint8_t bitmap_bit_offset = block_num % 8;
 244     uint8_t b = bitmap[bitmap_byte_offset];
 245     return 0 == (int) ((b >> bitmap_bit_offset) & 0x01);
 246 }
 247
 248 static inline void copy_to_iov (struct iovec *iov, int *p_index,
 249                                 uint8_t ** p_buf, int *p_left,
 250                                 uint8_t * source, int total)
 251 {
 252     int index = *p_index;
 253     uint8_t *buf = *p_buf;
 254     int left = *p_left;
 255
 256     if (left <= 0) {
 257         index++;
 258         buf = iov[index].iov_base;
 259         left = iov[index].iov_len;
 260     }
 261
 262     while (1) {
 263         if (left >= total) {
 264             memcpy (buf, source, total);
 265             *p_buf = buf + total;
 266             *p_left = left - total;
 267             *p_index = index;
 268             return;
 269         }
 270
 271         memcpy (buf, source, left);
 272         total -= left;
 273         source += left;
 274         index++;
 275         buf = iov[index].iov_base;
 276         left = iov[index].iov_len;
 277     }
 278 }
 279
 280 static inline void init_data_region (BDRVFvdState * s)
 281 {
 282     bdrv_truncate (s->fvd_data, s->data_offset * 512 + s->virtual_disk_size);
 283     s->data_region_prepared = TRUE;
 284 }
 285
 286 static inline void update_clean_shutdown_flag (BDRVFvdState * s, int clean)
 287 {
 288     FvdHeader header;
 289     if (!read_fvd_header (s, &header)) {
 290         header.clean_shutdown = clean;
 291
 292         if (!update_fvd_header (s, &header)) {
 293             QDEBUG ("Set clean_shutdown to %s\n", BOOL (clean));
 294         }
 295     }
 296 }
 297
 298 static inline int stale_bitmap_need_update (FvdAIOCB * acb)
 299 {
 300     BlockDriverState *bs = acb->common.bs;
 301     BDRVFvdState *s = bs->opaque;
 302     int64_t end = acb->sector_num + acb->nb_sectors;
 303
 304     if (end > s->nb_sectors_in_base_img) {
 305         end = s->nb_sectors_in_base_img;
 306     }
 307     int64_t block_end = (end - 1) / s->block_size;
 308     int64_t block_num = acb->sector_num / s->block_size;
 309
 310     for (; block_num <= block_end; block_num++) {
 311         int64_t bitmap_byte_offset = block_num / 8;
 312         uint8_t bitmap_bit_offset = block_num % 8;
 313         uint8_t mask = (uint8_t) (0x01 << bitmap_bit_offset);
 314         uint8_t b = s->stale_bitmap[bitmap_byte_offset];
 315         if (!(b & mask)) {
 316             return TRUE;
 317         }
 318     }
 319
 320     return FALSE;
 321 }
 322
 323 static int update_fresh_bitmap_and_check_stale_bitmap (FvdAIOCB * acb)
 324 {
 325     BlockDriverState *bs = acb->common.bs;
 326     BDRVFvdState *s = bs->opaque;
 327
 328     if (acb->sector_num >= s->nb_sectors_in_base_img) {
 329         return FALSE;
 330     }
 331
 332     int need_update = FALSE;
 333     int64_t end = acb->sector_num + acb->nb_sectors;
 334
 335     if (end > s->nb_sectors_in_base_img) {
 336         end = s->nb_sectors_in_base_img;
 337     }
 338
 339     int64_t block_end = (end - 1) / s->block_size;
 340     int64_t block_num = acb->sector_num / s->block_size;
 341
 342     for (; block_num <= block_end; block_num++) {
 343         int64_t bitmap_byte_offset = block_num / 8;
 344         uint8_t bitmap_bit_offset = block_num % 8;
 345         uint8_t mask = (uint8_t) (0x01 << bitmap_bit_offset);
 346         uint8_t b = s->stale_bitmap[bitmap_byte_offset];
 347         if (b & mask) {
 348             /* If the bit in stale_bitmap is set, the corresponding bit in
 349              * fresh_bitmap must be set already. */
 350             continue;
 351         }
 352
 353         need_update = TRUE;
 354         b = s->fresh_bitmap[bitmap_byte_offset];
 355         if (!(b & mask)) {
 356             b |= mask;
 357             s->fresh_bitmap[bitmap_byte_offset] = b;
 358         }
 359     }
 360
 361     return need_update;
 362 }
 363
 364 static void fvd_header_cpu_to_le (FvdHeader * header)
 365 {
 366     cpu_to_le32s (&header->magic);
 367     cpu_to_le32s (&header->version);
 368     cpu_to_le32s ((uint32_t *) & header->all_data_in_fvd_img);
 369     cpu_to_le32s ((uint32_t *) & header->generate_prefetch_profile);
 370     cpu_to_le64s ((uint64_t *) & header->metadata_size);
 371     cpu_to_le64s ((uint64_t *) & header->virtual_disk_size);
 372     cpu_to_le64s ((uint64_t *) & header->base_img_size);
 373     cpu_to_le64s ((uint64_t *) & header->max_outstanding_copy_on_read_data);
 374     cpu_to_le64s ((uint64_t *) & header->bitmap_offset);
 375     cpu_to_le64s ((uint64_t *) & header->prefetch_profile_offset);
 376     cpu_to_le64s ((uint64_t *) & header->prefetch_profile_entries);
 377     cpu_to_le64s ((uint64_t *) & header->bitmap_size);
 378     cpu_to_le32s ((uint32_t *) & header->copy_on_read);
 379     cpu_to_le32s ((uint32_t *) & header->need_zero_init);
 380     cpu_to_le32s ((uint32_t *) & header->prefetch_start_delay);
 381     cpu_to_le32s ((uint32_t *) & header->profile_directed_prefetch_start_delay);
 382     cpu_to_le32s ((uint32_t *) & header->num_prefetch_slots);
 383     cpu_to_le32s ((uint32_t *) & header->bytes_per_prefetch);
 384     cpu_to_le32s ((uint32_t *) & header->prefetch_throttle_time);
 385     cpu_to_le32s ((uint32_t *) & header->prefetch_read_throughput_measure_time);
 386     cpu_to_le32s ((uint32_t *) &header->prefetch_write_throughput_measure_time);
 387     cpu_to_le32s ((uint32_t *) & header->prefetch_perf_calc_alpha);
 388     cpu_to_le32s ((uint32_t *) & header->prefetch_min_read_throughput);
 389     cpu_to_le32s ((uint32_t *) & header->prefetch_min_write_throughput);
 390     cpu_to_le32s ((uint32_t *) & header->prefetch_max_read_throughput);
 391     cpu_to_le32s ((uint32_t *) & header->prefetch_max_write_throughput);
 392     cpu_to_le32s ((uint32_t *) & header->block_size);
 393     cpu_to_le32s ((uint32_t *) & header->unit_of_PrefetchProfileEntry_len);
 394     cpu_to_le32s ((uint32_t *) & header->compact_image);
 395     cpu_to_le64s ((uint64_t *) & header->chunk_size);
 396     cpu_to_le64s ((uint64_t *) & header->storage_grow_unit);
 397     cpu_to_le64s ((uint64_t *) & header->table_offset);
 398     cpu_to_le32s ((uint32_t *) & header->clean_shutdown);
 399     cpu_to_le64s ((uint64_t *) & header->journal_offset);
 400     cpu_to_le64s ((uint64_t *) & header->journal_size);
 401 }
 402
 403 static void fvd_header_le_to_cpu (FvdHeader * header)
 404 {
 405     le32_to_cpus (&header->magic);
 406     le32_to_cpus (&header->version);
 407     le32_to_cpus ((uint32_t *) & header->all_data_in_fvd_img);
 408     le32_to_cpus ((uint32_t *) & header->generate_prefetch_profile);
 409     le64_to_cpus ((uint64_t *) & header->metadata_size);
 410     le64_to_cpus ((uint64_t *) & header->virtual_disk_size);
 411     le64_to_cpus ((uint64_t *) & header->base_img_size);
 412     le64_to_cpus ((uint64_t *) & header->max_outstanding_copy_on_read_data);
 413     le64_to_cpus ((uint64_t *) & header->bitmap_offset);
 414     le64_to_cpus ((uint64_t *) & header->prefetch_profile_offset);
 415     le64_to_cpus ((uint64_t *) & header->prefetch_profile_entries);
 416     le64_to_cpus ((uint64_t *) & header->bitmap_size);
 417     le32_to_cpus ((uint32_t *) & header->copy_on_read);
 418     le32_to_cpus ((uint32_t *) & header->need_zero_init);
 419     le32_to_cpus ((uint32_t *) & header->prefetch_start_delay);
 420     le32_to_cpus ((uint32_t *) & header->profile_directed_prefetch_start_delay);
 421     le32_to_cpus ((uint32_t *) & header->num_prefetch_slots);
 422     le32_to_cpus ((uint32_t *) & header->bytes_per_prefetch);
 423     le32_to_cpus ((uint32_t *) & header->prefetch_throttle_time);
 424     le32_to_cpus ((uint32_t *) & header->prefetch_read_throughput_measure_time);
 425     le32_to_cpus ((uint32_t *) &header->prefetch_write_throughput_measure_time);
 426     le32_to_cpus ((uint32_t *) & header->prefetch_perf_calc_alpha);
 427     le32_to_cpus ((uint32_t *) & header->prefetch_min_read_throughput);
 428     le32_to_cpus ((uint32_t *) & header->prefetch_min_write_throughput);
 429     le32_to_cpus ((uint32_t *) & header->prefetch_max_read_throughput);
 430     le32_to_cpus ((uint32_t *) & header->prefetch_max_write_throughput);
 431     le32_to_cpus ((uint32_t *) & header->block_size);
 432     le32_to_cpus ((uint32_t *) & header->unit_of_PrefetchProfileEntry_len);
 433     le32_to_cpus ((uint32_t *) & header->compact_image);
 434     le64_to_cpus ((uint64_t *) & header->chunk_size);
 435     le64_to_cpus ((uint64_t *) & header->storage_grow_unit);
 436     le64_to_cpus ((uint64_t *) & header->table_offset);
 437     le32_to_cpus ((uint32_t *) & header->clean_shutdown);
 438     le64_to_cpus ((uint64_t *) & header->journal_offset);
 439     le64_to_cpus ((uint64_t *) & header->journal_size);
 440 }
 441
 442 static void flush_metadata_to_disk (BlockDriverState * bs)
 443 {
 444     BDRVFvdState *s = bs->opaque;
 445
 446     if (bs->read_only || !s->fvd_metadata) {
 447         return;
 448     }
 449
 450     if (s->stale_bitmap) {
 451         /* Flush fresh_bitmap to disk. */
 452         int nb = (int) (s->bitmap_size / 512);
 453         QDEBUG ("Flush FVD bitmap (%d sectors) to disk\n", nb);
 454         bdrv_write (s->fvd_metadata, s->bitmap_offset, s->fresh_bitmap, nb);
 455     }
 456
 457     if (s->table) {
 458         /* Flush table to disk. */
 459         int table_entries =
 460             (int) (ROUND_UP (s->virtual_disk_size, s->chunk_size * 512) /
 461                    (s->chunk_size * 512));
 462
 463         /* Clean the DIRTY_TABLE bit. */
 464         int i;
 465         for (i = 0; i < table_entries; i++) {
 466             CLEAN_DIRTY (s->table[i]);
 467         }
 468
 469         int64_t table_size = sizeof (uint32_t) * table_entries;
 470         table_size = ROUND_UP (table_size, DEF_PAGE_SIZE);
 471         int nb = (int) (table_size / 512);
 472         QDEBUG ("Flush FVD table (%d sectors) to disk\n", nb);
 473         bdrv_write (s->fvd_metadata, s->table_offset, (uint8_t *) s->table, nb);
 474     }
 475 }
 476
 477 static int read_fvd_header (BDRVFvdState * s, FvdHeader * header)
 478 {
 479     if (bdrv_pread (s->fvd_metadata, 0, header, sizeof (FvdHeader)) !=
 480         sizeof (FvdHeader)) {
 481         fprintf (stderr, "Failed to read the FVD header.\n");
 482         return -1;
 483     }
 484
 485     fvd_header_le_to_cpu (header);
 486
 487     if (header->magic != FVD_MAGIC || header->version != FVD_VERSION) {
 488         fprintf (stderr, "Error: image does not have the correct FVD format "
 489                  "magic number in header\n");
 490         return -1;
 491     }
 492
 493     return 0;
 494 }
 495
 496 static int update_fvd_header (BDRVFvdState * s, FvdHeader * header)
 497 {
 498     fvd_header_cpu_to_le (header);
 499     int ret = bdrv_pwrite (s->fvd_metadata, 0, header, sizeof (FvdHeader));
 500
 501     if (ret != sizeof (FvdHeader)) {
 502         fprintf (stderr, "Failed to update the FVD header.\n");
 503         ASSERT (FALSE);
 504         return -EIO;
 505     }
 506
 507     return 0;
 508 }
 509
 510 static void null_prefetch_cb (void *opaque, int ret)
 511 {
 512     /* Nothing to do and will never be invoked. Only need it to distinguish
 513      * copy-on-read from prefetch. */
 514     ASSERT (FALSE);
 515 }
 516
 517 static int count_iov (struct iovec *orig_iov, int *p_index, uint8_t ** p_buf,
 518                       size_t * p_left, size_t total)
 519 {
 520     int index = *p_index;
 521     uint8_t *buf = *p_buf;
 522     int left = *p_left;
 523     int count = 0;
 524
 525     if (left <= 0) {
 526         index++;
 527         buf = orig_iov[index].iov_base;
 528         left = orig_iov[index].iov_len;
 529     }
 530
 531     while (1) {
 532         if (left >= total) {
 533             *p_buf = buf + total;
 534             *p_left = left - total;
 535             *p_index = index;
 536             return count + 1;
 537         }
 538
 539         total -= left;
 540         index++;
 541         buf = orig_iov[index].iov_base;
 542         left = orig_iov[index].iov_len;
 543         count++;
 544     }
 545 }
 546
 547 static int setup_iov (struct iovec *orig_iov, struct iovec *new_iov,
 548                       int *p_index, uint8_t ** p_buf, size_t * p_left,
 549                       size_t total)
 550 {
 551     int index = *p_index;
 552     uint8_t *buf = *p_buf;
 553     int left = *p_left;
 554     int count = 0;
 555
 556     if (left <= 0) {
 557         index++;
 558         buf = orig_iov[index].iov_base;
 559         left = orig_iov[index].iov_len;
 560     }
 561
 562     while (1) {
 563         if (left >= total) {
 564             new_iov[count].iov_base = buf;
 565             new_iov[count].iov_len = total;
 566             *p_buf = buf + total;
 567             *p_left = left - total;
 568             *p_index = index;
 569             return count + 1;
 570         }
 571
 572         new_iov[count].iov_base = buf;
 573         new_iov[count].iov_len = left;
 574         total -= left;
 575         index++;
 576         buf = orig_iov[index].iov_base;
 577         left = orig_iov[index].iov_len;
 578         count++;
 579     }
 580 }
 581
 582 static int zero_iov (struct iovec *orig_iov, int *p_index, uint8_t ** p_buf,
 583                      size_t * p_left, size_t total)
 584 {
 585     int index = *p_index;
 586     uint8_t *buf = *p_buf;
 587     int left = *p_left;
 588     int count = 0;
 589
 590     if (left <= 0) {
 591         index++;
 592         buf = orig_iov[index].iov_base;
 593         left = orig_iov[index].iov_len;
 594     }
 595
 596     while (1) {
 597         if (left >= total) {
 598             memset (buf, 0, total);
 599             *p_buf = buf + total;
 600             *p_left = left - total;
 601             *p_index = index;
 602             return count + 1;
 603         }
 604
 605         memset (buf, 0, left);
 606         total -= left;
 607         index++;
 608         buf = orig_iov[index].iov_base;
 609         left = orig_iov[index].iov_len;
 610         count++;
 611     }
 612 }