fs/direct-io.c

   1 /*
   2  * fs/direct-io.c
   3  *
   4  * Copyright (C) 2002, Linus Torvalds.
   5  *
   6  * O_DIRECT
   7  *
   8  * 04Jul2002    akpm@zip.com.au
   9  *              Initial version
  10  * 11Sep2002    janetinc@us.ibm.com
  11  *              added readv/writev support.
  12  * 29Oct2002    akpm@zip.com.au
  13  *              rewrote bio_add_page() support.
  14  * 30Oct2002    pbadari@us.ibm.com
  15  *              added support for non-aligned IO.
  16  * 06Nov2002    pbadari@us.ibm.com
  17  *              added asynchronous IO support.
  18  */
  19
  20 #include <linux/kernel.h>
  21 #include <linux/types.h>
  22 #include <linux/fs.h>
  23 #include <linux/mm.h>
  24 #include <linux/slab.h>
  25 #include <linux/highmem.h>
  26 #include <linux/pagemap.h>
  27 #include <linux/bio.h>
  28 #include <linux/wait.h>
  29 #include <linux/err.h>
  30 #include <linux/blkdev.h>
  31 #include <linux/buffer_head.h>
  32 #include <linux/rwsem.h>
  33 #include <linux/uio.h>
  34 #include <asm/atomic.h>
  35
  36 /*
  37  * How many user pages to map in one call to get_user_pages().  This determines
  38  * the size of a structure on the stack.
  39  */
  40 #define DIO_PAGES       64
  41
  42 /*
  43  * This code generally works in units of "dio_blocks".  A dio_block is
  44  * somewhere between the hard sector size and the filesystem block size.  it
  45  * is determined on a per-invocation basis.   When talking to the filesystem
  46  * we need to convert dio_blocks to fs_blocks by scaling the dio_block quantity
  47  * down by dio->blkfactor.  Similarly, fs-blocksize quantities are converted
  48  * to bio_block quantities by shifting left by blkfactor.
  49  *
  50  * If blkfactor is zero then the user's request was aligned to the filesystem's
  51  * blocksize.
  52  */
  53
  54 struct dio {
  55         /* BIO submission state */
  56         struct bio *bio;                /* bio under assembly */
  57         struct inode *inode;
  58         int rw;
  59         unsigned blkbits;               /* doesn't change */
  60         unsigned blkfactor;             /* When we're using an alignment which
  61                                            is finer than the filesystem's soft
  62                                            blocksize, this specifies how much
  63                                            finer.  blkfactor=2 means 1/4-block
  64                                            alignment.  Does not change */
  65         unsigned start_zero_done;       /* flag: sub-blocksize zeroing has
  66                                            been performed at the start of a
  67                                            write */
  68         int pages_in_io;                /* approximate total IO pages */
  69         sector_t block_in_file;         /* Current offset into the underlying
  70                                            file in dio_block units. */
  71         unsigned blocks_available;      /* At block_in_file.  changes */
  72         sector_t final_block_in_request;/* doesn't change */
  73         unsigned first_block_in_page;   /* doesn't change, Used only once */
  74         int boundary;                   /* prev block is at a boundary */
  75         int reap_counter;               /* rate limit reaping */
  76         get_blocks_t *get_blocks;       /* block mapping function */
  77         sector_t final_block_in_bio;    /* current final block in bio + 1 */
  78         sector_t next_block_for_io;     /* next block to be put under IO,
  79                                            in dio_blocks units */
  80         struct buffer_head map_bh;      /* last get_blocks() result */
  81
  82         /*
  83          * Deferred addition of a page to the dio.  These variables are
  84          * private to dio_send_cur_page(), submit_page_section() and
  85          * dio_bio_add_page().
  86          */
  87         struct page *cur_page;          /* The page */
  88         unsigned cur_page_offset;       /* Offset into it, in bytes */
  89         unsigned cur_page_len;          /* Nr of bytes at cur_page_offset */
  90         sector_t cur_page_block;        /* Where it starts */
  91
  92         /*
  93          * Page fetching state. These variables belong to dio_refill_pages().
  94          */
  95         int curr_page;                  /* changes */
  96         int total_pages;                /* doesn't change */
  97         unsigned long curr_user_address;/* changes */
  98
  99         /*
 100          * Page queue.  These variables belong to dio_refill_pages() and
 101          * dio_get_page().
 102          */
 103         struct page *pages[DIO_PAGES];  /* page buffer */
 104         unsigned head;                  /* next page to process */
 105         unsigned tail;                  /* last valid page + 1 */
 106         int page_errors;                /* errno from get_user_pages() */
 107
 108         /* BIO completion state */
 109         atomic_t bio_count;             /* nr bios to be completed */
 110         atomic_t bios_in_flight;        /* nr bios in flight */
 111         spinlock_t bio_list_lock;       /* protects bio_list */
 112         struct bio *bio_list;           /* singly linked via bi_private */
 113         struct task_struct *waiter;     /* waiting task (NULL if none) */
 114
 115         /* AIO related stuff */
 116         struct kiocb *iocb;             /* kiocb */
 117         int is_async;                   /* is IO async ? */
 118         int result;                     /* IO result */
 119 };
 120
 121 /*
 122  * How many pages are in the queue?
 123  */
 124 static inline unsigned dio_pages_present(struct dio *dio)
 125 {
 126         return dio->tail - dio->head;
 127 }
 128
 129 /*
 130  * Go grab and pin some userspace pages.   Typically we'll get 64 at a time.
 131  */
 132 static int dio_refill_pages(struct dio *dio)
 133 {
 134         int ret;
 135         int nr_pages;
 136
 137         nr_pages = min(dio->total_pages - dio->curr_page, DIO_PAGES);
 138         down_read(&current->mm->mmap_sem);
 139         ret = get_user_pages(
 140                 current,                        /* Task for fault acounting */
 141                 current->mm,                    /* whose pages? */
 142                 dio->curr_user_address,         /* Where from? */
 143                 nr_pages,                       /* How many pages? */
 144                 dio->rw == READ,                /* Write to memory? */
 145                 0,                              /* force (?) */
 146                 &dio->pages[0],
 147                 NULL);                          /* vmas */
 148         up_read(&current->mm->mmap_sem);
 149
 150         if (ret < 0 && dio->blocks_available && (dio->rw == WRITE)) {
 151                 /*
 152                  * A memory fault, but the filesystem has some outstanding
 153                  * mapped blocks.  We need to use those blocks up to avoid
 154                  * leaking stale data in the file.
 155                  */
 156                 if (dio->page_errors == 0)
 157                         dio->page_errors = ret;
 158                 dio->pages[0] = ZERO_PAGE(dio->curr_user_address);
 159                 dio->head = 0;
 160                 dio->tail = 1;
 161                 ret = 0;
 162                 goto out;
 163         }
 164
 165         if (ret >= 0) {
 166                 dio->curr_user_address += ret * PAGE_SIZE;
 167                 dio->curr_page += ret;
 168                 dio->head = 0;
 169                 dio->tail = ret;
 170                 ret = 0;
 171         }
 172 out:
 173         return ret;
 174 }
 175
 176 /*
 177  * Get another userspace page.  Returns an ERR_PTR on error.  Pages are
 178  * buffered inside the dio so that we can call get_user_pages() against a
 179  * decent number of pages, less frequently.  To provide nicer use of the
 180  * L1 cache.
 181  */
 182 static struct page *dio_get_page(struct dio *dio)
 183 {
 184         if (dio_pages_present(dio) == 0) {
 185                 int ret;
 186
 187                 ret = dio_refill_pages(dio);
 188                 if (ret)
 189                         return ERR_PTR(ret);
 190                 BUG_ON(dio_pages_present(dio) == 0);
 191         }
 192         return dio->pages[dio->head++];
 193 }
 194
 195 /*
 196  * Called when a BIO has been processed.  If the count goes to zero then IO is
 197  * complete and we can signal this to the AIO layer.
 198  */
 199 static void finished_one_bio(struct dio *dio)
 200 {
 201         if (atomic_dec_and_test(&dio->bio_count)) {
 202                 if(dio->is_async) {
 203                         aio_complete(dio->iocb, dio->result, 0);
 204                         kfree(dio);
 205                 }
 206         }
 207 }
 208
 209 static int dio_bio_complete(struct dio *dio, struct bio *bio);
 210 /*
 211  * Asynchronous IO callback.
 212  */
 213 static int dio_bio_end_aio(struct bio *bio, unsigned int bytes_done, int error)
 214 {
 215         struct dio *dio = bio->bi_private;
 216
 217         if (bio->bi_size)
 218                 return 1;
 219
 220         /* cleanup the bio */
 221         dio_bio_complete(dio, bio);
 222         return 0;
 223 }
 224
 225 /*
 226  * The BIO completion handler simply queues the BIO up for the process-context
 227  * handler.
 228  *
 229  * During I/O bi_private points at the dio.  After I/O, bi_private is used to
 230  * implement a singly-linked list of completed BIOs, at dio->bio_list.
 231  */
 232 static int dio_bio_end_io(struct bio *bio, unsigned int bytes_done, int error)
 233 {
 234         struct dio *dio = bio->bi_private;
 235         unsigned long flags;
 236
 237         if (bio->bi_size)
 238                 return 1;
 239
 240         spin_lock_irqsave(&dio->bio_list_lock, flags);
 241         bio->bi_private = dio->bio_list;
 242         dio->bio_list = bio;
 243         atomic_dec(&dio->bios_in_flight);
 244         if (dio->waiter && atomic_read(&dio->bios_in_flight) == 0)
 245                 wake_up_process(dio->waiter);
 246         spin_unlock_irqrestore(&dio->bio_list_lock, flags);
 247         return 0;
 248 }
 249
 250 static int
 251 dio_bio_alloc(struct dio *dio, struct block_device *bdev,
 252                 sector_t first_sector, int nr_vecs)
 253 {
 254         struct bio *bio;
 255
 256         bio = bio_alloc(GFP_KERNEL, nr_vecs);
 257         if (bio == NULL)
 258                 return -ENOMEM;
 259
 260         bio->bi_bdev = bdev;
 261         bio->bi_sector = first_sector;
 262         if (dio->is_async)
 263                 bio->bi_end_io = dio_bio_end_aio;
 264         else
 265                 bio->bi_end_io = dio_bio_end_io;
 266
 267         dio->bio = bio;
 268         return 0;
 269 }
 270
 271 /*
 272  * In the AIO read case we speculatively dirty the pages before starting IO.
 273  * During IO completion, any of these pages which happen to have been written
 274  * back will be redirtied by bio_check_pages_dirty().
 275  */
 276 static void dio_bio_submit(struct dio *dio)
 277 {
 278         struct bio *bio = dio->bio;
 279
 280         bio->bi_private = dio;
 281         atomic_inc(&dio->bio_count);
 282         atomic_inc(&dio->bios_in_flight);
 283         if (dio->is_async && dio->rw == READ)
 284                 bio_set_pages_dirty(bio);
 285         submit_bio(dio->rw, bio);
 286
 287         dio->bio = NULL;
 288         dio->boundary = 0;
 289 }
 290
 291 /*
 292  * Release any resources in case of a failure
 293  */
 294 static void dio_cleanup(struct dio *dio)
 295 {
 296         while (dio_pages_present(dio))
 297                 page_cache_release(dio_get_page(dio));
 298 }
 299
 300 /*
 301  * Wait for the next BIO to complete.  Remove it and return it.
 302  */
 303 static struct bio *dio_await_one(struct dio *dio)
 304 {
 305         unsigned long flags;
 306         struct bio *bio;
 307
 308         spin_lock_irqsave(&dio->bio_list_lock, flags);
 309         while (dio->bio_list == NULL) {
 310                 set_current_state(TASK_UNINTERRUPTIBLE);
 311                 if (dio->bio_list == NULL) {
 312                         dio->waiter = current;
 313                         spin_unlock_irqrestore(&dio->bio_list_lock, flags);
 314                         blk_run_queues();
 315                         io_schedule();
 316                         spin_lock_irqsave(&dio->bio_list_lock, flags);
 317                         dio->waiter = NULL;
 318                 }
 319                 set_current_state(TASK_RUNNING);
 320         }
 321         bio = dio->bio_list;
 322         dio->bio_list = bio->bi_private;
 323         spin_unlock_irqrestore(&dio->bio_list_lock, flags);
 324         return bio;
 325 }
 326
 327 /*
 328  * Process one completed BIO.  No locks are held.
 329  */
 330 static int dio_bio_complete(struct dio *dio, struct bio *bio)
 331 {
 332         const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 333         struct bio_vec *bvec = bio->bi_io_vec;
 334         int page_no;
 335
 336         if (!uptodate)
 337                 dio->result = -EIO;
 338
 339         if (dio->is_async && dio->rw == READ) {
 340                 bio_check_pages_dirty(bio);     /* transfers ownership */
 341         } else {
 342                 for (page_no = 0; page_no < bio->bi_vcnt; page_no++) {
 343                         struct page *page = bvec[page_no].bv_page;
 344
 345                         if (dio->rw == READ)
 346                                 set_page_dirty_lock(page);
 347                         page_cache_release(page);
 348                 }
 349                 bio_put(bio);
 350         }
 351         finished_one_bio(dio);
 352         return uptodate ? 0 : -EIO;
 353 }
 354
 355 /*
 356  * Wait on and process all in-flight BIOs.
 357  */
 358 static int dio_await_completion(struct dio *dio)
 359 {
 360         int ret = 0;
 361
 362         if (dio->bio)
 363                 dio_bio_submit(dio);
 364
 365         while (atomic_read(&dio->bio_count)) {
 366                 struct bio *bio = dio_await_one(dio);
 367                 int ret2;
 368
 369                 ret2 = dio_bio_complete(dio, bio);
 370                 if (ret == 0)
 371                         ret = ret2;
 372         }
 373         return ret;
 374 }
 375
 376 /*
 377  * A really large O_DIRECT read or write can generate a lot of BIOs.  So
 378  * to keep the memory consumption sane we periodically reap any completed BIOs
 379  * during the BIO generation phase.
 380  *
 381  * This also helps to limit the peak amount of pinned userspace memory.
 382  */
 383 static int dio_bio_reap(struct dio *dio)
 384 {
 385         int ret = 0;
 386
 387         if (dio->reap_counter++ >= 64) {
 388                 while (dio->bio_list) {
 389                         unsigned long flags;
 390                         struct bio *bio;
 391
 392                         spin_lock_irqsave(&dio->bio_list_lock, flags);
 393                         bio = dio->bio_list;
 394                         dio->bio_list = bio->bi_private;
 395                         spin_unlock_irqrestore(&dio->bio_list_lock, flags);
 396                         ret = dio_bio_complete(dio, bio);
 397                 }
 398                 dio->reap_counter = 0;
 399         }
 400         return ret;
 401 }
 402
 403 /*
 404  * Call into the fs to map some more disk blocks.  We record the current number
 405  * of available blocks at dio->blocks_available.  These are in units of the
 406  * fs blocksize, (1 << inode->i_blkbits).
 407  *
 408  * The fs is allowed to map lots of blocks at once.  If it wants to do that,
 409  * it uses the passed inode-relative block number as the file offset, as usual.
 410  *
 411  * get_blocks() is passed the number of i_blkbits-sized blocks which direct_io
 412  * has remaining to do.  The fs should not map more than this number of blocks.
 413  *
 414  * If the fs has mapped a lot of blocks, it should populate bh->b_size to
 415  * indicate how much contiguous disk space has been made available at
 416  * bh->b_blocknr.
 417  *
 418  * If *any* of the mapped blocks are new, then the fs must set buffer_new().
 419  * This isn't very efficient...
 420  *
 421  * In the case of filesystem holes: the fs may return an arbitrarily-large
 422  * hole by returning an appropriate value in b_size and by clearing
 423  * buffer_mapped().  However the direct-io code will only process holes one
 424  * block at a time - it will repeatedly call get_blocks() as it walks the hole.
 425  */
 426 static int get_more_blocks(struct dio *dio)
 427 {
 428         int ret;
 429         struct buffer_head *map_bh = &dio->map_bh;
 430         sector_t fs_startblk;   /* Into file, in filesystem-sized blocks */
 431         unsigned long fs_count; /* Number of filesystem-sized blocks */
 432         unsigned long dio_count;/* Number of dio_block-sized blocks */
 433         unsigned long blkmask;
 434
 435         /*
 436          * If there was a memory error and we've overwritten all the
 437          * mapped blocks then we can now return that memory error
 438          */
 439         ret = dio->page_errors;
 440         if (ret == 0) {
 441                 map_bh->b_state = 0;
 442                 map_bh->b_size = 0;
 443                 BUG_ON(dio->block_in_file >= dio->final_block_in_request);
 444                 fs_startblk = dio->block_in_file >> dio->blkfactor;
 445                 dio_count = dio->final_block_in_request - dio->block_in_file;
 446                 fs_count = dio_count >> dio->blkfactor;
 447                 blkmask = (1 << dio->blkfactor) - 1;
 448                 if (dio_count & blkmask)
 449                         fs_count++;
 450
 451                 ret = (*dio->get_blocks)(dio->inode, fs_startblk, fs_count,
 452                                 map_bh, dio->rw == WRITE);
 453         }
 454         return ret;
 455 }
 456
 457 /*
 458  * There is no bio.  Make one now.
 459  */
 460 static int dio_new_bio(struct dio *dio, sector_t start_sector)
 461 {
 462         sector_t sector;
 463         int ret, nr_pages;
 464
 465         ret = dio_bio_reap(dio);
 466         if (ret)
 467                 goto out;
 468         sector = start_sector << (dio->blkbits - 9);
 469         nr_pages = min(dio->pages_in_io, bio_get_nr_vecs(dio->map_bh.b_bdev));
 470         BUG_ON(nr_pages <= 0);
 471         ret = dio_bio_alloc(dio, dio->map_bh.b_bdev, sector, nr_pages);
 472         dio->boundary = 0;
 473 out:
 474         return ret;
 475 }
 476
 477 /*
 478  * Attempt to put the current chunk of 'cur_page' into the current BIO.  If
 479  * that was successful then update final_block_in_bio and take a ref against
 480  * the just-added page.
 481  *
 482  * Return zero on success.  Non-zero means the caller needs to start a new BIO.
 483  */
 484 static int dio_bio_add_page(struct dio *dio)
 485 {
 486         int ret;
 487
 488         ret = bio_add_page(dio->bio, dio->cur_page,
 489                         dio->cur_page_len, dio->cur_page_offset);
 490         if (ret == dio->cur_page_len) {
 491                 dio->pages_in_io--;
 492                 page_cache_get(dio->cur_page);
 493                 dio->final_block_in_bio = dio->cur_page_block +
 494                         (dio->cur_page_len >> dio->blkbits);
 495                 ret = 0;
 496         } else {
 497                 ret = 1;
 498         }
 499         return ret;
 500 }
 501
 502 /*
 503  * Put cur_page under IO.  The section of cur_page which is described by
 504  * cur_page_offset,cur_page_len is put into a BIO.  The section of cur_page
 505  * starts on-disk at cur_page_block.
 506  *
 507  * We take a ref against the page here (on behalf of its presence in the bio).
 508  *
 509  * The caller of this function is responsible for removing cur_page from the
 510  * dio, and for dropping the refcount which came from that presence.
 511  */
 512 static int dio_send_cur_page(struct dio *dio)
 513 {
 514         int ret = 0;
 515
 516         if (dio->bio) {
 517                 /*
 518                  * See whether this new request is contiguous with the old
 519                  */
 520                 if (dio->final_block_in_bio != dio->cur_page_block)
 521                         dio_bio_submit(dio);
 522                 /*
 523                  * Submit now if the underlying fs is about to perform a
 524                  * metadata read
 525                  */
 526                 if (dio->boundary)
 527                         dio_bio_submit(dio);
 528         }
 529
 530         if (dio->bio == NULL) {
 531                 ret = dio_new_bio(dio, dio->cur_page_block);
 532                 if (ret)
 533                         goto out;
 534         }
 535
 536         if (dio_bio_add_page(dio) != 0) {
 537                 dio_bio_submit(dio);
 538                 ret = dio_new_bio(dio, dio->cur_page_block);
 539                 if (ret == 0) {
 540                         ret = dio_bio_add_page(dio);
 541                         BUG_ON(ret != 0);
 542                 }
 543         }
 544 out:
 545         return ret;
 546 }
 547
 548 /*
 549  * An autonomous function to put a chunk of a page under deferred IO.
 550  *
 551  * The caller doesn't actually know (or care) whether this piece of page is in
 552  * a BIO, or is under IO or whatever.  We just take care of all possible
 553  * situations here.  The separation between the logic of do_direct_IO() and
 554  * that of submit_page_section() is important for clarity.  Please don't break.
 555  *
 556  * The chunk of page starts on-disk at blocknr.
 557  *
 558  * We perform deferred IO, by recording the last-submitted page inside our
 559  * private part of the dio structure.  If possible, we just expand the IO
 560  * across that page here.
 561  *
 562  * If that doesn't work out then we put the old page into the bio and add this
 563  * page to the dio instead.
 564  */
 565 static int
 566 submit_page_section(struct dio *dio, struct page *page,
 567                 unsigned offset, unsigned len, sector_t blocknr)
 568 {
 569         int ret = 0;
 570
 571         /*
 572          * Can we just grow the current page's presence in the dio?
 573          */
 574         if (    (dio->cur_page == page) &&
 575                 (dio->cur_page_offset + dio->cur_page_len == offset) &&
 576                 (dio->cur_page_block +
 577                         (dio->cur_page_len >> dio->blkbits) == blocknr)) {
 578                 dio->cur_page_len += len;
 579
 580                 /*
 581                  * If dio->boundary then we want to schedule the IO now to
 582                  * avoid metadata seeks.
 583                  */
 584                 if (dio->boundary) {
 585                         ret = dio_send_cur_page(dio);
 586                         page_cache_release(dio->cur_page);
 587                         dio->cur_page = NULL;
 588                 }
 589                 goto out;
 590         }
 591
 592         /*
 593          * If there's a deferred page already there then send it.
 594          */
 595         if (dio->cur_page) {
 596                 ret = dio_send_cur_page(dio);
 597                 page_cache_release(dio->cur_page);
 598                 dio->cur_page = NULL;
 599                 if (ret)
 600                         goto out;
 601         }
 602
 603         page_cache_get(page);           /* It is in dio */
 604         dio->cur_page = page;
 605         dio->cur_page_offset = offset;
 606         dio->cur_page_len = len;
 607         dio->cur_page_block = blocknr;
 608 out:
 609         return ret;
 610 }
 611
 612 /*
 613  * Clean any dirty buffers in the blockdev mapping which alias newly-created
 614  * file blocks.  Only called for S_ISREG files - blockdevs do not set
 615  * buffer_new
 616  */
 617 static void clean_blockdev_aliases(struct dio *dio)
 618 {
 619         unsigned i;
 620
 621         for (i = 0; i < dio->blocks_available; i++) {
 622                 unmap_underlying_metadata(dio->map_bh.b_bdev,
 623                                         dio->map_bh.b_blocknr + i);
 624         }
 625 }
 626
 627 /*
 628  * If we are not writing the entire block and get_block() allocated
 629  * the block for us, we need to fill-in the unused portion of the
 630  * block with zeros. This happens only if user-buffer, fileoffset or
 631  * io length is not filesystem block-size multiple.
 632  *
 633  * `end' is zero if we're doing the start of the IO, 1 at the end of the
 634  * IO.
 635  */
 636 static void dio_zero_block(struct dio *dio, int end)
 637 {
 638         unsigned dio_blocks_per_fs_block;
 639         unsigned this_chunk_blocks;     /* In dio_blocks */
 640         unsigned this_chunk_bytes;
 641         struct page *page;
 642
 643         dio->start_zero_done = 1;
 644         if (!dio->blkfactor || !buffer_new(&dio->map_bh))
 645                 return;
 646
 647         dio_blocks_per_fs_block = 1 << dio->blkfactor;
 648         this_chunk_blocks = dio->block_in_file & (dio_blocks_per_fs_block - 1);
 649
 650         if (!this_chunk_blocks)
 651                 return;
 652
 653         /*
 654          * We need to zero out part of an fs block.  It is either at the
 655          * beginning or the end of the fs block.
 656          */
 657         if (end)
 658                 this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks;
 659
 660         this_chunk_bytes = this_chunk_blocks << dio->blkbits;
 661
 662         page = ZERO_PAGE(dio->cur_user_address);
 663         if (submit_page_section(dio, page, 0, this_chunk_bytes,
 664                                 dio->next_block_for_io))
 665                 return;
 666
 667         dio->next_block_for_io += this_chunk_blocks;
 668 }
 669
 670 /*
 671  * Walk the user pages, and the file, mapping blocks to disk and generating
 672  * a sequence of (page,offset,len,block) mappings.  These mappings are injected
 673  * into submit_page_section(), which takes care of the next stage of submission
 674  *
 675  * Direct IO against a blockdev is different from a file.  Because we can
 676  * happily perform page-sized but 512-byte aligned IOs.  It is important that
 677  * blockdev IO be able to have fine alignment and large sizes.
 678  *
 679  * So what we do is to permit the ->get_blocks function to populate bh.b_size
 680  * with the size of IO which is permitted at this offset and this i_blkbits.
 681  *
 682  * For best results, the blockdev should be set up with 512-byte i_blkbits and
 683  * it should set b_size to PAGE_SIZE or more inside get_blocks().  This gives
 684  * fine alignment but still allows this function to work in PAGE_SIZE units.
 685  */
 686 static int do_direct_IO(struct dio *dio)
 687 {
 688         const unsigned blkbits = dio->blkbits;
 689         const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
 690         struct page *page;
 691         unsigned block_in_page;
 692         struct buffer_head *map_bh = &dio->map_bh;
 693         int ret = 0;
 694
 695         /* The I/O can start at any block offset within the first page */
 696         block_in_page = dio->first_block_in_page;
 697
 698         while (dio->block_in_file < dio->final_block_in_request) {
 699                 page = dio_get_page(dio);
 700                 if (IS_ERR(page)) {
 701                         ret = PTR_ERR(page);
 702                         goto out;
 703                 }
 704
 705                 while (block_in_page < blocks_per_page) {
 706                         unsigned offset_in_page = block_in_page << blkbits;
 707                         unsigned this_chunk_bytes;      /* # of bytes mapped */
 708                         unsigned this_chunk_blocks;     /* # of blocks */
 709                         unsigned u;
 710
 711                         if (dio->blocks_available == 0) {
 712                                 /*
 713                                  * Need to go and map some more disk
 714                                  */
 715                                 unsigned long blkmask;
 716                                 unsigned long dio_remainder;
 717
 718                                 ret = get_more_blocks(dio);
 719                                 if (ret) {
 720                                         page_cache_release(page);
 721                                         goto out;
 722                                 }
 723                                 if (!buffer_mapped(map_bh))
 724                                         goto do_holes;
 725
 726                                 dio->blocks_available =
 727                                                 map_bh->b_size >> dio->blkbits;
 728                                 dio->next_block_for_io =
 729                                         map_bh->b_blocknr << dio->blkfactor;
 730                                 if (buffer_new(map_bh))
 731                                         clean_blockdev_aliases(dio);
 732
 733                                 if (!dio->blkfactor)
 734                                         goto do_holes;
 735
 736                                 blkmask = (1 << dio->blkfactor) - 1;
 737                                 dio_remainder = (dio->block_in_file & blkmask);
 738
 739                                 /*
 740                                  * If we are at the start of IO and that IO
 741                                  * starts partway into a fs-block,
 742                                  * dio_remainder will be non-zero.  If the IO
 743                                  * is a read then we can simply advance the IO
 744                                  * cursor to the first block which is to be
 745                                  * read.  But if the IO is a write and the
 746                                  * block was newly allocated we cannot do that;
 747                                  * the start of the fs block must be zeroed out
 748                                  * on-disk
 749                                  */
 750                                 if (!buffer_new(map_bh))
 751                                         dio->next_block_for_io += dio_remainder;
 752                                 dio->blocks_available -= dio_remainder;
 753                         }
 754 do_holes:
 755                         /* Handle holes */
 756                         if (!buffer_mapped(map_bh)) {
 757                                 char *kaddr;
 758
 759                                 if (dio->block_in_file >=
 760                                         i_size_read(dio->inode)>>blkbits) {
 761                                         /* We hit eof */
 762                                         page_cache_release(page);
 763                                         goto out;
 764                                 }
 765                                 kaddr = kmap_atomic(page, KM_USER0);
 766                                 memset(kaddr + (block_in_page << blkbits),
 767                                                 0, 1 << blkbits);
 768                                 flush_dcache_page(page);
 769                                 kunmap_atomic(kaddr, KM_USER0);
 770                                 dio->block_in_file++;
 771                                 block_in_page++;
 772                                 goto next_block;
 773                         }
 774
 775                         /*
 776                          * If we're performing IO which has an alignment which
 777                          * is finer than the underlying fs, go check to see if
 778                          * we must zero out the start of this block.
 779                          */
 780                         if (unlikely(dio->blkfactor && !dio->start_zero_done))
 781                                 dio_zero_block(dio, 0);
 782
 783                         /*
 784                          * Work out, in this_chunk_blocks, how much disk we
 785                          * can add to this page
 786                          */
 787                         this_chunk_blocks = dio->blocks_available;
 788                         u = (PAGE_SIZE - offset_in_page) >> blkbits;
 789                         if (this_chunk_blocks > u)
 790                                 this_chunk_blocks = u;
 791                         u = dio->final_block_in_request - dio->block_in_file;
 792                         if (this_chunk_blocks > u)
 793                                 this_chunk_blocks = u;
 794                         this_chunk_bytes = this_chunk_blocks << blkbits;
 795                         BUG_ON(this_chunk_bytes == 0);
 796
 797                         dio->boundary = buffer_boundary(map_bh);
 798                         ret = submit_page_section(dio, page, offset_in_page,
 799                                 this_chunk_bytes, dio->next_block_for_io);
 800                         if (ret) {
 801                                 page_cache_release(page);
 802                                 goto out;
 803                         }
 804                         dio->next_block_for_io += this_chunk_blocks;
 805
 806                         dio->block_in_file += this_chunk_blocks;
 807                         block_in_page += this_chunk_blocks;
 808                         dio->blocks_available -= this_chunk_blocks;
 809 next_block:
 810                         if (dio->block_in_file > dio->final_block_in_request)
 811                                 BUG();
 812                         if (dio->block_in_file == dio->final_block_in_request)
 813                                 break;
 814                 }
 815
 816                 /* Drop the ref which was taken in get_user_pages() */
 817                 page_cache_release(page);
 818                 block_in_page = 0;
 819         }
 820 out:
 821         return ret;
 822 }
 823
 824 static int
 825 direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode,
 826         const struct iovec *iov, loff_t offset, unsigned long nr_segs,
 827         unsigned blkbits, get_blocks_t get_blocks)
 828 {
 829         unsigned long user_addr;
 830         int seg;
 831         int ret = 0;
 832         int ret2;
 833         struct dio *dio;
 834         size_t bytes;
 835
 836         dio = kmalloc(sizeof(*dio), GFP_KERNEL);
 837         if (!dio)
 838                 return -ENOMEM;
 839         dio->is_async = !is_sync_kiocb(iocb);
 840
 841         dio->bio = NULL;
 842         dio->inode = inode;
 843         dio->rw = rw;
 844         dio->blkbits = blkbits;
 845         dio->blkfactor = inode->i_blkbits - blkbits;
 846         dio->start_zero_done = 0;
 847         dio->block_in_file = offset >> blkbits;
 848         dio->blocks_available = 0;
 849
 850         dio->cur_page = NULL;
 851
 852         dio->boundary = 0;
 853         dio->reap_counter = 0;
 854         dio->get_blocks = get_blocks;
 855         dio->final_block_in_bio = -1;
 856         dio->next_block_for_io = -1;
 857
 858         dio->page_errors = 0;
 859         dio->result = 0;
 860         dio->iocb = iocb;
 861
 862         /*
 863          * BIO completion state.
 864          *
 865          * ->bio_count starts out at one, and we decrement it to zero after all
 866          * BIOs are submitted.  This to avoid the situation where a really fast
 867          * (or synchronous) device could take the count to zero while we're
 868          * still submitting BIOs.
 869          */
 870         atomic_set(&dio->bio_count, 1);
 871         atomic_set(&dio->bios_in_flight, 0);
 872         spin_lock_init(&dio->bio_list_lock);
 873         dio->bio_list = NULL;
 874         dio->waiter = NULL;
 875
 876         dio->pages_in_io = 0;
 877         for (seg = 0; seg < nr_segs; seg++)
 878                 dio->pages_in_io += (iov[seg].iov_len >> blkbits) + 2;
 879
 880         for (seg = 0; seg < nr_segs; seg++) {
 881                 user_addr = (unsigned long)iov[seg].iov_base;
 882                 bytes = iov[seg].iov_len;
 883
 884                 /* Index into the first page of the first block */
 885                 dio->first_block_in_page = (user_addr & ~PAGE_MASK) >> blkbits;
 886                 dio->final_block_in_request = dio->block_in_file +
 887                                                 (bytes >> blkbits);
 888                 /* Page fetching state */
 889                 dio->head = 0;
 890                 dio->tail = 0;
 891                 dio->curr_page = 0;
 892
 893                 dio->total_pages = 0;
 894                 if (user_addr & (PAGE_SIZE-1)) {
 895                         dio->total_pages++;
 896                         bytes -= PAGE_SIZE - (user_addr & (PAGE_SIZE - 1));
 897                 }
 898                 dio->total_pages += (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
 899                 dio->curr_user_address = user_addr;
 900
 901                 ret = do_direct_IO(dio);
 902
 903                 dio->result += iov[seg].iov_len -
 904                         ((dio->final_block_in_request - dio->block_in_file) <<
 905                                         blkbits);
 906
 907                 if (ret) {
 908                         dio_cleanup(dio);
 909                         break;
 910                 }
 911         } /* end iovec loop */
 912
 913         /*
 914          * There may be some unwritten disk at the end of a part-written
 915          * fs-block-sized block.  Go zero that now.
 916          */
 917         dio_zero_block(dio, 1);
 918
 919         if (dio->cur_page) {
 920                 ret2 = dio_send_cur_page(dio);
 921                 if (ret == 0)
 922                         ret = ret2;
 923                 page_cache_release(dio->cur_page);
 924                 dio->cur_page = NULL;
 925         }
 926         if (dio->bio)
 927                 dio_bio_submit(dio);
 928
 929         /*
 930          * OK, all BIOs are submitted, so we can decrement bio_count to truly
 931          * reflect the number of to-be-processed BIOs.
 932          */
 933         if (dio->is_async) {
 934                 if (ret == 0)
 935                         ret = dio->result;      /* Bytes written */
 936                 finished_one_bio(dio);          /* This can free the dio */
 937                 blk_run_queues();
 938         } else {
 939                 finished_one_bio(dio);
 940                 ret2 = dio_await_completion(dio);
 941                 if (ret == 0)
 942                         ret = ret2;
 943                 if (ret == 0)
 944                         ret = dio->page_errors;
 945                 if (ret == 0 && dio->result) {
 946                         loff_t i_size = i_size_read(inode);
 947
 948                         ret = dio->result;
 949                         /*
 950                          * Adjust the return value if the read crossed a
 951                          * non-block-aligned EOF.
 952                          */
 953                         if (rw == READ && (offset + ret > i_size))
 954                                 ret = i_size - offset;
 955                 }
 956                 kfree(dio);
 957         }
 958         return ret;
 959 }
 960
 961 /*
 962  * This is a library function for use by filesystem drivers.
 963  */
 964 int
 965 blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
 966         struct block_device *bdev, const struct iovec *iov, loff_t offset,
 967         unsigned long nr_segs, get_blocks_t get_blocks)
 968 {
 969         int seg;
 970         size_t size;
 971         unsigned long addr;
 972         unsigned blkbits = inode->i_blkbits;
 973         unsigned bdev_blkbits = 0;
 974         unsigned blocksize_mask = (1 << blkbits) - 1;
 975         ssize_t retval = -EINVAL;
 976
 977         if (bdev)
 978                 bdev_blkbits = blksize_bits(bdev_hardsect_size(bdev));
 979
 980         if (offset & blocksize_mask) {
 981                 if (bdev)
 982                          blkbits = bdev_blkbits;
 983                 blocksize_mask = (1 << blkbits) - 1;
 984                 if (offset & blocksize_mask)
 985                         goto out;
 986         }
 987
 988         /* Check the memory alignment.  Blocks cannot straddle pages */
 989         for (seg = 0; seg < nr_segs; seg++) {
 990                 addr = (unsigned long)iov[seg].iov_base;
 991                 size = iov[seg].iov_len;
 992                 if ((addr & blocksize_mask) || (size & blocksize_mask))  {
 993                         if (bdev)
 994                                  blkbits = bdev_blkbits;
 995                         blocksize_mask = (1 << blkbits) - 1;
 996                         if ((addr & blocksize_mask) || (size & blocksize_mask))
 997                                 goto out;
 998                 }
 999         }
1000
1001         retval = direct_io_worker(rw, iocb, inode, iov, offset,
1002                                 nr_segs, blkbits, get_blocks);
1003 out:
1004         return retval;
1005 }