fs/cachefiles/rdwr.c

   1 /* Storage object read/write
   2  *
   3  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public Licence
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the Licence, or (at your option) any later version.
  10  */
  11
  12 #include <linux/mount.h>
  13 #include <linux/file.h>
  14 #include <linux/ima.h>
  15 #include "internal.h"
  16
  17 /*
  18  * detect wake up events generated by the unlocking of pages in which we're
  19  * interested
  20  * - we use this to detect read completion of backing pages
  21  * - the caller holds the waitqueue lock
  22  */
  23 static int cachefiles_read_waiter(wait_queue_t *wait, unsigned mode,
  24                                   int sync, void *_key)
  25 {
  26         struct cachefiles_one_read *monitor =
  27                 container_of(wait, struct cachefiles_one_read, monitor);
  28         struct cachefiles_object *object;
  29         struct wait_bit_key *key = _key;
  30         struct page *page = wait->private;
  31
  32         ASSERT(key);
  33
  34         _enter("{%lu},%u,%d,{%p,%u}",
  35                monitor->netfs_page->index, mode, sync,
  36                key->flags, key->bit_nr);
  37
  38         if (key->flags != &page->flags ||
  39             key->bit_nr != PG_locked)
  40                 return 0;
  41
  42         _debug("--- monitor %p %lx ---", page, page->flags);
  43
  44         if (!PageUptodate(page) && !PageError(page)) {
  45                 /* unlocked, not uptodate and not erronous? */
  46                 _debug("page probably truncated");
  47         }
  48
  49         /* remove from the waitqueue */
  50         list_del(&wait->task_list);
  51
  52         /* move onto the action list and queue for FS-Cache thread pool */
  53         ASSERT(monitor->op);
  54
  55         object = container_of(monitor->op->op.object,
  56                               struct cachefiles_object, fscache);
  57
  58         spin_lock(&object->work_lock);
  59         list_add_tail(&monitor->op_link, &monitor->op->to_do);
  60         spin_unlock(&object->work_lock);
  61
  62         fscache_enqueue_retrieval(monitor->op);
  63         return 0;
  64 }
  65
  66 /*
  67  * handle a probably truncated page
  68  * - check to see if the page is still relevant and reissue the read if
  69  *   possible
  70  * - return -EIO on error, -ENODATA if the page is gone, -EINPROGRESS if we
  71  *   must wait again and 0 if successful
  72  */
  73 static int cachefiles_read_reissue(struct cachefiles_object *object,
  74                                    struct cachefiles_one_read *monitor)
  75 {
  76         struct address_space *bmapping = object->backer->d_inode->i_mapping;
  77         struct page *backpage = monitor->back_page, *backpage2;
  78         int ret;
  79
  80         kenter("{ino=%lx},{%lx,%lx}",
  81                object->backer->d_inode->i_ino,
  82                backpage->index, backpage->flags);
  83
  84         /* skip if the page was truncated away completely */
  85         if (backpage->mapping != bmapping) {
  86                 kleave(" = -ENODATA [mapping]");
  87                 return -ENODATA;
  88         }
  89
  90         backpage2 = find_get_page(bmapping, backpage->index);
  91         if (!backpage2) {
  92                 kleave(" = -ENODATA [gone]");
  93                 return -ENODATA;
  94         }
  95
  96         if (backpage != backpage2) {
  97                 put_page(backpage2);
  98                 kleave(" = -ENODATA [different]");
  99                 return -ENODATA;
 100         }
 101
 102         /* the page is still there and we already have a ref on it, so we don't
 103          * need a second */
 104         put_page(backpage2);
 105
 106         INIT_LIST_HEAD(&monitor->op_link);
 107         add_page_wait_queue(backpage, &monitor->monitor);
 108
 109         if (trylock_page(backpage)) {
 110                 ret = -EIO;
 111                 if (PageError(backpage))
 112                         goto unlock_discard;
 113                 ret = 0;
 114                 if (PageUptodate(backpage))
 115                         goto unlock_discard;
 116
 117                 kdebug("reissue read");
 118                 ret = bmapping->a_ops->readpage(NULL, backpage);
 119                 if (ret < 0)
 120                         goto unlock_discard;
 121         }
 122
 123         /* but the page may have been read before the monitor was installed, so
 124          * the monitor may miss the event - so we have to ensure that we do get
 125          * one in such a case */
 126         if (trylock_page(backpage)) {
 127                 _debug("jumpstart %p {%lx}", backpage, backpage->flags);
 128                 unlock_page(backpage);
 129         }
 130
 131         /* it'll reappear on the todo list */
 132         kleave(" = -EINPROGRESS");
 133         return -EINPROGRESS;
 134
 135 unlock_discard:
 136         unlock_page(backpage);
 137         spin_lock_irq(&object->work_lock);
 138         list_del(&monitor->op_link);
 139         spin_unlock_irq(&object->work_lock);
 140         kleave(" = %d", ret);
 141         return ret;
 142 }
 143
 144 /*
 145  * copy data from backing pages to netfs pages to complete a read operation
 146  * - driven by FS-Cache's thread pool
 147  */
 148 static void cachefiles_read_copier(struct fscache_operation *_op)
 149 {
 150         struct cachefiles_one_read *monitor;
 151         struct cachefiles_object *object;
 152         struct fscache_retrieval *op;
 153         struct pagevec pagevec;
 154         int error, max;
 155
 156         op = container_of(_op, struct fscache_retrieval, op);
 157         object = container_of(op->op.object,
 158                               struct cachefiles_object, fscache);
 159
 160         _enter("{ino=%lu}", object->backer->d_inode->i_ino);
 161
 162         pagevec_init(&pagevec, 0);
 163
 164         max = 8;
 165         spin_lock_irq(&object->work_lock);
 166
 167         while (!list_empty(&op->to_do)) {
 168                 monitor = list_entry(op->to_do.next,
 169                                      struct cachefiles_one_read, op_link);
 170                 list_del(&monitor->op_link);
 171
 172                 spin_unlock_irq(&object->work_lock);
 173
 174                 _debug("- copy {%lu}", monitor->back_page->index);
 175
 176         recheck:
 177                 if (PageUptodate(monitor->back_page)) {
 178                         copy_highpage(monitor->netfs_page, monitor->back_page);
 179
 180                         pagevec_add(&pagevec, monitor->netfs_page);
 181                         fscache_mark_pages_cached(monitor->op, &pagevec);
 182                         error = 0;
 183                 } else if (!PageError(monitor->back_page)) {
 184                         /* the page has probably been truncated */
 185                         error = cachefiles_read_reissue(object, monitor);
 186                         if (error == -EINPROGRESS)
 187                                 goto next;
 188                         goto recheck;
 189                 } else {
 190                         cachefiles_io_error_obj(
 191                                 object,
 192                                 "Readpage failed on backing file %lx",
 193                                 (unsigned long) monitor->back_page->flags);
 194                         error = -EIO;
 195                 }
 196
 197                 page_cache_release(monitor->back_page);
 198
 199                 fscache_end_io(op, monitor->netfs_page, error);
 200                 page_cache_release(monitor->netfs_page);
 201                 fscache_put_retrieval(op);
 202                 kfree(monitor);
 203
 204         next:
 205                 /* let the thread pool have some air occasionally */
 206                 max--;
 207                 if (max < 0 || need_resched()) {
 208                         if (!list_empty(&op->to_do))
 209                                 fscache_enqueue_retrieval(op);
 210                         _leave(" [maxed out]");
 211                         return;
 212                 }
 213
 214                 spin_lock_irq(&object->work_lock);
 215         }
 216
 217         spin_unlock_irq(&object->work_lock);
 218         _leave("");
 219 }
 220
 221 /*
 222  * read the corresponding page to the given set from the backing file
 223  * - an uncertain page is simply discarded, to be tried again another time
 224  */
 225 static int cachefiles_read_backing_file_one(struct cachefiles_object *object,
 226                                             struct fscache_retrieval *op,
 227                                             struct page *netpage,
 228                                             struct pagevec *pagevec)
 229 {
 230         struct cachefiles_one_read *monitor;
 231         struct address_space *bmapping;
 232         struct page *newpage, *backpage;
 233         int ret;
 234
 235         _enter("");
 236
 237         pagevec_reinit(pagevec);
 238
 239         _debug("read back %p{%lu,%d}",
 240                netpage, netpage->index, page_count(netpage));
 241
 242         monitor = kzalloc(sizeof(*monitor), GFP_KERNEL);
 243         if (!monitor)
 244                 goto nomem;
 245
 246         monitor->netfs_page = netpage;
 247         monitor->op = fscache_get_retrieval(op);
 248
 249         init_waitqueue_func_entry(&monitor->monitor, cachefiles_read_waiter);
 250
 251         /* attempt to get hold of the backing page */
 252         bmapping = object->backer->d_inode->i_mapping;
 253         newpage = NULL;
 254
 255         for (;;) {
 256                 backpage = find_get_page(bmapping, netpage->index);
 257                 if (backpage)
 258                         goto backing_page_already_present;
 259
 260                 if (!newpage) {
 261                         newpage = page_cache_alloc_cold(bmapping);
 262                         if (!newpage)
 263                                 goto nomem_monitor;
 264                 }
 265
 266                 ret = add_to_page_cache(newpage, bmapping,
 267                                         netpage->index, GFP_KERNEL);
 268                 if (ret == 0)
 269                         goto installed_new_backing_page;
 270                 if (ret != -EEXIST)
 271                         goto nomem_page;
 272         }
 273
 274         /* we've installed a new backing page, so now we need to add it
 275          * to the LRU list and start it reading */
 276 installed_new_backing_page:
 277         _debug("- new %p", newpage);
 278
 279         backpage = newpage;
 280         newpage = NULL;
 281
 282         page_cache_get(backpage);
 283         pagevec_add(pagevec, backpage);
 284         __pagevec_lru_add_file(pagevec);
 285
 286 read_backing_page:
 287         ret = bmapping->a_ops->readpage(NULL, backpage);
 288         if (ret < 0)
 289                 goto read_error;
 290
 291         /* set the monitor to transfer the data across */
 292 monitor_backing_page:
 293         _debug("- monitor add");
 294
 295         /* install the monitor */
 296         page_cache_get(monitor->netfs_page);
 297         page_cache_get(backpage);
 298         monitor->back_page = backpage;
 299         monitor->monitor.private = backpage;
 300         add_page_wait_queue(backpage, &monitor->monitor);
 301         monitor = NULL;
 302
 303         /* but the page may have been read before the monitor was installed, so
 304          * the monitor may miss the event - so we have to ensure that we do get
 305          * one in such a case */
 306         if (trylock_page(backpage)) {
 307                 _debug("jumpstart %p {%lx}", backpage, backpage->flags);
 308                 unlock_page(backpage);
 309         }
 310         goto success;
 311
 312         /* if the backing page is already present, it can be in one of
 313          * three states: read in progress, read failed or read okay */
 314 backing_page_already_present:
 315         _debug("- present");
 316
 317         if (newpage) {
 318                 page_cache_release(newpage);
 319                 newpage = NULL;
 320         }
 321
 322         if (PageError(backpage))
 323                 goto io_error;
 324
 325         if (PageUptodate(backpage))
 326                 goto backing_page_already_uptodate;
 327
 328         if (!trylock_page(backpage))
 329                 goto monitor_backing_page;
 330         _debug("read %p {%lx}", backpage, backpage->flags);
 331         goto read_backing_page;
 332
 333         /* the backing page is already up to date, attach the netfs
 334          * page to the pagecache and LRU and copy the data across */
 335 backing_page_already_uptodate:
 336         _debug("- uptodate");
 337
 338         pagevec_add(pagevec, netpage);
 339         fscache_mark_pages_cached(op, pagevec);
 340
 341         copy_highpage(netpage, backpage);
 342         fscache_end_io(op, netpage, 0);
 343
 344 success:
 345         _debug("success");
 346         ret = 0;
 347
 348 out:
 349         if (backpage)
 350                 page_cache_release(backpage);
 351         if (monitor) {
 352                 fscache_put_retrieval(monitor->op);
 353                 kfree(monitor);
 354         }
 355         _leave(" = %d", ret);
 356         return ret;
 357
 358 read_error:
 359         _debug("read error %d", ret);
 360         if (ret == -ENOMEM)
 361                 goto out;
 362 io_error:
 363         cachefiles_io_error_obj(object, "Page read error on backing file");
 364         ret = -ENOBUFS;
 365         goto out;
 366
 367 nomem_page:
 368         page_cache_release(newpage);
 369 nomem_monitor:
 370         fscache_put_retrieval(monitor->op);
 371         kfree(monitor);
 372 nomem:
 373         _leave(" = -ENOMEM");
 374         return -ENOMEM;
 375 }
 376
 377 /*
 378  * read a page from the cache or allocate a block in which to store it
 379  * - cache withdrawal is prevented by the caller
 380  * - returns -EINTR if interrupted
 381  * - returns -ENOMEM if ran out of memory
 382  * - returns -ENOBUFS if no buffers can be made available
 383  * - returns -ENOBUFS if page is beyond EOF
 384  * - if the page is backed by a block in the cache:
 385  *   - a read will be started which will call the callback on completion
 386  *   - 0 will be returned
 387  * - else if the page is unbacked:
 388  *   - the metadata will be retained
 389  *   - -ENODATA will be returned
 390  */
 391 int cachefiles_read_or_alloc_page(struct fscache_retrieval *op,
 392                                   struct page *page,
 393                                   gfp_t gfp)
 394 {
 395         struct cachefiles_object *object;
 396         struct cachefiles_cache *cache;
 397         struct pagevec pagevec;
 398         struct inode *inode;
 399         sector_t block0, block;
 400         unsigned shift;
 401         int ret;
 402
 403         object = container_of(op->op.object,
 404                               struct cachefiles_object, fscache);
 405         cache = container_of(object->fscache.cache,
 406                              struct cachefiles_cache, cache);
 407
 408         _enter("{%p},{%lx},,,", object, page->index);
 409
 410         if (!object->backer)
 411                 return -ENOBUFS;
 412
 413         inode = object->backer->d_inode;
 414         ASSERT(S_ISREG(inode->i_mode));
 415         ASSERT(inode->i_mapping->a_ops->bmap);
 416         ASSERT(inode->i_mapping->a_ops->readpages);
 417
 418         /* calculate the shift required to use bmap */
 419         if (inode->i_sb->s_blocksize > PAGE_SIZE)
 420                 return -ENOBUFS;
 421
 422         shift = PAGE_SHIFT - inode->i_sb->s_blocksize_bits;
 423
 424         op->op.flags &= FSCACHE_OP_KEEP_FLAGS;
 425         op->op.flags |= FSCACHE_OP_FAST;
 426         op->op.processor = cachefiles_read_copier;
 427
 428         pagevec_init(&pagevec, 0);
 429
 430         /* we assume the absence or presence of the first block is a good
 431          * enough indication for the page as a whole
 432          * - TODO: don't use bmap() for this as it is _not_ actually good
 433          *   enough for this as it doesn't indicate errors, but it's all we've
 434          *   got for the moment
 435          */
 436         block0 = page->index;
 437         block0 <<= shift;
 438
 439         block = inode->i_mapping->a_ops->bmap(inode->i_mapping, block0);
 440         _debug("%llx -> %llx",
 441                (unsigned long long) block0,
 442                (unsigned long long) block);
 443
 444         if (block) {
 445                 /* submit the apparently valid page to the backing fs to be
 446                  * read from disk */
 447                 ret = cachefiles_read_backing_file_one(object, op, page,
 448                                                        &pagevec);
 449         } else if (cachefiles_has_space(cache, 0, 1) == 0) {
 450                 /* there's space in the cache we can use */
 451                 pagevec_add(&pagevec, page);
 452                 fscache_mark_pages_cached(op, &pagevec);
 453                 ret = -ENODATA;
 454         } else {
 455                 ret = -ENOBUFS;
 456         }
 457
 458         _leave(" = %d", ret);
 459         return ret;
 460 }
 461
 462 /*
 463  * read the corresponding pages to the given set from the backing file
 464  * - any uncertain pages are simply discarded, to be tried again another time
 465  */
 466 static int cachefiles_read_backing_file(struct cachefiles_object *object,
 467                                         struct fscache_retrieval *op,
 468                                         struct list_head *list,
 469                                         struct pagevec *mark_pvec)
 470 {
 471         struct cachefiles_one_read *monitor = NULL;
 472         struct address_space *bmapping = object->backer->d_inode->i_mapping;
 473         struct pagevec lru_pvec;
 474         struct page *newpage = NULL, *netpage, *_n, *backpage = NULL;
 475         int ret = 0;
 476
 477         _enter("");
 478
 479         pagevec_init(&lru_pvec, 0);
 480
 481         list_for_each_entry_safe(netpage, _n, list, lru) {
 482                 list_del(&netpage->lru);
 483
 484                 _debug("read back %p{%lu,%d}",
 485                        netpage, netpage->index, page_count(netpage));
 486
 487                 if (!monitor) {
 488                         monitor = kzalloc(sizeof(*monitor), GFP_KERNEL);
 489                         if (!monitor)
 490                                 goto nomem;
 491
 492                         monitor->op = fscache_get_retrieval(op);
 493                         init_waitqueue_func_entry(&monitor->monitor,
 494                                                   cachefiles_read_waiter);
 495                 }
 496
 497                 for (;;) {
 498                         backpage = find_get_page(bmapping, netpage->index);
 499                         if (backpage)
 500                                 goto backing_page_already_present;
 501
 502                         if (!newpage) {
 503                                 newpage = page_cache_alloc_cold(bmapping);
 504                                 if (!newpage)
 505                                         goto nomem;
 506                         }
 507
 508                         ret = add_to_page_cache(newpage, bmapping,
 509                                                 netpage->index, GFP_KERNEL);
 510                         if (ret == 0)
 511                                 goto installed_new_backing_page;
 512                         if (ret != -EEXIST)
 513                                 goto nomem;
 514                 }
 515
 516                 /* we've installed a new backing page, so now we need to add it
 517                  * to the LRU list and start it reading */
 518         installed_new_backing_page:
 519                 _debug("- new %p", newpage);
 520
 521                 backpage = newpage;
 522                 newpage = NULL;
 523
 524                 page_cache_get(backpage);
 525                 if (!pagevec_add(&lru_pvec, backpage))
 526                         __pagevec_lru_add_file(&lru_pvec);
 527
 528         reread_backing_page:
 529                 ret = bmapping->a_ops->readpage(NULL, backpage);
 530                 if (ret < 0)
 531                         goto read_error;
 532
 533                 /* add the netfs page to the pagecache and LRU, and set the
 534                  * monitor to transfer the data across */
 535         monitor_backing_page:
 536                 _debug("- monitor add");
 537
 538                 ret = add_to_page_cache(netpage, op->mapping, netpage->index,
 539                                         GFP_KERNEL);
 540                 if (ret < 0) {
 541                         if (ret == -EEXIST) {
 542                                 page_cache_release(netpage);
 543                                 continue;
 544                         }
 545                         goto nomem;
 546                 }
 547
 548                 page_cache_get(netpage);
 549                 if (!pagevec_add(&lru_pvec, netpage))
 550                         __pagevec_lru_add_file(&lru_pvec);
 551
 552                 /* install a monitor */
 553                 page_cache_get(netpage);
 554                 monitor->netfs_page = netpage;
 555
 556                 page_cache_get(backpage);
 557                 monitor->back_page = backpage;
 558                 monitor->monitor.private = backpage;
 559                 add_page_wait_queue(backpage, &monitor->monitor);
 560                 monitor = NULL;
 561
 562                 /* but the page may have been read before the monitor was
 563                  * installed, so the monitor may miss the event - so we have to
 564                  * ensure that we do get one in such a case */
 565                 if (trylock_page(backpage)) {
 566                         _debug("2unlock %p {%lx}", backpage, backpage->flags);
 567                         unlock_page(backpage);
 568                 }
 569
 570                 page_cache_release(backpage);
 571                 backpage = NULL;
 572
 573                 page_cache_release(netpage);
 574                 netpage = NULL;
 575                 continue;
 576
 577                 /* if the backing page is already present, it can be in one of
 578                  * three states: read in progress, read failed or read okay */
 579         backing_page_already_present:
 580                 _debug("- present %p", backpage);
 581
 582                 if (PageError(backpage))
 583                         goto io_error;
 584
 585                 if (PageUptodate(backpage))
 586                         goto backing_page_already_uptodate;
 587
 588                 _debug("- not ready %p{%lx}", backpage, backpage->flags);
 589
 590                 if (!trylock_page(backpage))
 591                         goto monitor_backing_page;
 592
 593                 if (PageError(backpage)) {
 594                         _debug("error %lx", backpage->flags);
 595                         unlock_page(backpage);
 596                         goto io_error;
 597                 }
 598
 599                 if (PageUptodate(backpage))
 600                         goto backing_page_already_uptodate_unlock;
 601
 602                 /* we've locked a page that's neither up to date nor erroneous,
 603                  * so we need to attempt to read it again */
 604                 goto reread_backing_page;
 605
 606                 /* the backing page is already up to date, attach the netfs
 607                  * page to the pagecache and LRU and copy the data across */
 608         backing_page_already_uptodate_unlock:
 609                 _debug("uptodate %lx", backpage->flags);
 610                 unlock_page(backpage);
 611         backing_page_already_uptodate:
 612                 _debug("- uptodate");
 613
 614                 ret = add_to_page_cache(netpage, op->mapping, netpage->index,
 615                                         GFP_KERNEL);
 616                 if (ret < 0) {
 617                         if (ret == -EEXIST) {
 618                                 page_cache_release(netpage);
 619                                 continue;
 620                         }
 621                         goto nomem;
 622                 }
 623
 624                 copy_highpage(netpage, backpage);
 625
 626                 page_cache_release(backpage);
 627                 backpage = NULL;
 628
 629                 if (!pagevec_add(mark_pvec, netpage))
 630                         fscache_mark_pages_cached(op, mark_pvec);
 631
 632                 page_cache_get(netpage);
 633                 if (!pagevec_add(&lru_pvec, netpage))
 634                         __pagevec_lru_add_file(&lru_pvec);
 635
 636                 fscache_end_io(op, netpage, 0);
 637                 page_cache_release(netpage);
 638                 netpage = NULL;
 639                 continue;
 640         }
 641
 642         netpage = NULL;
 643
 644         _debug("out");
 645
 646 out:
 647         /* tidy up */
 648         pagevec_lru_add_file(&lru_pvec);
 649
 650         if (newpage)
 651                 page_cache_release(newpage);
 652         if (netpage)
 653                 page_cache_release(netpage);
 654         if (backpage)
 655                 page_cache_release(backpage);
 656         if (monitor) {
 657                 fscache_put_retrieval(op);
 658                 kfree(monitor);
 659         }
 660
 661         list_for_each_entry_safe(netpage, _n, list, lru) {
 662                 list_del(&netpage->lru);
 663                 page_cache_release(netpage);
 664         }
 665
 666         _leave(" = %d", ret);
 667         return ret;
 668
 669 nomem:
 670         _debug("nomem");
 671         ret = -ENOMEM;
 672         goto out;
 673
 674 read_error:
 675         _debug("read error %d", ret);
 676         if (ret == -ENOMEM)
 677                 goto out;
 678 io_error:
 679         cachefiles_io_error_obj(object, "Page read error on backing file");
 680         ret = -ENOBUFS;
 681         goto out;
 682 }
 683
 684 /*
 685  * read a list of pages from the cache or allocate blocks in which to store
 686  * them
 687  */
 688 int cachefiles_read_or_alloc_pages(struct fscache_retrieval *op,
 689                                    struct list_head *pages,
 690                                    unsigned *nr_pages,
 691                                    gfp_t gfp)
 692 {
 693         struct cachefiles_object *object;
 694         struct cachefiles_cache *cache;
 695         struct list_head backpages;
 696         struct pagevec pagevec;
 697         struct inode *inode;
 698         struct page *page, *_n;
 699         unsigned shift, nrbackpages;
 700         int ret, ret2, space;
 701
 702         object = container_of(op->op.object,
 703                               struct cachefiles_object, fscache);
 704         cache = container_of(object->fscache.cache,
 705                              struct cachefiles_cache, cache);
 706
 707         _enter("{OBJ%x,%d},,%d,,",
 708                object->fscache.debug_id, atomic_read(&op->op.usage),
 709                *nr_pages);
 710
 711         if (!object->backer)
 712                 return -ENOBUFS;
 713
 714         space = 1;
 715         if (cachefiles_has_space(cache, 0, *nr_pages) < 0)
 716                 space = 0;
 717
 718         inode = object->backer->d_inode;
 719         ASSERT(S_ISREG(inode->i_mode));
 720         ASSERT(inode->i_mapping->a_ops->bmap);
 721         ASSERT(inode->i_mapping->a_ops->readpages);
 722
 723         /* calculate the shift required to use bmap */
 724         if (inode->i_sb->s_blocksize > PAGE_SIZE)
 725                 return -ENOBUFS;
 726
 727         shift = PAGE_SHIFT - inode->i_sb->s_blocksize_bits;
 728
 729         pagevec_init(&pagevec, 0);
 730
 731         op->op.flags &= FSCACHE_OP_KEEP_FLAGS;
 732         op->op.flags |= FSCACHE_OP_FAST;
 733         op->op.processor = cachefiles_read_copier;
 734
 735         INIT_LIST_HEAD(&backpages);
 736         nrbackpages = 0;
 737
 738         ret = space ? -ENODATA : -ENOBUFS;
 739         list_for_each_entry_safe(page, _n, pages, lru) {
 740                 sector_t block0, block;
 741
 742                 /* we assume the absence or presence of the first block is a
 743                  * good enough indication for the page as a whole
 744                  * - TODO: don't use bmap() for this as it is _not_ actually
 745                  *   good enough for this as it doesn't indicate errors, but
 746                  *   it's all we've got for the moment
 747                  */
 748                 block0 = page->index;
 749                 block0 <<= shift;
 750
 751                 block = inode->i_mapping->a_ops->bmap(inode->i_mapping,
 752                                                       block0);
 753                 _debug("%llx -> %llx",
 754                        (unsigned long long) block0,
 755                        (unsigned long long) block);
 756
 757                 if (block) {
 758                         /* we have data - add it to the list to give to the
 759                          * backing fs */
 760                         list_move(&page->lru, &backpages);
 761                         (*nr_pages)--;
 762                         nrbackpages++;
 763                 } else if (space && pagevec_add(&pagevec, page) == 0) {
 764                         fscache_mark_pages_cached(op, &pagevec);
 765                         ret = -ENODATA;
 766                 }
 767         }
 768
 769         if (pagevec_count(&pagevec) > 0)
 770                 fscache_mark_pages_cached(op, &pagevec);
 771
 772         if (list_empty(pages))
 773                 ret = 0;
 774
 775         /* submit the apparently valid pages to the backing fs to be read from
 776          * disk */
 777         if (nrbackpages > 0) {
 778                 ret2 = cachefiles_read_backing_file(object, op, &backpages,
 779                                                     &pagevec);
 780                 if (ret2 == -ENOMEM || ret2 == -EINTR)
 781                         ret = ret2;
 782         }
 783
 784         if (pagevec_count(&pagevec) > 0)
 785                 fscache_mark_pages_cached(op, &pagevec);
 786
 787         _leave(" = %d [nr=%u%s]",
 788                ret, *nr_pages, list_empty(pages) ? " empty" : "");
 789         return ret;
 790 }
 791
 792 /*
 793  * allocate a block in the cache in which to store a page
 794  * - cache withdrawal is prevented by the caller
 795  * - returns -EINTR if interrupted
 796  * - returns -ENOMEM if ran out of memory
 797  * - returns -ENOBUFS if no buffers can be made available
 798  * - returns -ENOBUFS if page is beyond EOF
 799  * - otherwise:
 800  *   - the metadata will be retained
 801  *   - 0 will be returned
 802  */
 803 int cachefiles_allocate_page(struct fscache_retrieval *op,
 804                              struct page *page,
 805                              gfp_t gfp)
 806 {
 807         struct cachefiles_object *object;
 808         struct cachefiles_cache *cache;
 809         struct pagevec pagevec;
 810         int ret;
 811
 812         object = container_of(op->op.object,
 813                               struct cachefiles_object, fscache);
 814         cache = container_of(object->fscache.cache,
 815                              struct cachefiles_cache, cache);
 816
 817         _enter("%p,{%lx},", object, page->index);
 818
 819         ret = cachefiles_has_space(cache, 0, 1);
 820         if (ret == 0) {
 821                 pagevec_init(&pagevec, 0);
 822                 pagevec_add(&pagevec, page);
 823                 fscache_mark_pages_cached(op, &pagevec);
 824         } else {
 825                 ret = -ENOBUFS;
 826         }
 827
 828         _leave(" = %d", ret);
 829         return ret;
 830 }
 831
 832 /*
 833  * allocate blocks in the cache in which to store a set of pages
 834  * - cache withdrawal is prevented by the caller
 835  * - returns -EINTR if interrupted
 836  * - returns -ENOMEM if ran out of memory
 837  * - returns -ENOBUFS if some buffers couldn't be made available
 838  * - returns -ENOBUFS if some pages are beyond EOF
 839  * - otherwise:
 840  *   - -ENODATA will be returned
 841  * - metadata will be retained for any page marked
 842  */
 843 int cachefiles_allocate_pages(struct fscache_retrieval *op,
 844                               struct list_head *pages,
 845                               unsigned *nr_pages,
 846                               gfp_t gfp)
 847 {
 848         struct cachefiles_object *object;
 849         struct cachefiles_cache *cache;
 850         struct pagevec pagevec;
 851         struct page *page;
 852         int ret;
 853
 854         object = container_of(op->op.object,
 855                               struct cachefiles_object, fscache);
 856         cache = container_of(object->fscache.cache,
 857                              struct cachefiles_cache, cache);
 858
 859         _enter("%p,,,%d,", object, *nr_pages);
 860
 861         ret = cachefiles_has_space(cache, 0, *nr_pages);
 862         if (ret == 0) {
 863                 pagevec_init(&pagevec, 0);
 864
 865                 list_for_each_entry(page, pages, lru) {
 866                         if (pagevec_add(&pagevec, page) == 0)
 867                                 fscache_mark_pages_cached(op, &pagevec);
 868                 }
 869
 870                 if (pagevec_count(&pagevec) > 0)
 871                         fscache_mark_pages_cached(op, &pagevec);
 872                 ret = -ENODATA;
 873         } else {
 874                 ret = -ENOBUFS;
 875         }
 876
 877         _leave(" = %d", ret);
 878         return ret;
 879 }
 880
 881 /*
 882  * request a page be stored in the cache
 883  * - cache withdrawal is prevented by the caller
 884  * - this request may be ignored if there's no cache block available, in which
 885  *   case -ENOBUFS will be returned
 886  * - if the op is in progress, 0 will be returned
 887  */
 888 int cachefiles_write_page(struct fscache_storage *op, struct page *page)
 889 {
 890         struct cachefiles_object *object;
 891         struct cachefiles_cache *cache;
 892         mm_segment_t old_fs;
 893         struct file *file;
 894         loff_t pos, eof;
 895         size_t len;
 896         void *data;
 897         int ret;
 898
 899         ASSERT(op != NULL);
 900         ASSERT(page != NULL);
 901
 902         object = container_of(op->op.object,
 903                               struct cachefiles_object, fscache);
 904
 905         _enter("%p,%p{%lx},,,", object, page, page->index);
 906
 907         if (!object->backer) {
 908                 _leave(" = -ENOBUFS");
 909                 return -ENOBUFS;
 910         }
 911
 912         ASSERT(S_ISREG(object->backer->d_inode->i_mode));
 913
 914         cache = container_of(object->fscache.cache,
 915                              struct cachefiles_cache, cache);
 916
 917         /* write the page to the backing filesystem and let it store it in its
 918          * own time */
 919         dget(object->backer);
 920         mntget(cache->mnt);
 921         file = dentry_open(object->backer, cache->mnt, O_RDWR,
 922                            cache->cache_cred);
 923         if (IS_ERR(file)) {
 924                 ret = PTR_ERR(file);
 925         } else {
 926                 ima_counts_get(file);
 927                 ret = -EIO;
 928                 if (file->f_op->write) {
 929                         pos = (loff_t) page->index << PAGE_SHIFT;
 930
 931                         /* we mustn't write more data than we have, so we have
 932                          * to beware of a partial page at EOF */
 933                         eof = object->fscache.store_limit_l;
 934                         len = PAGE_SIZE;
 935                         if (eof & ~PAGE_MASK) {
 936                                 ASSERTCMP(pos, <, eof);
 937                                 if (eof - pos < PAGE_SIZE) {
 938                                         _debug("cut short %llx to %llx",
 939                                                pos, eof);
 940                                         len = eof - pos;
 941                                         ASSERTCMP(pos + len, ==, eof);
 942                                 }
 943                         }
 944
 945                         data = kmap(page);
 946                         old_fs = get_fs();
 947                         set_fs(KERNEL_DS);
 948                         ret = file->f_op->write(
 949                                 file, (const void __user *) data, len, &pos);
 950                         set_fs(old_fs);
 951                         kunmap(page);
 952                         if (ret != len)
 953                                 ret = -EIO;
 954                 }
 955                 fput(file);
 956         }
 957
 958         if (ret < 0) {
 959                 if (ret == -EIO)
 960                         cachefiles_io_error_obj(
 961                                 object, "Write page to backing file failed");
 962                 ret = -ENOBUFS;
 963         }
 964
 965         _leave(" = %d", ret);
 966         return ret;
 967 }
 968
 969 /*
 970  * detach a backing block from a page
 971  * - cache withdrawal is prevented by the caller
 972  */
 973 void cachefiles_uncache_page(struct fscache_object *_object, struct page *page)
 974 {
 975         struct cachefiles_object *object;
 976         struct cachefiles_cache *cache;
 977
 978         object = container_of(_object, struct cachefiles_object, fscache);
 979         cache = container_of(object->fscache.cache,
 980                              struct cachefiles_cache, cache);
 981
 982         _enter("%p,{%lu}", object, page->index);
 983
 984         spin_unlock(&object->fscache.cookie->lock);
 985 }