fs/fscache/page.c

   1 /* Cache page management and data I/O routines
   2  *
   3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define FSCACHE_DEBUG_LEVEL PAGE
  13 #include <linux/module.h>
  14 #include <linux/fscache-cache.h>
  15 #include <linux/buffer_head.h>
  16 #include <linux/pagevec.h>
  17 #include <linux/slab.h>
  18 #include "internal.h"
  19
  20 /*
  21  * check to see if a page is being written to the cache
  22  */
  23 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  24 {
  25         void *val;
  26
  27         rcu_read_lock();
  28         val = radix_tree_lookup(&cookie->stores, page->index);
  29         rcu_read_unlock();
  30
  31         return val != NULL;
  32 }
  33 EXPORT_SYMBOL(__fscache_check_page_write);
  34
  35 /*
  36  * wait for a page to finish being written to the cache
  37  */
  38 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  39 {
  40         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  41
  42         wait_event(*wq, !__fscache_check_page_write(cookie, page));
  43 }
  44 EXPORT_SYMBOL(__fscache_wait_on_page_write);
  45
  46 /*
  47  * decide whether a page can be released, possibly by cancelling a store to it
  48  * - we're allowed to sleep if __GFP_WAIT is flagged
  49  */
  50 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
  51                                   struct page *page,
  52                                   gfp_t gfp)
  53 {
  54         struct page *xpage;
  55         void *val;
  56
  57         _enter("%p,%p,%x", cookie, page, gfp);
  58
  59         rcu_read_lock();
  60         val = radix_tree_lookup(&cookie->stores, page->index);
  61         if (!val) {
  62                 rcu_read_unlock();
  63                 fscache_stat(&fscache_n_store_vmscan_not_storing);
  64                 __fscache_uncache_page(cookie, page);
  65                 return true;
  66         }
  67
  68         /* see if the page is actually undergoing storage - if so we can't get
  69          * rid of it till the cache has finished with it */
  70         if (radix_tree_tag_get(&cookie->stores, page->index,
  71                                FSCACHE_COOKIE_STORING_TAG)) {
  72                 rcu_read_unlock();
  73                 goto page_busy;
  74         }
  75
  76         /* the page is pending storage, so we attempt to cancel the store and
  77          * discard the store request so that the page can be reclaimed */
  78         spin_lock(&cookie->stores_lock);
  79         rcu_read_unlock();
  80
  81         if (radix_tree_tag_get(&cookie->stores, page->index,
  82                                FSCACHE_COOKIE_STORING_TAG)) {
  83                 /* the page started to undergo storage whilst we were looking,
  84                  * so now we can only wait or return */
  85                 spin_unlock(&cookie->stores_lock);
  86                 goto page_busy;
  87         }
  88
  89         xpage = radix_tree_delete(&cookie->stores, page->index);
  90         spin_unlock(&cookie->stores_lock);
  91
  92         if (xpage) {
  93                 fscache_stat(&fscache_n_store_vmscan_cancelled);
  94                 fscache_stat(&fscache_n_store_radix_deletes);
  95                 ASSERTCMP(xpage, ==, page);
  96         } else {
  97                 fscache_stat(&fscache_n_store_vmscan_gone);
  98         }
  99
 100         wake_up_bit(&cookie->flags, 0);
 101         if (xpage)
 102                 page_cache_release(xpage);
 103         __fscache_uncache_page(cookie, page);
 104         return true;
 105
 106 page_busy:
 107         /* we might want to wait here, but that could deadlock the allocator as
 108          * the work threads writing to the cache may all end up sleeping
 109          * on memory allocation */
 110         fscache_stat(&fscache_n_store_vmscan_busy);
 111         return false;
 112 }
 113 EXPORT_SYMBOL(__fscache_maybe_release_page);
 114
 115 /*
 116  * note that a page has finished being written to the cache
 117  */
 118 static void fscache_end_page_write(struct fscache_object *object,
 119                                    struct page *page)
 120 {
 121         struct fscache_cookie *cookie;
 122         struct page *xpage = NULL;
 123
 124         spin_lock(&object->lock);
 125         cookie = object->cookie;
 126         if (cookie) {
 127                 /* delete the page from the tree if it is now no longer
 128                  * pending */
 129                 spin_lock(&cookie->stores_lock);
 130                 radix_tree_tag_clear(&cookie->stores, page->index,
 131                                      FSCACHE_COOKIE_STORING_TAG);
 132                 if (!radix_tree_tag_get(&cookie->stores, page->index,
 133                                         FSCACHE_COOKIE_PENDING_TAG)) {
 134                         fscache_stat(&fscache_n_store_radix_deletes);
 135                         xpage = radix_tree_delete(&cookie->stores, page->index);
 136                 }
 137                 spin_unlock(&cookie->stores_lock);
 138                 wake_up_bit(&cookie->flags, 0);
 139         }
 140         spin_unlock(&object->lock);
 141         if (xpage)
 142                 page_cache_release(xpage);
 143 }
 144
 145 /*
 146  * actually apply the changed attributes to a cache object
 147  */
 148 static void fscache_attr_changed_op(struct fscache_operation *op)
 149 {
 150         struct fscache_object *object = op->object;
 151         int ret;
 152
 153         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
 154
 155         fscache_stat(&fscache_n_attr_changed_calls);
 156
 157         if (fscache_object_is_active(object)) {
 158                 fscache_set_op_state(op, "CallFS");
 159                 fscache_stat(&fscache_n_cop_attr_changed);
 160                 ret = object->cache->ops->attr_changed(object);
 161                 fscache_stat_d(&fscache_n_cop_attr_changed);
 162                 fscache_set_op_state(op, "Done");
 163                 if (ret < 0)
 164                         fscache_abort_object(object);
 165         }
 166
 167         _leave("");
 168 }
 169
 170 /*
 171  * notification that the attributes on an object have changed
 172  */
 173 int __fscache_attr_changed(struct fscache_cookie *cookie)
 174 {
 175         struct fscache_operation *op;
 176         struct fscache_object *object;
 177
 178         _enter("%p", cookie);
 179
 180         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 181
 182         fscache_stat(&fscache_n_attr_changed);
 183
 184         op = kzalloc(sizeof(*op), GFP_KERNEL);
 185         if (!op) {
 186                 fscache_stat(&fscache_n_attr_changed_nomem);
 187                 _leave(" = -ENOMEM");
 188                 return -ENOMEM;
 189         }
 190
 191         fscache_operation_init(op, fscache_attr_changed_op, NULL);
 192         op->flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_EXCLUSIVE);
 193         fscache_set_op_name(op, "Attr");
 194
 195         spin_lock(&cookie->lock);
 196
 197         if (hlist_empty(&cookie->backing_objects))
 198                 goto nobufs;
 199         object = hlist_entry(cookie->backing_objects.first,
 200                              struct fscache_object, cookie_link);
 201
 202         if (fscache_submit_exclusive_op(object, op) < 0)
 203                 goto nobufs;
 204         spin_unlock(&cookie->lock);
 205         fscache_stat(&fscache_n_attr_changed_ok);
 206         fscache_put_operation(op);
 207         _leave(" = 0");
 208         return 0;
 209
 210 nobufs:
 211         spin_unlock(&cookie->lock);
 212         kfree(op);
 213         fscache_stat(&fscache_n_attr_changed_nobufs);
 214         _leave(" = %d", -ENOBUFS);
 215         return -ENOBUFS;
 216 }
 217 EXPORT_SYMBOL(__fscache_attr_changed);
 218
 219 /*
 220  * release a retrieval op reference
 221  */
 222 static void fscache_release_retrieval_op(struct fscache_operation *_op)
 223 {
 224         struct fscache_retrieval *op =
 225                 container_of(_op, struct fscache_retrieval, op);
 226
 227         _enter("{OP%x}", op->op.debug_id);
 228
 229         fscache_hist(fscache_retrieval_histogram, op->start_time);
 230         if (op->context)
 231                 fscache_put_context(op->op.object->cookie, op->context);
 232
 233         _leave("");
 234 }
 235
 236 /*
 237  * allocate a retrieval op
 238  */
 239 static struct fscache_retrieval *fscache_alloc_retrieval(
 240         struct address_space *mapping,
 241         fscache_rw_complete_t end_io_func,
 242         void *context)
 243 {
 244         struct fscache_retrieval *op;
 245
 246         /* allocate a retrieval operation and attempt to submit it */
 247         op = kzalloc(sizeof(*op), GFP_NOIO);
 248         if (!op) {
 249                 fscache_stat(&fscache_n_retrievals_nomem);
 250                 return NULL;
 251         }
 252
 253         fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op);
 254         op->op.flags    = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING);
 255         op->mapping     = mapping;
 256         op->end_io_func = end_io_func;
 257         op->context     = context;
 258         op->start_time  = jiffies;
 259         INIT_LIST_HEAD(&op->to_do);
 260         fscache_set_op_name(&op->op, "Retr");
 261         return op;
 262 }
 263
 264 /*
 265  * wait for a deferred lookup to complete
 266  */
 267 static int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 268 {
 269         unsigned long jif;
 270
 271         _enter("");
 272
 273         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 274                 _leave(" = 0 [imm]");
 275                 return 0;
 276         }
 277
 278         fscache_stat(&fscache_n_retrievals_wait);
 279
 280         jif = jiffies;
 281         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 282                         fscache_wait_bit_interruptible,
 283                         TASK_INTERRUPTIBLE) != 0) {
 284                 fscache_stat(&fscache_n_retrievals_intr);
 285                 _leave(" = -ERESTARTSYS");
 286                 return -ERESTARTSYS;
 287         }
 288
 289         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 290
 291         smp_rmb();
 292         fscache_hist(fscache_retrieval_delay_histogram, jif);
 293         _leave(" = 0 [dly]");
 294         return 0;
 295 }
 296
 297 /*
 298  * wait for an object to become active (or dead)
 299  */
 300 static int fscache_wait_for_retrieval_activation(struct fscache_object *object,
 301                                                  struct fscache_retrieval *op,
 302                                                  atomic_t *stat_op_waits,
 303                                                  atomic_t *stat_object_dead)
 304 {
 305         int ret;
 306
 307         if (!test_bit(FSCACHE_OP_WAITING, &op->op.flags))
 308                 goto check_if_dead;
 309
 310         _debug(">>> WT");
 311         fscache_stat(stat_op_waits);
 312         if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 313                         fscache_wait_bit_interruptible,
 314                         TASK_INTERRUPTIBLE) < 0) {
 315                 ret = fscache_cancel_op(&op->op);
 316                 if (ret == 0)
 317                         return -ERESTARTSYS;
 318
 319                 /* it's been removed from the pending queue by another party,
 320                  * so we should get to run shortly */
 321                 wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 322                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 323         }
 324         _debug("<<< GO");
 325
 326 check_if_dead:
 327         if (unlikely(fscache_object_is_dead(object))) {
 328                 fscache_stat(stat_object_dead);
 329                 return -ENOBUFS;
 330         }
 331         return 0;
 332 }
 333
 334 /*
 335  * read a page from the cache or allocate a block in which to store it
 336  * - we return:
 337  *   -ENOMEM    - out of memory, nothing done
 338  *   -ERESTARTSYS - interrupted
 339  *   -ENOBUFS   - no backing object available in which to cache the block
 340  *   -ENODATA   - no data available in the backing object for this block
 341  *   0          - dispatched a read - it'll call end_io_func() when finished
 342  */
 343 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 344                                  struct page *page,
 345                                  fscache_rw_complete_t end_io_func,
 346                                  void *context,
 347                                  gfp_t gfp)
 348 {
 349         struct fscache_retrieval *op;
 350         struct fscache_object *object;
 351         int ret;
 352
 353         _enter("%p,%p,,,", cookie, page);
 354
 355         fscache_stat(&fscache_n_retrievals);
 356
 357         if (hlist_empty(&cookie->backing_objects))
 358                 goto nobufs;
 359
 360         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 361         ASSERTCMP(page, !=, NULL);
 362
 363         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 364                 return -ERESTARTSYS;
 365
 366         op = fscache_alloc_retrieval(page->mapping, end_io_func, context);
 367         if (!op) {
 368                 _leave(" = -ENOMEM");
 369                 return -ENOMEM;
 370         }
 371         fscache_set_op_name(&op->op, "RetrRA1");
 372
 373         spin_lock(&cookie->lock);
 374
 375         if (hlist_empty(&cookie->backing_objects))
 376                 goto nobufs_unlock;
 377         object = hlist_entry(cookie->backing_objects.first,
 378                              struct fscache_object, cookie_link);
 379
 380         ASSERTCMP(object->state, >, FSCACHE_OBJECT_LOOKING_UP);
 381
 382         atomic_inc(&object->n_reads);
 383         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 384
 385         if (fscache_submit_op(object, &op->op) < 0)
 386                 goto nobufs_unlock;
 387         spin_unlock(&cookie->lock);
 388
 389         fscache_stat(&fscache_n_retrieval_ops);
 390
 391         /* pin the netfs read context in case we need to do the actual netfs
 392          * read because we've encountered a cache read failure */
 393         fscache_get_context(object->cookie, op->context);
 394
 395         /* we wait for the operation to become active, and then process it
 396          * *here*, in this thread, and not in the thread pool */
 397         ret = fscache_wait_for_retrieval_activation(
 398                 object, op,
 399                 __fscache_stat(&fscache_n_retrieval_op_waits),
 400                 __fscache_stat(&fscache_n_retrievals_object_dead));
 401         if (ret < 0)
 402                 goto error;
 403
 404         /* ask the cache to honour the operation */
 405         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 406                 fscache_stat(&fscache_n_cop_allocate_page);
 407                 ret = object->cache->ops->allocate_page(op, page, gfp);
 408                 fscache_stat_d(&fscache_n_cop_allocate_page);
 409                 if (ret == 0)
 410                         ret = -ENODATA;
 411         } else {
 412                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
 413                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 414                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
 415         }
 416
 417 error:
 418         if (ret == -ENOMEM)
 419                 fscache_stat(&fscache_n_retrievals_nomem);
 420         else if (ret == -ERESTARTSYS)
 421                 fscache_stat(&fscache_n_retrievals_intr);
 422         else if (ret == -ENODATA)
 423                 fscache_stat(&fscache_n_retrievals_nodata);
 424         else if (ret < 0)
 425                 fscache_stat(&fscache_n_retrievals_nobufs);
 426         else
 427                 fscache_stat(&fscache_n_retrievals_ok);
 428
 429         fscache_put_retrieval(op);
 430         _leave(" = %d", ret);
 431         return ret;
 432
 433 nobufs_unlock:
 434         spin_unlock(&cookie->lock);
 435         kfree(op);
 436 nobufs:
 437         fscache_stat(&fscache_n_retrievals_nobufs);
 438         _leave(" = -ENOBUFS");
 439         return -ENOBUFS;
 440 }
 441 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 442
 443 /*
 444  * read a list of page from the cache or allocate a block in which to store
 445  * them
 446  * - we return:
 447  *   -ENOMEM    - out of memory, some pages may be being read
 448  *   -ERESTARTSYS - interrupted, some pages may be being read
 449  *   -ENOBUFS   - no backing object or space available in which to cache any
 450  *                pages not being read
 451  *   -ENODATA   - no data available in the backing object for some or all of
 452  *                the pages
 453  *   0          - dispatched a read on all pages
 454  *
 455  * end_io_func() will be called for each page read from the cache as it is
 456  * finishes being read
 457  *
 458  * any pages for which a read is dispatched will be removed from pages and
 459  * nr_pages
 460  */
 461 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 462                                   struct address_space *mapping,
 463                                   struct list_head *pages,
 464                                   unsigned *nr_pages,
 465                                   fscache_rw_complete_t end_io_func,
 466                                   void *context,
 467                                   gfp_t gfp)
 468 {
 469         struct fscache_retrieval *op;
 470         struct fscache_object *object;
 471         int ret;
 472
 473         _enter("%p,,%d,,,", cookie, *nr_pages);
 474
 475         fscache_stat(&fscache_n_retrievals);
 476
 477         if (hlist_empty(&cookie->backing_objects))
 478                 goto nobufs;
 479
 480         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 481         ASSERTCMP(*nr_pages, >, 0);
 482         ASSERT(!list_empty(pages));
 483
 484         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 485                 return -ERESTARTSYS;
 486
 487         op = fscache_alloc_retrieval(mapping, end_io_func, context);
 488         if (!op)
 489                 return -ENOMEM;
 490         fscache_set_op_name(&op->op, "RetrRAN");
 491
 492         spin_lock(&cookie->lock);
 493
 494         if (hlist_empty(&cookie->backing_objects))
 495                 goto nobufs_unlock;
 496         object = hlist_entry(cookie->backing_objects.first,
 497                              struct fscache_object, cookie_link);
 498
 499         atomic_inc(&object->n_reads);
 500         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 501
 502         if (fscache_submit_op(object, &op->op) < 0)
 503                 goto nobufs_unlock;
 504         spin_unlock(&cookie->lock);
 505
 506         fscache_stat(&fscache_n_retrieval_ops);
 507
 508         /* pin the netfs read context in case we need to do the actual netfs
 509          * read because we've encountered a cache read failure */
 510         fscache_get_context(object->cookie, op->context);
 511
 512         /* we wait for the operation to become active, and then process it
 513          * *here*, in this thread, and not in the thread pool */
 514         ret = fscache_wait_for_retrieval_activation(
 515                 object, op,
 516                 __fscache_stat(&fscache_n_retrieval_op_waits),
 517                 __fscache_stat(&fscache_n_retrievals_object_dead));
 518         if (ret < 0)
 519                 goto error;
 520
 521         /* ask the cache to honour the operation */
 522         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 523                 fscache_stat(&fscache_n_cop_allocate_pages);
 524                 ret = object->cache->ops->allocate_pages(
 525                         op, pages, nr_pages, gfp);
 526                 fscache_stat_d(&fscache_n_cop_allocate_pages);
 527         } else {
 528                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
 529                 ret = object->cache->ops->read_or_alloc_pages(
 530                         op, pages, nr_pages, gfp);
 531                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
 532         }
 533
 534 error:
 535         if (ret == -ENOMEM)
 536                 fscache_stat(&fscache_n_retrievals_nomem);
 537         else if (ret == -ERESTARTSYS)
 538                 fscache_stat(&fscache_n_retrievals_intr);
 539         else if (ret == -ENODATA)
 540                 fscache_stat(&fscache_n_retrievals_nodata);
 541         else if (ret < 0)
 542                 fscache_stat(&fscache_n_retrievals_nobufs);
 543         else
 544                 fscache_stat(&fscache_n_retrievals_ok);
 545
 546         fscache_put_retrieval(op);
 547         _leave(" = %d", ret);
 548         return ret;
 549
 550 nobufs_unlock:
 551         spin_unlock(&cookie->lock);
 552         kfree(op);
 553 nobufs:
 554         fscache_stat(&fscache_n_retrievals_nobufs);
 555         _leave(" = -ENOBUFS");
 556         return -ENOBUFS;
 557 }
 558 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 559
 560 /*
 561  * allocate a block in the cache on which to store a page
 562  * - we return:
 563  *   -ENOMEM    - out of memory, nothing done
 564  *   -ERESTARTSYS - interrupted
 565  *   -ENOBUFS   - no backing object available in which to cache the block
 566  *   0          - block allocated
 567  */
 568 int __fscache_alloc_page(struct fscache_cookie *cookie,
 569                          struct page *page,
 570                          gfp_t gfp)
 571 {
 572         struct fscache_retrieval *op;
 573         struct fscache_object *object;
 574         int ret;
 575
 576         _enter("%p,%p,,,", cookie, page);
 577
 578         fscache_stat(&fscache_n_allocs);
 579
 580         if (hlist_empty(&cookie->backing_objects))
 581                 goto nobufs;
 582
 583         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 584         ASSERTCMP(page, !=, NULL);
 585
 586         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 587                 return -ERESTARTSYS;
 588
 589         op = fscache_alloc_retrieval(page->mapping, NULL, NULL);
 590         if (!op)
 591                 return -ENOMEM;
 592         fscache_set_op_name(&op->op, "RetrAL1");
 593
 594         spin_lock(&cookie->lock);
 595
 596         if (hlist_empty(&cookie->backing_objects))
 597                 goto nobufs_unlock;
 598         object = hlist_entry(cookie->backing_objects.first,
 599                              struct fscache_object, cookie_link);
 600
 601         if (fscache_submit_op(object, &op->op) < 0)
 602                 goto nobufs_unlock;
 603         spin_unlock(&cookie->lock);
 604
 605         fscache_stat(&fscache_n_alloc_ops);
 606
 607         ret = fscache_wait_for_retrieval_activation(
 608                 object, op,
 609                 __fscache_stat(&fscache_n_alloc_op_waits),
 610                 __fscache_stat(&fscache_n_allocs_object_dead));
 611         if (ret < 0)
 612                 goto error;
 613
 614         /* ask the cache to honour the operation */
 615         fscache_stat(&fscache_n_cop_allocate_page);
 616         ret = object->cache->ops->allocate_page(op, page, gfp);
 617         fscache_stat_d(&fscache_n_cop_allocate_page);
 618
 619 error:
 620         if (ret == -ERESTARTSYS)
 621                 fscache_stat(&fscache_n_allocs_intr);
 622         else if (ret < 0)
 623                 fscache_stat(&fscache_n_allocs_nobufs);
 624         else
 625                 fscache_stat(&fscache_n_allocs_ok);
 626
 627         fscache_put_retrieval(op);
 628         _leave(" = %d", ret);
 629         return ret;
 630
 631 nobufs_unlock:
 632         spin_unlock(&cookie->lock);
 633         kfree(op);
 634 nobufs:
 635         fscache_stat(&fscache_n_allocs_nobufs);
 636         _leave(" = -ENOBUFS");
 637         return -ENOBUFS;
 638 }
 639 EXPORT_SYMBOL(__fscache_alloc_page);
 640
 641 /*
 642  * release a write op reference
 643  */
 644 static void fscache_release_write_op(struct fscache_operation *_op)
 645 {
 646         _enter("{OP%x}", _op->debug_id);
 647 }
 648
 649 /*
 650  * perform the background storage of a page into the cache
 651  */
 652 static void fscache_write_op(struct fscache_operation *_op)
 653 {
 654         struct fscache_storage *op =
 655                 container_of(_op, struct fscache_storage, op);
 656         struct fscache_object *object = op->op.object;
 657         struct fscache_cookie *cookie;
 658         struct page *page;
 659         unsigned n;
 660         void *results[1];
 661         int ret;
 662
 663         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 664
 665         fscache_set_op_state(&op->op, "GetPage");
 666
 667         spin_lock(&object->lock);
 668         cookie = object->cookie;
 669
 670         if (!fscache_object_is_active(object) || !cookie) {
 671                 spin_unlock(&object->lock);
 672                 _leave("");
 673                 return;
 674         }
 675
 676         spin_lock(&cookie->stores_lock);
 677
 678         fscache_stat(&fscache_n_store_calls);
 679
 680         /* find a page to store */
 681         page = NULL;
 682         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 683                                        FSCACHE_COOKIE_PENDING_TAG);
 684         if (n != 1)
 685                 goto superseded;
 686         page = results[0];
 687         _debug("gang %d [%lx]", n, page->index);
 688         if (page->index > op->store_limit) {
 689                 fscache_stat(&fscache_n_store_pages_over_limit);
 690                 goto superseded;
 691         }
 692
 693         radix_tree_tag_set(&cookie->stores, page->index,
 694                            FSCACHE_COOKIE_STORING_TAG);
 695         radix_tree_tag_clear(&cookie->stores, page->index,
 696                              FSCACHE_COOKIE_PENDING_TAG);
 697
 698         spin_unlock(&cookie->stores_lock);
 699         spin_unlock(&object->lock);
 700
 701         fscache_set_op_state(&op->op, "Store");
 702         fscache_stat(&fscache_n_store_pages);
 703         fscache_stat(&fscache_n_cop_write_page);
 704         ret = object->cache->ops->write_page(op, page);
 705         fscache_stat_d(&fscache_n_cop_write_page);
 706         fscache_set_op_state(&op->op, "EndWrite");
 707         fscache_end_page_write(object, page);
 708         if (ret < 0) {
 709                 fscache_set_op_state(&op->op, "Abort");
 710                 fscache_abort_object(object);
 711         } else {
 712                 fscache_enqueue_operation(&op->op);
 713         }
 714
 715         _leave("");
 716         return;
 717
 718 superseded:
 719         /* this writer is going away and there aren't any more things to
 720          * write */
 721         _debug("cease");
 722         spin_unlock(&cookie->stores_lock);
 723         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 724         spin_unlock(&object->lock);
 725         _leave("");
 726 }
 727
 728 /*
 729  * request a page be stored in the cache
 730  * - returns:
 731  *   -ENOMEM    - out of memory, nothing done
 732  *   -ENOBUFS   - no backing object available in which to cache the page
 733  *   0          - dispatched a write - it'll call end_io_func() when finished
 734  *
 735  * if the cookie still has a backing object at this point, that object can be
 736  * in one of a few states with respect to storage processing:
 737  *
 738  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 739  *      set)
 740  *
 741  *      (a) no writes yet (set FSCACHE_COOKIE_PENDING_FILL and queue deferred
 742  *          fill op)
 743  *
 744  *      (b) writes deferred till post-creation (mark page for writing and
 745  *          return immediately)
 746  *
 747  *  (2) negative lookup, object created, initial fill being made from netfs
 748  *      (FSCACHE_COOKIE_INITIAL_FILL is set)
 749  *
 750  *      (a) fill point not yet reached this page (mark page for writing and
 751  *          return)
 752  *
 753  *      (b) fill point passed this page (queue op to store this page)
 754  *
 755  *  (3) object extant (queue op to store this page)
 756  *
 757  * any other state is invalid
 758  */
 759 int __fscache_write_page(struct fscache_cookie *cookie,
 760                          struct page *page,
 761                          gfp_t gfp)
 762 {
 763         struct fscache_storage *op;
 764         struct fscache_object *object;
 765         int ret;
 766
 767         _enter("%p,%x,", cookie, (u32) page->flags);
 768
 769         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 770         ASSERT(PageFsCache(page));
 771
 772         fscache_stat(&fscache_n_stores);
 773
 774         op = kzalloc(sizeof(*op), GFP_NOIO);
 775         if (!op)
 776                 goto nomem;
 777
 778         fscache_operation_init(&op->op, fscache_write_op,
 779                                fscache_release_write_op);
 780         op->op.flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_WAITING);
 781         fscache_set_op_name(&op->op, "Write1");
 782
 783         ret = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
 784         if (ret < 0)
 785                 goto nomem_free;
 786
 787         ret = -ENOBUFS;
 788         spin_lock(&cookie->lock);
 789
 790         if (hlist_empty(&cookie->backing_objects))
 791                 goto nobufs;
 792         object = hlist_entry(cookie->backing_objects.first,
 793                              struct fscache_object, cookie_link);
 794         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 795                 goto nobufs;
 796
 797         /* add the page to the pending-storage radix tree on the backing
 798          * object */
 799         spin_lock(&object->lock);
 800         spin_lock(&cookie->stores_lock);
 801
 802         _debug("store limit %llx", (unsigned long long) object->store_limit);
 803
 804         ret = radix_tree_insert(&cookie->stores, page->index, page);
 805         if (ret < 0) {
 806                 if (ret == -EEXIST)
 807                         goto already_queued;
 808                 _debug("insert failed %d", ret);
 809                 goto nobufs_unlock_obj;
 810         }
 811
 812         radix_tree_tag_set(&cookie->stores, page->index,
 813                            FSCACHE_COOKIE_PENDING_TAG);
 814         page_cache_get(page);
 815
 816         /* we only want one writer at a time, but we do need to queue new
 817          * writers after exclusive ops */
 818         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 819                 goto already_pending;
 820
 821         spin_unlock(&cookie->stores_lock);
 822         spin_unlock(&object->lock);
 823
 824         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 825         op->store_limit = object->store_limit;
 826
 827         if (fscache_submit_op(object, &op->op) < 0)
 828                 goto submit_failed;
 829
 830         spin_unlock(&cookie->lock);
 831         radix_tree_preload_end();
 832         fscache_stat(&fscache_n_store_ops);
 833         fscache_stat(&fscache_n_stores_ok);
 834
 835         /* the work queue now carries its own ref on the object */
 836         fscache_put_operation(&op->op);
 837         _leave(" = 0");
 838         return 0;
 839
 840 already_queued:
 841         fscache_stat(&fscache_n_stores_again);
 842 already_pending:
 843         spin_unlock(&cookie->stores_lock);
 844         spin_unlock(&object->lock);
 845         spin_unlock(&cookie->lock);
 846         radix_tree_preload_end();
 847         kfree(op);
 848         fscache_stat(&fscache_n_stores_ok);
 849         _leave(" = 0");
 850         return 0;
 851
 852 submit_failed:
 853         spin_lock(&cookie->stores_lock);
 854         radix_tree_delete(&cookie->stores, page->index);
 855         spin_unlock(&cookie->stores_lock);
 856         page_cache_release(page);
 857         ret = -ENOBUFS;
 858         goto nobufs;
 859
 860 nobufs_unlock_obj:
 861         spin_unlock(&cookie->stores_lock);
 862         spin_unlock(&object->lock);
 863 nobufs:
 864         spin_unlock(&cookie->lock);
 865         radix_tree_preload_end();
 866         kfree(op);
 867         fscache_stat(&fscache_n_stores_nobufs);
 868         _leave(" = -ENOBUFS");
 869         return -ENOBUFS;
 870
 871 nomem_free:
 872         kfree(op);
 873 nomem:
 874         fscache_stat(&fscache_n_stores_oom);
 875         _leave(" = -ENOMEM");
 876         return -ENOMEM;
 877 }
 878 EXPORT_SYMBOL(__fscache_write_page);
 879
 880 /*
 881  * remove a page from the cache
 882  */
 883 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
 884 {
 885         struct fscache_object *object;
 886
 887         _enter(",%p", page);
 888
 889         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 890         ASSERTCMP(page, !=, NULL);
 891
 892         fscache_stat(&fscache_n_uncaches);
 893
 894         /* cache withdrawal may beat us to it */
 895         if (!PageFsCache(page))
 896                 goto done;
 897
 898         /* get the object */
 899         spin_lock(&cookie->lock);
 900
 901         if (hlist_empty(&cookie->backing_objects)) {
 902                 ClearPageFsCache(page);
 903                 goto done_unlock;
 904         }
 905
 906         object = hlist_entry(cookie->backing_objects.first,
 907                              struct fscache_object, cookie_link);
 908
 909         /* there might now be stuff on disk we could read */
 910         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 911
 912         /* only invoke the cache backend if we managed to mark the page
 913          * uncached here; this deals with synchronisation vs withdrawal */
 914         if (TestClearPageFsCache(page) &&
 915             object->cache->ops->uncache_page) {
 916                 /* the cache backend releases the cookie lock */
 917                 fscache_stat(&fscache_n_cop_uncache_page);
 918                 object->cache->ops->uncache_page(object, page);
 919                 fscache_stat_d(&fscache_n_cop_uncache_page);
 920                 goto done;
 921         }
 922
 923 done_unlock:
 924         spin_unlock(&cookie->lock);
 925 done:
 926         _leave("");
 927 }
 928 EXPORT_SYMBOL(__fscache_uncache_page);
 929
 930 /**
 931  * fscache_mark_pages_cached - Mark pages as being cached
 932  * @op: The retrieval op pages are being marked for
 933  * @pagevec: The pages to be marked
 934  *
 935  * Mark a bunch of netfs pages as being cached.  After this is called,
 936  * the netfs must call fscache_uncache_page() to remove the mark.
 937  */
 938 void fscache_mark_pages_cached(struct fscache_retrieval *op,
 939                                struct pagevec *pagevec)
 940 {
 941         struct fscache_cookie *cookie = op->op.object->cookie;
 942         unsigned long loop;
 943
 944 #ifdef CONFIG_FSCACHE_STATS
 945         atomic_add(pagevec->nr, &fscache_n_marks);
 946 #endif
 947
 948         for (loop = 0; loop < pagevec->nr; loop++) {
 949                 struct page *page = pagevec->pages[loop];
 950
 951                 _debug("- mark %p{%lx}", page, page->index);
 952                 if (TestSetPageFsCache(page)) {
 953                         static bool once_only;
 954                         if (!once_only) {
 955                                 once_only = true;
 956                                 printk(KERN_WARNING "FS-Cache:"
 957                                        " Cookie type %s marked page %lx"
 958                                        " multiple times\n",
 959                                        cookie->def->name, page->index);
 960                         }
 961                 }
 962         }
 963
 964         if (cookie->def->mark_pages_cached)
 965                 cookie->def->mark_pages_cached(cookie->netfs_data,
 966                                                op->mapping, pagevec);
 967         pagevec_reinit(pagevec);
 968 }
 969 EXPORT_SYMBOL(fscache_mark_pages_cached);