4 * Write file data over NFS.
6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
9 #include <linux/types.h>
10 #include <linux/slab.h>
12 #include <linux/pagemap.h>
13 #include <linux/file.h>
14 #include <linux/writeback.h>
15 #include <linux/swap.h>
17 #include <linux/sunrpc/clnt.h>
18 #include <linux/nfs_fs.h>
19 #include <linux/nfs_mount.h>
20 #include <linux/nfs_page.h>
21 #include <linux/backing-dev.h>
23 #include <asm/uaccess.h>
25 #include "delegation.h"
29 #define NFSDBG_FACILITY NFSDBG_PAGECACHE
31 #define MIN_POOL_WRITE (32)
32 #define MIN_POOL_COMMIT (4)
35 * Local function declarations
37 static struct nfs_page
* nfs_update_request(struct nfs_open_context
*,
39 unsigned int, unsigned int);
40 static void nfs_pageio_init_write(struct nfs_pageio_descriptor
*desc
,
41 struct inode
*inode
, int ioflags
);
42 static const struct rpc_call_ops nfs_write_partial_ops
;
43 static const struct rpc_call_ops nfs_write_full_ops
;
44 static const struct rpc_call_ops nfs_commit_ops
;
46 static struct kmem_cache
*nfs_wdata_cachep
;
47 static mempool_t
*nfs_wdata_mempool
;
48 static mempool_t
*nfs_commit_mempool
;
50 struct nfs_write_data
*nfs_commit_alloc(void)
52 struct nfs_write_data
*p
= mempool_alloc(nfs_commit_mempool
, GFP_NOFS
);
55 memset(p
, 0, sizeof(*p
));
56 INIT_LIST_HEAD(&p
->pages
);
61 static void nfs_commit_rcu_free(struct rcu_head
*head
)
63 struct nfs_write_data
*p
= container_of(head
, struct nfs_write_data
, task
.u
.tk_rcu
);
64 if (p
&& (p
->pagevec
!= &p
->page_array
[0]))
66 mempool_free(p
, nfs_commit_mempool
);
69 void nfs_commit_free(struct nfs_write_data
*wdata
)
71 call_rcu_bh(&wdata
->task
.u
.tk_rcu
, nfs_commit_rcu_free
);
74 struct nfs_write_data
*nfs_writedata_alloc(unsigned int pagecount
)
76 struct nfs_write_data
*p
= mempool_alloc(nfs_wdata_mempool
, GFP_NOFS
);
79 memset(p
, 0, sizeof(*p
));
80 INIT_LIST_HEAD(&p
->pages
);
81 p
->npages
= pagecount
;
82 if (pagecount
<= ARRAY_SIZE(p
->page_array
))
83 p
->pagevec
= p
->page_array
;
85 p
->pagevec
= kcalloc(pagecount
, sizeof(struct page
*), GFP_NOFS
);
87 mempool_free(p
, nfs_wdata_mempool
);
95 static void nfs_writedata_rcu_free(struct rcu_head
*head
)
97 struct nfs_write_data
*p
= container_of(head
, struct nfs_write_data
, task
.u
.tk_rcu
);
98 if (p
&& (p
->pagevec
!= &p
->page_array
[0]))
100 mempool_free(p
, nfs_wdata_mempool
);
103 static void nfs_writedata_free(struct nfs_write_data
*wdata
)
105 call_rcu_bh(&wdata
->task
.u
.tk_rcu
, nfs_writedata_rcu_free
);
108 void nfs_writedata_release(void *wdata
)
110 nfs_writedata_free(wdata
);
113 static void nfs_context_set_write_error(struct nfs_open_context
*ctx
, int error
)
117 set_bit(NFS_CONTEXT_ERROR_WRITE
, &ctx
->flags
);
120 static struct nfs_page
*nfs_page_find_request_locked(struct page
*page
)
122 struct nfs_page
*req
= NULL
;
124 if (PagePrivate(page
)) {
125 req
= (struct nfs_page
*)page_private(page
);
127 kref_get(&req
->wb_kref
);
132 static struct nfs_page
*nfs_page_find_request(struct page
*page
)
134 struct inode
*inode
= page
->mapping
->host
;
135 struct nfs_page
*req
= NULL
;
137 spin_lock(&inode
->i_lock
);
138 req
= nfs_page_find_request_locked(page
);
139 spin_unlock(&inode
->i_lock
);
143 /* Adjust the file length if we're writing beyond the end */
144 static void nfs_grow_file(struct page
*page
, unsigned int offset
, unsigned int count
)
146 struct inode
*inode
= page
->mapping
->host
;
147 loff_t end
, i_size
= i_size_read(inode
);
148 pgoff_t end_index
= (i_size
- 1) >> PAGE_CACHE_SHIFT
;
150 if (i_size
> 0 && page
->index
< end_index
)
152 end
= ((loff_t
)page
->index
<< PAGE_CACHE_SHIFT
) + ((loff_t
)offset
+count
);
155 nfs_inc_stats(inode
, NFSIOS_EXTENDWRITE
);
156 i_size_write(inode
, end
);
159 /* A writeback failed: mark the page as bad, and invalidate the page cache */
160 static void nfs_set_pageerror(struct page
*page
)
163 nfs_zap_mapping(page
->mapping
->host
, page
->mapping
);
166 /* We can set the PG_uptodate flag if we see that a write request
167 * covers the full page.
169 static void nfs_mark_uptodate(struct page
*page
, unsigned int base
, unsigned int count
)
171 if (PageUptodate(page
))
175 if (count
!= nfs_page_length(page
))
177 SetPageUptodate(page
);
180 static int nfs_writepage_setup(struct nfs_open_context
*ctx
, struct page
*page
,
181 unsigned int offset
, unsigned int count
)
183 struct nfs_page
*req
;
187 req
= nfs_update_request(ctx
, page
, offset
, count
);
193 ret
= nfs_wb_page(page
->mapping
->host
, page
);
197 /* Update file length */
198 nfs_grow_file(page
, offset
, count
);
199 nfs_clear_page_tag_locked(req
);
203 static int wb_priority(struct writeback_control
*wbc
)
205 if (wbc
->for_reclaim
)
206 return FLUSH_HIGHPRI
| FLUSH_STABLE
;
207 if (wbc
->for_kupdate
)
213 * NFS congestion control
216 int nfs_congestion_kb
;
218 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))
219 #define NFS_CONGESTION_OFF_THRESH \
220 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
222 static int nfs_set_page_writeback(struct page
*page
)
224 int ret
= test_set_page_writeback(page
);
227 struct inode
*inode
= page
->mapping
->host
;
228 struct nfs_server
*nfss
= NFS_SERVER(inode
);
230 if (atomic_long_inc_return(&nfss
->writeback
) >
231 NFS_CONGESTION_ON_THRESH
)
232 set_bdi_congested(&nfss
->backing_dev_info
, WRITE
);
237 static void nfs_end_page_writeback(struct page
*page
)
239 struct inode
*inode
= page
->mapping
->host
;
240 struct nfs_server
*nfss
= NFS_SERVER(inode
);
242 end_page_writeback(page
);
243 if (atomic_long_dec_return(&nfss
->writeback
) < NFS_CONGESTION_OFF_THRESH
)
244 clear_bdi_congested(&nfss
->backing_dev_info
, WRITE
);
248 * Find an associated nfs write request, and prepare to flush it out
249 * May return an error if the user signalled nfs_wait_on_request().
251 static int nfs_page_async_flush(struct nfs_pageio_descriptor
*pgio
,
254 struct inode
*inode
= page
->mapping
->host
;
255 struct nfs_page
*req
;
258 spin_lock(&inode
->i_lock
);
260 req
= nfs_page_find_request_locked(page
);
262 spin_unlock(&inode
->i_lock
);
265 if (nfs_set_page_tag_locked(req
))
267 /* Note: If we hold the page lock, as is the case in nfs_writepage,
268 * then the call to nfs_set_page_tag_locked() will always
269 * succeed provided that someone hasn't already marked the
270 * request as dirty (in which case we don't care).
272 spin_unlock(&inode
->i_lock
);
273 ret
= nfs_wait_on_request(req
);
274 nfs_release_request(req
);
277 spin_lock(&inode
->i_lock
);
279 if (test_bit(PG_NEED_COMMIT
, &req
->wb_flags
)) {
280 /* This request is marked for commit */
281 spin_unlock(&inode
->i_lock
);
282 nfs_clear_page_tag_locked(req
);
283 nfs_pageio_complete(pgio
);
286 if (nfs_set_page_writeback(page
) != 0) {
287 spin_unlock(&inode
->i_lock
);
290 spin_unlock(&inode
->i_lock
);
291 nfs_pageio_add_request(pgio
, req
);
295 static int nfs_do_writepage(struct page
*page
, struct writeback_control
*wbc
, struct nfs_pageio_descriptor
*pgio
)
297 struct inode
*inode
= page
->mapping
->host
;
299 nfs_inc_stats(inode
, NFSIOS_VFSWRITEPAGE
);
300 nfs_add_stats(inode
, NFSIOS_WRITEPAGES
, 1);
302 nfs_pageio_cond_complete(pgio
, page
->index
);
303 return nfs_page_async_flush(pgio
, page
);
307 * Write an mmapped page to the server.
309 static int nfs_writepage_locked(struct page
*page
, struct writeback_control
*wbc
)
311 struct nfs_pageio_descriptor pgio
;
314 nfs_pageio_init_write(&pgio
, page
->mapping
->host
, wb_priority(wbc
));
315 err
= nfs_do_writepage(page
, wbc
, &pgio
);
316 nfs_pageio_complete(&pgio
);
319 if (pgio
.pg_error
< 0)
320 return pgio
.pg_error
;
324 int nfs_writepage(struct page
*page
, struct writeback_control
*wbc
)
328 ret
= nfs_writepage_locked(page
, wbc
);
333 static int nfs_writepages_callback(struct page
*page
, struct writeback_control
*wbc
, void *data
)
337 ret
= nfs_do_writepage(page
, wbc
, data
);
342 int nfs_writepages(struct address_space
*mapping
, struct writeback_control
*wbc
)
344 struct inode
*inode
= mapping
->host
;
345 struct nfs_pageio_descriptor pgio
;
348 nfs_inc_stats(inode
, NFSIOS_VFSWRITEPAGES
);
350 nfs_pageio_init_write(&pgio
, inode
, wb_priority(wbc
));
351 err
= write_cache_pages(mapping
, wbc
, nfs_writepages_callback
, &pgio
);
352 nfs_pageio_complete(&pgio
);
355 if (pgio
.pg_error
< 0)
356 return pgio
.pg_error
;
361 * Insert a write request into an inode
363 static int nfs_inode_add_request(struct inode
*inode
, struct nfs_page
*req
)
365 struct nfs_inode
*nfsi
= NFS_I(inode
);
368 error
= radix_tree_insert(&nfsi
->nfs_page_tree
, req
->wb_index
, req
);
369 BUG_ON(error
== -EEXIST
);
374 if (nfs_have_delegation(inode
, FMODE_WRITE
))
377 SetPagePrivate(req
->wb_page
);
378 set_page_private(req
->wb_page
, (unsigned long)req
);
380 kref_get(&req
->wb_kref
);
381 radix_tree_tag_set(&nfsi
->nfs_page_tree
, req
->wb_index
, NFS_PAGE_TAG_LOCKED
);
386 * Remove a write request from an inode
388 static void nfs_inode_remove_request(struct nfs_page
*req
)
390 struct inode
*inode
= req
->wb_context
->path
.dentry
->d_inode
;
391 struct nfs_inode
*nfsi
= NFS_I(inode
);
393 BUG_ON (!NFS_WBACK_BUSY(req
));
395 spin_lock(&inode
->i_lock
);
396 set_page_private(req
->wb_page
, 0);
397 ClearPagePrivate(req
->wb_page
);
398 radix_tree_delete(&nfsi
->nfs_page_tree
, req
->wb_index
);
401 spin_unlock(&inode
->i_lock
);
404 spin_unlock(&inode
->i_lock
);
405 nfs_clear_request(req
);
406 nfs_release_request(req
);
410 nfs_redirty_request(struct nfs_page
*req
)
412 __set_page_dirty_nobuffers(req
->wb_page
);
416 * Check if a request is dirty
419 nfs_dirty_request(struct nfs_page
*req
)
421 struct page
*page
= req
->wb_page
;
423 if (page
== NULL
|| test_bit(PG_NEED_COMMIT
, &req
->wb_flags
))
425 return !PageWriteback(req
->wb_page
);
428 #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
430 * Add a request to the inode's commit list.
433 nfs_mark_request_commit(struct nfs_page
*req
)
435 struct inode
*inode
= req
->wb_context
->path
.dentry
->d_inode
;
436 struct nfs_inode
*nfsi
= NFS_I(inode
);
438 spin_lock(&inode
->i_lock
);
440 set_bit(PG_NEED_COMMIT
, &(req
)->wb_flags
);
441 radix_tree_tag_set(&nfsi
->nfs_page_tree
,
443 NFS_PAGE_TAG_COMMIT
);
444 spin_unlock(&inode
->i_lock
);
445 inc_zone_page_state(req
->wb_page
, NR_UNSTABLE_NFS
);
446 inc_bdi_stat(req
->wb_page
->mapping
->backing_dev_info
, BDI_RECLAIMABLE
);
447 __mark_inode_dirty(inode
, I_DIRTY_DATASYNC
);
451 int nfs_write_need_commit(struct nfs_write_data
*data
)
453 return data
->verf
.committed
!= NFS_FILE_SYNC
;
457 int nfs_reschedule_unstable_write(struct nfs_page
*req
)
459 if (test_bit(PG_NEED_COMMIT
, &req
->wb_flags
)) {
460 nfs_mark_request_commit(req
);
463 if (test_and_clear_bit(PG_NEED_RESCHED
, &req
->wb_flags
)) {
464 nfs_redirty_request(req
);
471 nfs_mark_request_commit(struct nfs_page
*req
)
476 int nfs_write_need_commit(struct nfs_write_data
*data
)
482 int nfs_reschedule_unstable_write(struct nfs_page
*req
)
489 * Wait for a request to complete.
491 * Interruptible by fatal signals only.
493 static int nfs_wait_on_requests_locked(struct inode
*inode
, pgoff_t idx_start
, unsigned int npages
)
495 struct nfs_inode
*nfsi
= NFS_I(inode
);
496 struct nfs_page
*req
;
497 pgoff_t idx_end
, next
;
498 unsigned int res
= 0;
504 idx_end
= idx_start
+ npages
- 1;
507 while (radix_tree_gang_lookup_tag(&nfsi
->nfs_page_tree
, (void **)&req
, next
, 1, NFS_PAGE_TAG_LOCKED
)) {
508 if (req
->wb_index
> idx_end
)
511 next
= req
->wb_index
+ 1;
512 BUG_ON(!NFS_WBACK_BUSY(req
));
514 kref_get(&req
->wb_kref
);
515 spin_unlock(&inode
->i_lock
);
516 error
= nfs_wait_on_request(req
);
517 nfs_release_request(req
);
518 spin_lock(&inode
->i_lock
);
526 static void nfs_cancel_commit_list(struct list_head
*head
)
528 struct nfs_page
*req
;
530 while(!list_empty(head
)) {
531 req
= nfs_list_entry(head
->next
);
532 dec_zone_page_state(req
->wb_page
, NR_UNSTABLE_NFS
);
533 dec_bdi_stat(req
->wb_page
->mapping
->backing_dev_info
,
535 nfs_list_remove_request(req
);
536 clear_bit(PG_NEED_COMMIT
, &(req
)->wb_flags
);
537 nfs_inode_remove_request(req
);
538 nfs_unlock_request(req
);
542 #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
544 * nfs_scan_commit - Scan an inode for commit requests
545 * @inode: NFS inode to scan
546 * @dst: destination list
547 * @idx_start: lower bound of page->index to scan.
548 * @npages: idx_start + npages sets the upper bound to scan.
550 * Moves requests from the inode's 'commit' request list.
551 * The requests are *not* checked to ensure that they form a contiguous set.
554 nfs_scan_commit(struct inode
*inode
, struct list_head
*dst
, pgoff_t idx_start
, unsigned int npages
)
556 struct nfs_inode
*nfsi
= NFS_I(inode
);
559 if (nfsi
->ncommit
!= 0) {
560 res
= nfs_scan_list(nfsi
, dst
, idx_start
, npages
,
561 NFS_PAGE_TAG_COMMIT
);
562 nfsi
->ncommit
-= res
;
567 static inline int nfs_scan_commit(struct inode
*inode
, struct list_head
*dst
, pgoff_t idx_start
, unsigned int npages
)
574 * Try to update any existing write request, or create one if there is none.
575 * In order to match, the request's credentials must match those of
576 * the calling process.
578 * Note: Should always be called with the Page Lock held!
580 static struct nfs_page
* nfs_update_request(struct nfs_open_context
* ctx
,
581 struct page
*page
, unsigned int offset
, unsigned int bytes
)
583 struct address_space
*mapping
= page
->mapping
;
584 struct inode
*inode
= mapping
->host
;
585 struct nfs_page
*req
, *new = NULL
;
588 end
= offset
+ bytes
;
591 /* Loop over all inode entries and see if we find
592 * A request for the page we wish to update
594 spin_lock(&inode
->i_lock
);
595 req
= nfs_page_find_request_locked(page
);
597 if (!nfs_set_page_tag_locked(req
)) {
600 spin_unlock(&inode
->i_lock
);
601 error
= nfs_wait_on_request(req
);
602 nfs_release_request(req
);
605 nfs_release_request(new);
606 return ERR_PTR(error
);
610 spin_unlock(&inode
->i_lock
);
612 nfs_release_request(new);
618 nfs_lock_request_dontget(new);
619 error
= nfs_inode_add_request(inode
, new);
621 spin_unlock(&inode
->i_lock
);
622 nfs_unlock_request(new);
623 return ERR_PTR(error
);
625 spin_unlock(&inode
->i_lock
);
629 spin_unlock(&inode
->i_lock
);
631 new = nfs_create_request(ctx
, inode
, page
, offset
, bytes
);
636 /* We have a request for our page.
637 * If the creds don't match, or the
638 * page addresses don't match,
639 * tell the caller to wait on the conflicting
642 rqend
= req
->wb_offset
+ req
->wb_bytes
;
643 if (req
->wb_context
!= ctx
644 || req
->wb_page
!= page
645 || !nfs_dirty_request(req
)
646 || offset
> rqend
|| end
< req
->wb_offset
) {
647 nfs_clear_page_tag_locked(req
);
648 return ERR_PTR(-EBUSY
);
651 /* Okay, the request matches. Update the region */
652 if (offset
< req
->wb_offset
) {
653 req
->wb_offset
= offset
;
654 req
->wb_pgbase
= offset
;
655 req
->wb_bytes
= max(end
, rqend
) - req
->wb_offset
;
660 req
->wb_bytes
= end
- req
->wb_offset
;
664 /* If this page might potentially be marked as up to date,
665 * then we need to zero any uninitalised data. */
666 if (req
->wb_pgbase
== 0 && req
->wb_bytes
!= PAGE_CACHE_SIZE
667 && !PageUptodate(req
->wb_page
))
668 zero_user_segment(req
->wb_page
, req
->wb_bytes
, PAGE_CACHE_SIZE
);
672 int nfs_flush_incompatible(struct file
*file
, struct page
*page
)
674 struct nfs_open_context
*ctx
= nfs_file_open_context(file
);
675 struct nfs_page
*req
;
676 int do_flush
, status
;
678 * Look for a request corresponding to this page. If there
679 * is one, and it belongs to another file, we flush it out
680 * before we try to copy anything into the page. Do this
681 * due to the lack of an ACCESS-type call in NFSv2.
682 * Also do the same if we find a request from an existing
686 req
= nfs_page_find_request(page
);
689 do_flush
= req
->wb_page
!= page
|| req
->wb_context
!= ctx
690 || !nfs_dirty_request(req
);
691 nfs_release_request(req
);
694 status
= nfs_wb_page(page
->mapping
->host
, page
);
695 } while (status
== 0);
700 * Update and possibly write a cached page of an NFS file.
702 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
703 * things with a page scheduled for an RPC call (e.g. invalidate it).
705 int nfs_updatepage(struct file
*file
, struct page
*page
,
706 unsigned int offset
, unsigned int count
)
708 struct nfs_open_context
*ctx
= nfs_file_open_context(file
);
709 struct inode
*inode
= page
->mapping
->host
;
712 nfs_inc_stats(inode
, NFSIOS_VFSUPDATEPAGE
);
714 dprintk("NFS: nfs_updatepage(%s/%s %d@%Ld)\n",
715 file
->f_path
.dentry
->d_parent
->d_name
.name
,
716 file
->f_path
.dentry
->d_name
.name
, count
,
717 (long long)(page_offset(page
) +offset
));
719 /* If we're not using byte range locks, and we know the page
720 * is entirely in cache, it may be more efficient to avoid
721 * fragmenting write requests.
723 if (PageUptodate(page
) && inode
->i_flock
== NULL
&& !(file
->f_mode
& O_SYNC
)) {
724 count
= max(count
+ offset
, nfs_page_length(page
));
728 status
= nfs_writepage_setup(ctx
, page
, offset
, count
);
729 __set_page_dirty_nobuffers(page
);
731 dprintk("NFS: nfs_updatepage returns %d (isize %Ld)\n",
732 status
, (long long)i_size_read(inode
));
734 nfs_set_pageerror(page
);
738 static void nfs_writepage_release(struct nfs_page
*req
)
741 if (PageError(req
->wb_page
)) {
742 nfs_end_page_writeback(req
->wb_page
);
743 nfs_inode_remove_request(req
);
744 } else if (!nfs_reschedule_unstable_write(req
)) {
745 /* Set the PG_uptodate flag */
746 nfs_mark_uptodate(req
->wb_page
, req
->wb_pgbase
, req
->wb_bytes
);
747 nfs_end_page_writeback(req
->wb_page
);
748 nfs_inode_remove_request(req
);
750 nfs_end_page_writeback(req
->wb_page
);
751 nfs_clear_page_tag_locked(req
);
754 static int flush_task_priority(int how
)
756 switch (how
& (FLUSH_HIGHPRI
|FLUSH_LOWPRI
)) {
758 return RPC_PRIORITY_HIGH
;
760 return RPC_PRIORITY_LOW
;
762 return RPC_PRIORITY_NORMAL
;
766 * Set up the argument/result storage required for the RPC call.
768 static void nfs_write_rpcsetup(struct nfs_page
*req
,
769 struct nfs_write_data
*data
,
770 const struct rpc_call_ops
*call_ops
,
771 unsigned int count
, unsigned int offset
,
774 struct inode
*inode
= req
->wb_context
->path
.dentry
->d_inode
;
775 int flags
= (how
& FLUSH_SYNC
) ? 0 : RPC_TASK_ASYNC
;
776 int priority
= flush_task_priority(how
);
777 struct rpc_task
*task
;
778 struct rpc_message msg
= {
779 .rpc_argp
= &data
->args
,
780 .rpc_resp
= &data
->res
,
781 .rpc_cred
= req
->wb_context
->cred
,
783 struct rpc_task_setup task_setup_data
= {
784 .rpc_client
= NFS_CLIENT(inode
),
787 .callback_ops
= call_ops
,
788 .callback_data
= data
,
790 .priority
= priority
,
793 /* Set up the RPC argument and reply structs
794 * NB: take care not to mess about with data->commit et al. */
797 data
->inode
= inode
= req
->wb_context
->path
.dentry
->d_inode
;
798 data
->cred
= msg
.rpc_cred
;
800 data
->args
.fh
= NFS_FH(inode
);
801 data
->args
.offset
= req_offset(req
) + offset
;
802 data
->args
.pgbase
= req
->wb_pgbase
+ offset
;
803 data
->args
.pages
= data
->pagevec
;
804 data
->args
.count
= count
;
805 data
->args
.context
= req
->wb_context
;
806 data
->args
.stable
= NFS_UNSTABLE
;
807 if (how
& FLUSH_STABLE
) {
808 data
->args
.stable
= NFS_DATA_SYNC
;
809 if (!NFS_I(inode
)->ncommit
)
810 data
->args
.stable
= NFS_FILE_SYNC
;
813 data
->res
.fattr
= &data
->fattr
;
814 data
->res
.count
= count
;
815 data
->res
.verf
= &data
->verf
;
816 nfs_fattr_init(&data
->fattr
);
818 /* Set up the initial task struct. */
819 NFS_PROTO(inode
)->write_setup(data
, &msg
);
821 dprintk("NFS: %5u initiated write call "
822 "(req %s/%Ld, %u bytes @ offset %Lu)\n",
825 (long long)NFS_FILEID(inode
),
827 (unsigned long long)data
->args
.offset
);
829 task
= rpc_run_task(&task_setup_data
);
835 * Generate multiple small requests to write out a single
836 * contiguous dirty area on one page.
838 static int nfs_flush_multi(struct inode
*inode
, struct list_head
*head
, unsigned int npages
, size_t count
, int how
)
840 struct nfs_page
*req
= nfs_list_entry(head
->next
);
841 struct page
*page
= req
->wb_page
;
842 struct nfs_write_data
*data
;
843 size_t wsize
= NFS_SERVER(inode
)->wsize
, nbytes
;
848 nfs_list_remove_request(req
);
852 size_t len
= min(nbytes
, wsize
);
854 data
= nfs_writedata_alloc(1);
857 list_add(&data
->pages
, &list
);
860 } while (nbytes
!= 0);
861 atomic_set(&req
->wb_complete
, requests
);
863 ClearPageError(page
);
867 data
= list_entry(list
.next
, struct nfs_write_data
, pages
);
868 list_del_init(&data
->pages
);
870 data
->pagevec
[0] = page
;
874 nfs_write_rpcsetup(req
, data
, &nfs_write_partial_ops
,
878 } while (nbytes
!= 0);
883 while (!list_empty(&list
)) {
884 data
= list_entry(list
.next
, struct nfs_write_data
, pages
);
885 list_del(&data
->pages
);
886 nfs_writedata_release(data
);
888 nfs_redirty_request(req
);
889 nfs_end_page_writeback(req
->wb_page
);
890 nfs_clear_page_tag_locked(req
);
895 * Create an RPC task for the given write request and kick it.
896 * The page must have been locked by the caller.
898 * It may happen that the page we're passed is not marked dirty.
899 * This is the case if nfs_updatepage detects a conflicting request
900 * that has been written but not committed.
902 static int nfs_flush_one(struct inode
*inode
, struct list_head
*head
, unsigned int npages
, size_t count
, int how
)
904 struct nfs_page
*req
;
906 struct nfs_write_data
*data
;
908 data
= nfs_writedata_alloc(npages
);
912 pages
= data
->pagevec
;
913 while (!list_empty(head
)) {
914 req
= nfs_list_entry(head
->next
);
915 nfs_list_remove_request(req
);
916 nfs_list_add_request(req
, &data
->pages
);
917 ClearPageError(req
->wb_page
);
918 *pages
++ = req
->wb_page
;
920 req
= nfs_list_entry(data
->pages
.next
);
922 /* Set up the argument struct */
923 nfs_write_rpcsetup(req
, data
, &nfs_write_full_ops
, count
, 0, how
);
927 while (!list_empty(head
)) {
928 req
= nfs_list_entry(head
->next
);
929 nfs_list_remove_request(req
);
930 nfs_redirty_request(req
);
931 nfs_end_page_writeback(req
->wb_page
);
932 nfs_clear_page_tag_locked(req
);
937 static void nfs_pageio_init_write(struct nfs_pageio_descriptor
*pgio
,
938 struct inode
*inode
, int ioflags
)
940 size_t wsize
= NFS_SERVER(inode
)->wsize
;
942 if (wsize
< PAGE_CACHE_SIZE
)
943 nfs_pageio_init(pgio
, inode
, nfs_flush_multi
, wsize
, ioflags
);
945 nfs_pageio_init(pgio
, inode
, nfs_flush_one
, wsize
, ioflags
);
949 * Handle a write reply that flushed part of a page.
951 static void nfs_writeback_done_partial(struct rpc_task
*task
, void *calldata
)
953 struct nfs_write_data
*data
= calldata
;
954 struct nfs_page
*req
= data
->req
;
955 struct page
*page
= req
->wb_page
;
957 dprintk("NFS: write (%s/%Ld %d@%Ld)",
958 req
->wb_context
->path
.dentry
->d_inode
->i_sb
->s_id
,
959 (long long)NFS_FILEID(req
->wb_context
->path
.dentry
->d_inode
),
961 (long long)req_offset(req
));
963 if (nfs_writeback_done(task
, data
) != 0)
966 if (task
->tk_status
< 0) {
967 nfs_set_pageerror(page
);
968 nfs_context_set_write_error(req
->wb_context
, task
->tk_status
);
969 dprintk(", error = %d\n", task
->tk_status
);
973 if (nfs_write_need_commit(data
)) {
974 struct inode
*inode
= page
->mapping
->host
;
976 spin_lock(&inode
->i_lock
);
977 if (test_bit(PG_NEED_RESCHED
, &req
->wb_flags
)) {
978 /* Do nothing we need to resend the writes */
979 } else if (!test_and_set_bit(PG_NEED_COMMIT
, &req
->wb_flags
)) {
980 memcpy(&req
->wb_verf
, &data
->verf
, sizeof(req
->wb_verf
));
981 dprintk(" defer commit\n");
982 } else if (memcmp(&req
->wb_verf
, &data
->verf
, sizeof(req
->wb_verf
))) {
983 set_bit(PG_NEED_RESCHED
, &req
->wb_flags
);
984 clear_bit(PG_NEED_COMMIT
, &req
->wb_flags
);
985 dprintk(" server reboot detected\n");
987 spin_unlock(&inode
->i_lock
);
992 if (atomic_dec_and_test(&req
->wb_complete
))
993 nfs_writepage_release(req
);
996 static const struct rpc_call_ops nfs_write_partial_ops
= {
997 .rpc_call_done
= nfs_writeback_done_partial
,
998 .rpc_release
= nfs_writedata_release
,
1002 * Handle a write reply that flushes a whole page.
1004 * FIXME: There is an inherent race with invalidate_inode_pages and
1005 * writebacks since the page->count is kept > 1 for as long
1006 * as the page has a write request pending.
1008 static void nfs_writeback_done_full(struct rpc_task
*task
, void *calldata
)
1010 struct nfs_write_data
*data
= calldata
;
1011 struct nfs_page
*req
;
1014 if (nfs_writeback_done(task
, data
) != 0)
1017 /* Update attributes as result of writeback. */
1018 while (!list_empty(&data
->pages
)) {
1019 req
= nfs_list_entry(data
->pages
.next
);
1020 nfs_list_remove_request(req
);
1021 page
= req
->wb_page
;
1023 dprintk("NFS: write (%s/%Ld %d@%Ld)",
1024 req
->wb_context
->path
.dentry
->d_inode
->i_sb
->s_id
,
1025 (long long)NFS_FILEID(req
->wb_context
->path
.dentry
->d_inode
),
1027 (long long)req_offset(req
));
1029 if (task
->tk_status
< 0) {
1030 nfs_set_pageerror(page
);
1031 nfs_context_set_write_error(req
->wb_context
, task
->tk_status
);
1032 dprintk(", error = %d\n", task
->tk_status
);
1033 goto remove_request
;
1036 if (nfs_write_need_commit(data
)) {
1037 memcpy(&req
->wb_verf
, &data
->verf
, sizeof(req
->wb_verf
));
1038 nfs_mark_request_commit(req
);
1039 nfs_end_page_writeback(page
);
1040 dprintk(" marked for commit\n");
1043 /* Set the PG_uptodate flag? */
1044 nfs_mark_uptodate(page
, req
->wb_pgbase
, req
->wb_bytes
);
1047 nfs_end_page_writeback(page
);
1048 nfs_inode_remove_request(req
);
1050 nfs_clear_page_tag_locked(req
);
1054 static const struct rpc_call_ops nfs_write_full_ops
= {
1055 .rpc_call_done
= nfs_writeback_done_full
,
1056 .rpc_release
= nfs_writedata_release
,
1061 * This function is called when the WRITE call is complete.
1063 int nfs_writeback_done(struct rpc_task
*task
, struct nfs_write_data
*data
)
1065 struct nfs_writeargs
*argp
= &data
->args
;
1066 struct nfs_writeres
*resp
= &data
->res
;
1069 dprintk("NFS: %5u nfs_writeback_done (status %d)\n",
1070 task
->tk_pid
, task
->tk_status
);
1073 * ->write_done will attempt to use post-op attributes to detect
1074 * conflicting writes by other clients. A strict interpretation
1075 * of close-to-open would allow us to continue caching even if
1076 * another writer had changed the file, but some applications
1077 * depend on tighter cache coherency when writing.
1079 status
= NFS_PROTO(data
->inode
)->write_done(task
, data
);
1082 nfs_add_stats(data
->inode
, NFSIOS_SERVERWRITTENBYTES
, resp
->count
);
1084 #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
1085 if (resp
->verf
->committed
< argp
->stable
&& task
->tk_status
>= 0) {
1086 /* We tried a write call, but the server did not
1087 * commit data to stable storage even though we
1089 * Note: There is a known bug in Tru64 < 5.0 in which
1090 * the server reports NFS_DATA_SYNC, but performs
1091 * NFS_FILE_SYNC. We therefore implement this checking
1092 * as a dprintk() in order to avoid filling syslog.
1094 static unsigned long complain
;
1096 if (time_before(complain
, jiffies
)) {
1097 dprintk("NFS: faulty NFS server %s:"
1098 " (committed = %d) != (stable = %d)\n",
1099 NFS_SERVER(data
->inode
)->nfs_client
->cl_hostname
,
1100 resp
->verf
->committed
, argp
->stable
);
1101 complain
= jiffies
+ 300 * HZ
;
1105 /* Is this a short write? */
1106 if (task
->tk_status
>= 0 && resp
->count
< argp
->count
) {
1107 static unsigned long complain
;
1109 nfs_inc_stats(data
->inode
, NFSIOS_SHORTWRITE
);
1111 /* Has the server at least made some progress? */
1112 if (resp
->count
!= 0) {
1113 /* Was this an NFSv2 write or an NFSv3 stable write? */
1114 if (resp
->verf
->committed
!= NFS_UNSTABLE
) {
1115 /* Resend from where the server left off */
1116 argp
->offset
+= resp
->count
;
1117 argp
->pgbase
+= resp
->count
;
1118 argp
->count
-= resp
->count
;
1120 /* Resend as a stable write in order to avoid
1121 * headaches in the case of a server crash.
1123 argp
->stable
= NFS_FILE_SYNC
;
1125 rpc_restart_call(task
);
1128 if (time_before(complain
, jiffies
)) {
1130 "NFS: Server wrote zero bytes, expected %u.\n",
1132 complain
= jiffies
+ 300 * HZ
;
1134 /* Can't do anything about it except throw an error. */
1135 task
->tk_status
= -EIO
;
1141 #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
1142 void nfs_commit_release(void *wdata
)
1144 nfs_commit_free(wdata
);
1148 * Set up the argument/result storage required for the RPC call.
1150 static void nfs_commit_rpcsetup(struct list_head
*head
,
1151 struct nfs_write_data
*data
,
1154 struct nfs_page
*first
= nfs_list_entry(head
->next
);
1155 struct inode
*inode
= first
->wb_context
->path
.dentry
->d_inode
;
1156 int flags
= (how
& FLUSH_SYNC
) ? 0 : RPC_TASK_ASYNC
;
1157 int priority
= flush_task_priority(how
);
1158 struct rpc_task
*task
;
1159 struct rpc_message msg
= {
1160 .rpc_argp
= &data
->args
,
1161 .rpc_resp
= &data
->res
,
1162 .rpc_cred
= first
->wb_context
->cred
,
1164 struct rpc_task_setup task_setup_data
= {
1165 .task
= &data
->task
,
1166 .rpc_client
= NFS_CLIENT(inode
),
1167 .rpc_message
= &msg
,
1168 .callback_ops
= &nfs_commit_ops
,
1169 .callback_data
= data
,
1171 .priority
= priority
,
1174 /* Set up the RPC argument and reply structs
1175 * NB: take care not to mess about with data->commit et al. */
1177 list_splice_init(head
, &data
->pages
);
1179 data
->inode
= inode
;
1180 data
->cred
= msg
.rpc_cred
;
1182 data
->args
.fh
= NFS_FH(data
->inode
);
1183 /* Note: we always request a commit of the entire inode */
1184 data
->args
.offset
= 0;
1185 data
->args
.count
= 0;
1186 data
->res
.count
= 0;
1187 data
->res
.fattr
= &data
->fattr
;
1188 data
->res
.verf
= &data
->verf
;
1189 nfs_fattr_init(&data
->fattr
);
1191 /* Set up the initial task struct. */
1192 NFS_PROTO(inode
)->commit_setup(data
, &msg
);
1194 dprintk("NFS: %5u initiated commit call\n", data
->task
.tk_pid
);
1196 task
= rpc_run_task(&task_setup_data
);
1202 * Commit dirty pages
1205 nfs_commit_list(struct inode
*inode
, struct list_head
*head
, int how
)
1207 struct nfs_write_data
*data
;
1208 struct nfs_page
*req
;
1210 data
= nfs_commit_alloc();
1215 /* Set up the argument struct */
1216 nfs_commit_rpcsetup(head
, data
, how
);
1220 while (!list_empty(head
)) {
1221 req
= nfs_list_entry(head
->next
);
1222 nfs_list_remove_request(req
);
1223 nfs_mark_request_commit(req
);
1224 dec_zone_page_state(req
->wb_page
, NR_UNSTABLE_NFS
);
1225 dec_bdi_stat(req
->wb_page
->mapping
->backing_dev_info
,
1227 nfs_clear_page_tag_locked(req
);
1233 * COMMIT call returned
1235 static void nfs_commit_done(struct rpc_task
*task
, void *calldata
)
1237 struct nfs_write_data
*data
= calldata
;
1238 struct nfs_page
*req
;
1240 dprintk("NFS: %5u nfs_commit_done (status %d)\n",
1241 task
->tk_pid
, task
->tk_status
);
1243 /* Call the NFS version-specific code */
1244 if (NFS_PROTO(data
->inode
)->commit_done(task
, data
) != 0)
1247 while (!list_empty(&data
->pages
)) {
1248 req
= nfs_list_entry(data
->pages
.next
);
1249 nfs_list_remove_request(req
);
1250 clear_bit(PG_NEED_COMMIT
, &(req
)->wb_flags
);
1251 dec_zone_page_state(req
->wb_page
, NR_UNSTABLE_NFS
);
1252 dec_bdi_stat(req
->wb_page
->mapping
->backing_dev_info
,
1255 dprintk("NFS: commit (%s/%Ld %d@%Ld)",
1256 req
->wb_context
->path
.dentry
->d_inode
->i_sb
->s_id
,
1257 (long long)NFS_FILEID(req
->wb_context
->path
.dentry
->d_inode
),
1259 (long long)req_offset(req
));
1260 if (task
->tk_status
< 0) {
1261 nfs_context_set_write_error(req
->wb_context
, task
->tk_status
);
1262 nfs_inode_remove_request(req
);
1263 dprintk(", error = %d\n", task
->tk_status
);
1267 /* Okay, COMMIT succeeded, apparently. Check the verifier
1268 * returned by the server against all stored verfs. */
1269 if (!memcmp(req
->wb_verf
.verifier
, data
->verf
.verifier
, sizeof(data
->verf
.verifier
))) {
1270 /* We have a match */
1271 /* Set the PG_uptodate flag */
1272 nfs_mark_uptodate(req
->wb_page
, req
->wb_pgbase
,
1274 nfs_inode_remove_request(req
);
1278 /* We have a mismatch. Write the page again */
1279 dprintk(" mismatch\n");
1280 nfs_redirty_request(req
);
1282 nfs_clear_page_tag_locked(req
);
1286 static const struct rpc_call_ops nfs_commit_ops
= {
1287 .rpc_call_done
= nfs_commit_done
,
1288 .rpc_release
= nfs_commit_release
,
1291 int nfs_commit_inode(struct inode
*inode
, int how
)
1296 spin_lock(&inode
->i_lock
);
1297 res
= nfs_scan_commit(inode
, &head
, 0, 0);
1298 spin_unlock(&inode
->i_lock
);
1300 int error
= nfs_commit_list(inode
, &head
, how
);
1307 static inline int nfs_commit_list(struct inode
*inode
, struct list_head
*head
, int how
)
1313 long nfs_sync_mapping_wait(struct address_space
*mapping
, struct writeback_control
*wbc
, int how
)
1315 struct inode
*inode
= mapping
->host
;
1316 pgoff_t idx_start
, idx_end
;
1317 unsigned int npages
= 0;
1319 int nocommit
= how
& FLUSH_NOCOMMIT
;
1323 if (wbc
->range_cyclic
)
1326 idx_start
= wbc
->range_start
>> PAGE_CACHE_SHIFT
;
1327 idx_end
= wbc
->range_end
>> PAGE_CACHE_SHIFT
;
1328 if (idx_end
> idx_start
) {
1329 pgoff_t l_npages
= 1 + idx_end
- idx_start
;
1331 if (sizeof(npages
) != sizeof(l_npages
) &&
1332 (pgoff_t
)npages
!= l_npages
)
1336 how
&= ~FLUSH_NOCOMMIT
;
1337 spin_lock(&inode
->i_lock
);
1339 ret
= nfs_wait_on_requests_locked(inode
, idx_start
, npages
);
1344 pages
= nfs_scan_commit(inode
, &head
, idx_start
, npages
);
1347 if (how
& FLUSH_INVALIDATE
) {
1348 spin_unlock(&inode
->i_lock
);
1349 nfs_cancel_commit_list(&head
);
1351 spin_lock(&inode
->i_lock
);
1354 pages
+= nfs_scan_commit(inode
, &head
, 0, 0);
1355 spin_unlock(&inode
->i_lock
);
1356 ret
= nfs_commit_list(inode
, &head
, how
);
1357 spin_lock(&inode
->i_lock
);
1360 spin_unlock(&inode
->i_lock
);
1364 static int __nfs_write_mapping(struct address_space
*mapping
, struct writeback_control
*wbc
, int how
)
1368 ret
= nfs_writepages(mapping
, wbc
);
1371 ret
= nfs_sync_mapping_wait(mapping
, wbc
, how
);
1376 __mark_inode_dirty(mapping
->host
, I_DIRTY_PAGES
);
1380 /* Two pass sync: first using WB_SYNC_NONE, then WB_SYNC_ALL */
1381 static int nfs_write_mapping(struct address_space
*mapping
, int how
)
1383 struct writeback_control wbc
= {
1384 .bdi
= mapping
->backing_dev_info
,
1385 .sync_mode
= WB_SYNC_NONE
,
1386 .nr_to_write
= LONG_MAX
,
1387 .for_writepages
= 1,
1392 ret
= __nfs_write_mapping(mapping
, &wbc
, how
);
1395 wbc
.sync_mode
= WB_SYNC_ALL
;
1396 return __nfs_write_mapping(mapping
, &wbc
, how
);
1400 * flush the inode to disk.
1402 int nfs_wb_all(struct inode
*inode
)
1404 return nfs_write_mapping(inode
->i_mapping
, 0);
1407 int nfs_wb_nocommit(struct inode
*inode
)
1409 return nfs_write_mapping(inode
->i_mapping
, FLUSH_NOCOMMIT
);
1412 int nfs_wb_page_cancel(struct inode
*inode
, struct page
*page
)
1414 struct nfs_page
*req
;
1415 loff_t range_start
= page_offset(page
);
1416 loff_t range_end
= range_start
+ (loff_t
)(PAGE_CACHE_SIZE
- 1);
1417 struct writeback_control wbc
= {
1418 .bdi
= page
->mapping
->backing_dev_info
,
1419 .sync_mode
= WB_SYNC_ALL
,
1420 .nr_to_write
= LONG_MAX
,
1421 .range_start
= range_start
,
1422 .range_end
= range_end
,
1426 BUG_ON(!PageLocked(page
));
1428 req
= nfs_page_find_request(page
);
1431 if (test_bit(PG_NEED_COMMIT
, &req
->wb_flags
)) {
1432 nfs_release_request(req
);
1435 if (nfs_lock_request_dontget(req
)) {
1436 nfs_inode_remove_request(req
);
1438 * In case nfs_inode_remove_request has marked the
1439 * page as being dirty
1441 cancel_dirty_page(page
, PAGE_CACHE_SIZE
);
1442 nfs_unlock_request(req
);
1445 ret
= nfs_wait_on_request(req
);
1449 if (!PagePrivate(page
))
1451 ret
= nfs_sync_mapping_wait(page
->mapping
, &wbc
, FLUSH_INVALIDATE
);
1456 static int nfs_wb_page_priority(struct inode
*inode
, struct page
*page
,
1459 loff_t range_start
= page_offset(page
);
1460 loff_t range_end
= range_start
+ (loff_t
)(PAGE_CACHE_SIZE
- 1);
1461 struct writeback_control wbc
= {
1462 .bdi
= page
->mapping
->backing_dev_info
,
1463 .sync_mode
= WB_SYNC_ALL
,
1464 .nr_to_write
= LONG_MAX
,
1465 .range_start
= range_start
,
1466 .range_end
= range_end
,
1470 BUG_ON(!PageLocked(page
));
1471 if (clear_page_dirty_for_io(page
)) {
1472 ret
= nfs_writepage_locked(page
, &wbc
);
1476 if (!PagePrivate(page
))
1478 ret
= nfs_sync_mapping_wait(page
->mapping
, &wbc
, how
);
1482 __mark_inode_dirty(inode
, I_DIRTY_PAGES
);
1487 * Write back all requests on one page - we do this before reading it.
1489 int nfs_wb_page(struct inode
*inode
, struct page
* page
)
1491 return nfs_wb_page_priority(inode
, page
, FLUSH_STABLE
);
1494 int __init
nfs_init_writepagecache(void)
1496 nfs_wdata_cachep
= kmem_cache_create("nfs_write_data",
1497 sizeof(struct nfs_write_data
),
1498 0, SLAB_HWCACHE_ALIGN
,
1500 if (nfs_wdata_cachep
== NULL
)
1503 nfs_wdata_mempool
= mempool_create_slab_pool(MIN_POOL_WRITE
,
1505 if (nfs_wdata_mempool
== NULL
)
1508 nfs_commit_mempool
= mempool_create_slab_pool(MIN_POOL_COMMIT
,
1510 if (nfs_commit_mempool
== NULL
)
1514 * NFS congestion size, scale with available memory.
1526 * This allows larger machines to have larger/more transfers.
1527 * Limit the default to 256M
1529 nfs_congestion_kb
= (16*int_sqrt(totalram_pages
)) << (PAGE_SHIFT
-10);
1530 if (nfs_congestion_kb
> 256*1024)
1531 nfs_congestion_kb
= 256*1024;
1536 void nfs_destroy_writepagecache(void)
1538 mempool_destroy(nfs_commit_mempool
);
1539 mempool_destroy(nfs_wdata_mempool
);
1540 kmem_cache_destroy(nfs_wdata_cachep
);