6 * Partial copy of Linus' read cache modifications to fs/nfs/file.c
7 * modified for async RPC by okir@monad.swb.de
9 * We do an ugly hack here in order to return proper error codes to the
10 * user program when a read request failed: since generic_file_read
11 * only checks the return value of inode->i_op->readpage() which is always 0
12 * for async RPC, we set the error bit of the page to 1 when an error occurs,
13 * and make nfs_readpage transmit requests synchronously when encountering this.
14 * This is only a small problem, though, since we now retry all operations
15 * within the RPC code when root squashing is suspected.
18 #include <linux/config.h>
19 #include <linux/time.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/fcntl.h>
23 #include <linux/stat.h>
25 #include <linux/slab.h>
26 #include <linux/pagemap.h>
27 #include <linux/sunrpc/clnt.h>
28 #include <linux/nfs_fs.h>
29 #include <linux/nfs_page.h>
30 #include <linux/smp_lock.h>
32 #include <asm/system.h>
34 #define NFSDBG_FACILITY NFSDBG_PAGECACHE
36 static int nfs_pagein_one(struct list_head
*, struct inode
*);
37 static void nfs_readpage_result_partial(struct nfs_read_data
*, int);
38 static void nfs_readpage_result_full(struct nfs_read_data
*, int);
40 static kmem_cache_t
*nfs_rdata_cachep
;
41 mempool_t
*nfs_rdata_mempool
;
43 #define MIN_POOL_READ (32)
45 void nfs_readdata_release(void *data
)
47 nfs_readdata_free(data
);
51 unsigned int nfs_page_length(struct inode
*inode
, struct page
*page
)
53 loff_t i_size
= i_size_read(inode
);
58 idx
= (i_size
- 1) >> PAGE_CACHE_SHIFT
;
59 if (page
->index
> idx
)
61 if (page
->index
!= idx
)
62 return PAGE_CACHE_SIZE
;
63 return 1 + ((i_size
- 1) & (PAGE_CACHE_SIZE
- 1));
67 int nfs_return_empty_page(struct page
*page
)
69 memclear_highpage_flush(page
, 0, PAGE_CACHE_SIZE
);
70 SetPageUptodate(page
);
76 * Read a page synchronously.
78 static int nfs_readpage_sync(struct nfs_open_context
*ctx
, struct inode
*inode
,
81 unsigned int rsize
= NFS_SERVER(inode
)->rsize
;
82 unsigned int count
= PAGE_CACHE_SIZE
;
84 struct nfs_read_data
*rdata
;
86 rdata
= nfs_readdata_alloc(1);
90 memset(rdata
, 0, sizeof(*rdata
));
91 rdata
->flags
= (IS_SWAPFILE(inode
)? NFS_RPC_SWAPFLAGS
: 0);
92 rdata
->cred
= ctx
->cred
;
94 INIT_LIST_HEAD(&rdata
->pages
);
95 rdata
->args
.fh
= NFS_FH(inode
);
96 rdata
->args
.context
= ctx
;
97 rdata
->args
.pages
= &page
;
98 rdata
->args
.pgbase
= 0UL;
99 rdata
->args
.count
= rsize
;
100 rdata
->res
.fattr
= &rdata
->fattr
;
102 dprintk("NFS: nfs_readpage_sync(%p)\n", page
);
105 * This works now because the socket layer never tries to DMA
106 * into this buffer directly.
110 rdata
->args
.count
= count
;
111 rdata
->res
.count
= rdata
->args
.count
;
112 rdata
->args
.offset
= page_offset(page
) + rdata
->args
.pgbase
;
114 dprintk("NFS: nfs_proc_read(%s, (%s/%Ld), %Lu, %u)\n",
115 NFS_SERVER(inode
)->hostname
,
117 (long long)NFS_FILEID(inode
),
118 (unsigned long long)rdata
->args
.pgbase
,
122 result
= NFS_PROTO(inode
)->read(rdata
);
126 * Even if we had a partial success we can't mark the page
130 if (result
== -EISDIR
)
135 rdata
->args
.pgbase
+= result
;
136 /* Note: result == 0 should only happen if we're caching
137 * a write that extends the file and punches a hole.
139 if (rdata
->res
.eof
!= 0 || result
== 0)
142 spin_lock(&inode
->i_lock
);
143 NFS_I(inode
)->cache_validity
|= NFS_INO_INVALID_ATIME
;
144 spin_unlock(&inode
->i_lock
);
147 memclear_highpage_flush(page
, rdata
->args
.pgbase
, count
);
148 SetPageUptodate(page
);
150 ClearPageError(page
);
155 nfs_readdata_free(rdata
);
159 static int nfs_readpage_async(struct nfs_open_context
*ctx
, struct inode
*inode
,
162 LIST_HEAD(one_request
);
163 struct nfs_page
*new;
166 len
= nfs_page_length(inode
, page
);
168 return nfs_return_empty_page(page
);
169 new = nfs_create_request(ctx
, inode
, page
, 0, len
);
174 if (len
< PAGE_CACHE_SIZE
)
175 memclear_highpage_flush(page
, len
, PAGE_CACHE_SIZE
- len
);
177 nfs_list_add_request(new, &one_request
);
178 nfs_pagein_one(&one_request
, inode
);
182 static void nfs_readpage_release(struct nfs_page
*req
)
184 unlock_page(req
->wb_page
);
186 dprintk("NFS: read done (%s/%Ld %d@%Ld)\n",
187 req
->wb_context
->dentry
->d_inode
->i_sb
->s_id
,
188 (long long)NFS_FILEID(req
->wb_context
->dentry
->d_inode
),
190 (long long)req_offset(req
));
191 nfs_clear_request(req
);
192 nfs_release_request(req
);
196 * Set up the NFS read request struct
198 static void nfs_read_rpcsetup(struct nfs_page
*req
, struct nfs_read_data
*data
,
199 unsigned int count
, unsigned int offset
)
204 data
->inode
= inode
= req
->wb_context
->dentry
->d_inode
;
205 data
->cred
= req
->wb_context
->cred
;
207 data
->args
.fh
= NFS_FH(inode
);
208 data
->args
.offset
= req_offset(req
) + offset
;
209 data
->args
.pgbase
= req
->wb_pgbase
+ offset
;
210 data
->args
.pages
= data
->pagevec
;
211 data
->args
.count
= count
;
212 data
->args
.context
= req
->wb_context
;
214 data
->res
.fattr
= &data
->fattr
;
215 data
->res
.count
= count
;
217 nfs_fattr_init(&data
->fattr
);
219 NFS_PROTO(inode
)->read_setup(data
);
221 data
->task
.tk_cookie
= (unsigned long)inode
;
223 dprintk("NFS: %4d initiated read call (req %s/%Ld, %u bytes @ offset %Lu)\n",
226 (long long)NFS_FILEID(inode
),
228 (unsigned long long)data
->args
.offset
);
232 nfs_async_read_error(struct list_head
*head
)
234 struct nfs_page
*req
;
236 while (!list_empty(head
)) {
237 req
= nfs_list_entry(head
->next
);
238 nfs_list_remove_request(req
);
239 SetPageError(req
->wb_page
);
240 nfs_readpage_release(req
);
245 * Start an async read operation
247 static void nfs_execute_read(struct nfs_read_data
*data
)
249 struct rpc_clnt
*clnt
= NFS_CLIENT(data
->inode
);
252 rpc_clnt_sigmask(clnt
, &oldset
);
254 rpc_execute(&data
->task
);
256 rpc_clnt_sigunmask(clnt
, &oldset
);
260 * Generate multiple requests to fill a single page.
262 * We optimize to reduce the number of read operations on the wire. If we
263 * detect that we're reading a page, or an area of a page, that is past the
264 * end of file, we do not generate NFS read operations but just clear the
265 * parts of the page that would have come back zero from the server anyway.
267 * We rely on the cached value of i_size to make this determination; another
268 * client can fill pages on the server past our cached end-of-file, but we
269 * won't see the new data until our attribute cache is updated. This is more
270 * or less conventional NFS client behavior.
272 static int nfs_pagein_multi(struct list_head
*head
, struct inode
*inode
)
274 struct nfs_page
*req
= nfs_list_entry(head
->next
);
275 struct page
*page
= req
->wb_page
;
276 struct nfs_read_data
*data
;
277 unsigned int rsize
= NFS_SERVER(inode
)->rsize
;
278 unsigned int nbytes
, offset
;
282 nfs_list_remove_request(req
);
284 nbytes
= req
->wb_bytes
;
286 data
= nfs_readdata_alloc(1);
289 INIT_LIST_HEAD(&data
->pages
);
290 list_add(&data
->pages
, &list
);
296 atomic_set(&req
->wb_complete
, requests
);
298 ClearPageError(page
);
300 nbytes
= req
->wb_bytes
;
302 data
= list_entry(list
.next
, struct nfs_read_data
, pages
);
303 list_del_init(&data
->pages
);
305 data
->pagevec
[0] = page
;
306 data
->complete
= nfs_readpage_result_partial
;
308 if (nbytes
> rsize
) {
309 nfs_read_rpcsetup(req
, data
, rsize
, offset
);
313 nfs_read_rpcsetup(req
, data
, nbytes
, offset
);
316 nfs_execute_read(data
);
317 } while (nbytes
!= 0);
322 while (!list_empty(&list
)) {
323 data
= list_entry(list
.next
, struct nfs_read_data
, pages
);
324 list_del(&data
->pages
);
325 nfs_readdata_free(data
);
328 nfs_readpage_release(req
);
332 static int nfs_pagein_one(struct list_head
*head
, struct inode
*inode
)
334 struct nfs_page
*req
;
336 struct nfs_read_data
*data
;
339 if (NFS_SERVER(inode
)->rsize
< PAGE_CACHE_SIZE
)
340 return nfs_pagein_multi(head
, inode
);
342 data
= nfs_readdata_alloc(NFS_SERVER(inode
)->rpages
);
346 INIT_LIST_HEAD(&data
->pages
);
347 pages
= data
->pagevec
;
349 while (!list_empty(head
)) {
350 req
= nfs_list_entry(head
->next
);
351 nfs_list_remove_request(req
);
352 nfs_list_add_request(req
, &data
->pages
);
353 ClearPageError(req
->wb_page
);
354 *pages
++ = req
->wb_page
;
355 count
+= req
->wb_bytes
;
357 req
= nfs_list_entry(data
->pages
.next
);
359 data
->complete
= nfs_readpage_result_full
;
360 nfs_read_rpcsetup(req
, data
, count
, 0);
362 nfs_execute_read(data
);
365 nfs_async_read_error(head
);
370 nfs_pagein_list(struct list_head
*head
, int rpages
)
372 LIST_HEAD(one_request
);
373 struct nfs_page
*req
;
375 unsigned int pages
= 0;
377 while (!list_empty(head
)) {
378 pages
+= nfs_coalesce_requests(head
, &one_request
, rpages
);
379 req
= nfs_list_entry(one_request
.next
);
380 error
= nfs_pagein_one(&one_request
, req
->wb_context
->dentry
->d_inode
);
387 nfs_async_read_error(head
);
392 * Handle a read reply that fills part of a page.
394 static void nfs_readpage_result_partial(struct nfs_read_data
*data
, int status
)
396 struct nfs_page
*req
= data
->req
;
397 struct page
*page
= req
->wb_page
;
400 unsigned int request
= data
->args
.count
;
401 unsigned int result
= data
->res
.count
;
403 if (result
< request
) {
404 memclear_highpage_flush(page
,
405 data
->args
.pgbase
+ result
,
411 if (atomic_dec_and_test(&req
->wb_complete
)) {
412 if (!PageError(page
))
413 SetPageUptodate(page
);
414 nfs_readpage_release(req
);
419 * This is the callback from RPC telling us whether a reply was
420 * received or some error occurred (timeout or socket shutdown).
422 static void nfs_readpage_result_full(struct nfs_read_data
*data
, int status
)
424 unsigned int count
= data
->res
.count
;
426 while (!list_empty(&data
->pages
)) {
427 struct nfs_page
*req
= nfs_list_entry(data
->pages
.next
);
428 struct page
*page
= req
->wb_page
;
429 nfs_list_remove_request(req
);
432 if (count
< PAGE_CACHE_SIZE
) {
433 if (count
< req
->wb_bytes
)
434 memclear_highpage_flush(page
,
435 req
->wb_pgbase
+ count
,
436 req
->wb_bytes
- count
);
439 count
-= PAGE_CACHE_SIZE
;
440 SetPageUptodate(page
);
443 nfs_readpage_release(req
);
448 * This is the callback from RPC telling us whether a reply was
449 * received or some error occurred (timeout or socket shutdown).
451 void nfs_readpage_result(struct rpc_task
*task
, void *calldata
)
453 struct nfs_read_data
*data
= calldata
;
454 struct nfs_readargs
*argp
= &data
->args
;
455 struct nfs_readres
*resp
= &data
->res
;
456 int status
= task
->tk_status
;
458 dprintk("NFS: %4d nfs_readpage_result, (status %d)\n",
459 task
->tk_pid
, status
);
461 /* Is this a short read? */
462 if (task
->tk_status
>= 0 && resp
->count
< argp
->count
&& !resp
->eof
) {
463 /* Has the server at least made some progress? */
464 if (resp
->count
!= 0) {
465 /* Yes, so retry the read at the end of the data */
466 argp
->offset
+= resp
->count
;
467 argp
->pgbase
+= resp
->count
;
468 argp
->count
-= resp
->count
;
469 rpc_restart_call(task
);
472 task
->tk_status
= -EIO
;
474 spin_lock(&data
->inode
->i_lock
);
475 NFS_I(data
->inode
)->cache_validity
|= NFS_INO_INVALID_ATIME
;
476 spin_unlock(&data
->inode
->i_lock
);
477 data
->complete(data
, status
);
481 * Read a page over NFS.
482 * We read the page synchronously in the following case:
483 * - The error flag is set for this page. This happens only when a
484 * previous async read operation failed.
486 int nfs_readpage(struct file
*file
, struct page
*page
)
488 struct nfs_open_context
*ctx
;
489 struct inode
*inode
= page
->mapping
->host
;
492 dprintk("NFS: nfs_readpage (%p %ld@%lu)\n",
493 page
, PAGE_CACHE_SIZE
, page
->index
);
495 * Try to flush any pending writes to the file..
497 * NOTE! Because we own the page lock, there cannot
498 * be any new pending writes generated at this point
499 * for this page (other pages can be written to).
501 error
= nfs_wb_page(inode
, page
);
506 ctx
= nfs_find_open_context(inode
, NULL
, FMODE_READ
);
510 ctx
= get_nfs_open_context((struct nfs_open_context
*)
512 if (!IS_SYNC(inode
)) {
513 error
= nfs_readpage_async(ctx
, inode
, page
);
517 error
= nfs_readpage_sync(ctx
, inode
, page
);
518 if (error
< 0 && IS_SWAPFILE(inode
))
519 printk("Aiee.. nfs swap-in of page failed!\n");
521 put_nfs_open_context(ctx
);
529 struct nfs_readdesc
{
530 struct list_head
*head
;
531 struct nfs_open_context
*ctx
;
535 readpage_async_filler(void *data
, struct page
*page
)
537 struct nfs_readdesc
*desc
= (struct nfs_readdesc
*)data
;
538 struct inode
*inode
= page
->mapping
->host
;
539 struct nfs_page
*new;
542 nfs_wb_page(inode
, page
);
543 len
= nfs_page_length(inode
, page
);
545 return nfs_return_empty_page(page
);
546 new = nfs_create_request(desc
->ctx
, inode
, page
, 0, len
);
552 if (len
< PAGE_CACHE_SIZE
)
553 memclear_highpage_flush(page
, len
, PAGE_CACHE_SIZE
- len
);
554 nfs_list_add_request(new, desc
->head
);
558 int nfs_readpages(struct file
*filp
, struct address_space
*mapping
,
559 struct list_head
*pages
, unsigned nr_pages
)
562 struct nfs_readdesc desc
= {
565 struct inode
*inode
= mapping
->host
;
566 struct nfs_server
*server
= NFS_SERVER(inode
);
569 dprintk("NFS: nfs_readpages (%s/%Ld %d)\n",
571 (long long)NFS_FILEID(inode
),
575 desc
.ctx
= nfs_find_open_context(inode
, NULL
, FMODE_READ
);
576 if (desc
.ctx
== NULL
)
579 desc
.ctx
= get_nfs_open_context((struct nfs_open_context
*)
581 ret
= read_cache_pages(mapping
, pages
, readpage_async_filler
, &desc
);
582 if (!list_empty(&head
)) {
583 int err
= nfs_pagein_list(&head
, server
->rpages
);
587 put_nfs_open_context(desc
.ctx
);
591 int nfs_init_readpagecache(void)
593 nfs_rdata_cachep
= kmem_cache_create("nfs_read_data",
594 sizeof(struct nfs_read_data
),
595 0, SLAB_HWCACHE_ALIGN
,
597 if (nfs_rdata_cachep
== NULL
)
600 nfs_rdata_mempool
= mempool_create(MIN_POOL_READ
,
604 if (nfs_rdata_mempool
== NULL
)
610 void nfs_destroy_readpagecache(void)
612 mempool_destroy(nfs_rdata_mempool
);
613 if (kmem_cache_destroy(nfs_rdata_cachep
))
614 printk(KERN_INFO
"nfs_read_data: not all structures were freed\n");