[PATCH] Make address_space_operations->invalidatepage return void
[linux-2.6.22.y-op.git] / fs / xfs / linux-2.6 / xfs_aops.c
blob4f2476f188b04dcc3f41d76376eea9c10cb52fdc
1 /*
2 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3 * All Rights Reserved.
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 #include "xfs.h"
19 #include "xfs_bit.h"
20 #include "xfs_log.h"
21 #include "xfs_inum.h"
22 #include "xfs_sb.h"
23 #include "xfs_ag.h"
24 #include "xfs_dir.h"
25 #include "xfs_dir2.h"
26 #include "xfs_trans.h"
27 #include "xfs_dmapi.h"
28 #include "xfs_mount.h"
29 #include "xfs_bmap_btree.h"
30 #include "xfs_alloc_btree.h"
31 #include "xfs_ialloc_btree.h"
32 #include "xfs_dir_sf.h"
33 #include "xfs_dir2_sf.h"
34 #include "xfs_attr_sf.h"
35 #include "xfs_dinode.h"
36 #include "xfs_inode.h"
37 #include "xfs_alloc.h"
38 #include "xfs_btree.h"
39 #include "xfs_error.h"
40 #include "xfs_rw.h"
41 #include "xfs_iomap.h"
42 #include <linux/mpage.h>
43 #include <linux/pagevec.h>
44 #include <linux/writeback.h>
46 STATIC void
47 xfs_count_page_state(
48 struct page *page,
49 int *delalloc,
50 int *unmapped,
51 int *unwritten)
53 struct buffer_head *bh, *head;
55 *delalloc = *unmapped = *unwritten = 0;
57 bh = head = page_buffers(page);
58 do {
59 if (buffer_uptodate(bh) && !buffer_mapped(bh))
60 (*unmapped) = 1;
61 else if (buffer_unwritten(bh) && !buffer_delay(bh))
62 clear_buffer_unwritten(bh);
63 else if (buffer_unwritten(bh))
64 (*unwritten) = 1;
65 else if (buffer_delay(bh))
66 (*delalloc) = 1;
67 } while ((bh = bh->b_this_page) != head);
70 #if defined(XFS_RW_TRACE)
71 void
72 xfs_page_trace(
73 int tag,
74 struct inode *inode,
75 struct page *page,
76 int mask)
78 xfs_inode_t *ip;
79 vnode_t *vp = vn_from_inode(inode);
80 loff_t isize = i_size_read(inode);
81 loff_t offset = page_offset(page);
82 int delalloc = -1, unmapped = -1, unwritten = -1;
84 if (page_has_buffers(page))
85 xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
87 ip = xfs_vtoi(vp);
88 if (!ip->i_rwtrace)
89 return;
91 ktrace_enter(ip->i_rwtrace,
92 (void *)((unsigned long)tag),
93 (void *)ip,
94 (void *)inode,
95 (void *)page,
96 (void *)((unsigned long)mask),
97 (void *)((unsigned long)((ip->i_d.di_size >> 32) & 0xffffffff)),
98 (void *)((unsigned long)(ip->i_d.di_size & 0xffffffff)),
99 (void *)((unsigned long)((isize >> 32) & 0xffffffff)),
100 (void *)((unsigned long)(isize & 0xffffffff)),
101 (void *)((unsigned long)((offset >> 32) & 0xffffffff)),
102 (void *)((unsigned long)(offset & 0xffffffff)),
103 (void *)((unsigned long)delalloc),
104 (void *)((unsigned long)unmapped),
105 (void *)((unsigned long)unwritten),
106 (void *)((unsigned long)current_pid()),
107 (void *)NULL);
109 #else
110 #define xfs_page_trace(tag, inode, page, mask)
111 #endif
114 * Schedule IO completion handling on a xfsdatad if this was
115 * the final hold on this ioend.
117 STATIC void
118 xfs_finish_ioend(
119 xfs_ioend_t *ioend)
121 if (atomic_dec_and_test(&ioend->io_remaining))
122 queue_work(xfsdatad_workqueue, &ioend->io_work);
126 * We're now finished for good with this ioend structure.
127 * Update the page state via the associated buffer_heads,
128 * release holds on the inode and bio, and finally free
129 * up memory. Do not use the ioend after this.
131 STATIC void
132 xfs_destroy_ioend(
133 xfs_ioend_t *ioend)
135 struct buffer_head *bh, *next;
137 for (bh = ioend->io_buffer_head; bh; bh = next) {
138 next = bh->b_private;
139 bh->b_end_io(bh, ioend->io_uptodate);
142 vn_iowake(ioend->io_vnode);
143 mempool_free(ioend, xfs_ioend_pool);
147 * Buffered IO write completion for delayed allocate extents.
148 * TODO: Update ondisk isize now that we know the file data
149 * has been flushed (i.e. the notorious "NULL file" problem).
151 STATIC void
152 xfs_end_bio_delalloc(
153 void *data)
155 xfs_ioend_t *ioend = data;
157 xfs_destroy_ioend(ioend);
161 * Buffered IO write completion for regular, written extents.
163 STATIC void
164 xfs_end_bio_written(
165 void *data)
167 xfs_ioend_t *ioend = data;
169 xfs_destroy_ioend(ioend);
173 * IO write completion for unwritten extents.
175 * Issue transactions to convert a buffer range from unwritten
176 * to written extents.
178 STATIC void
179 xfs_end_bio_unwritten(
180 void *data)
182 xfs_ioend_t *ioend = data;
183 vnode_t *vp = ioend->io_vnode;
184 xfs_off_t offset = ioend->io_offset;
185 size_t size = ioend->io_size;
186 int error;
188 if (ioend->io_uptodate)
189 VOP_BMAP(vp, offset, size, BMAPI_UNWRITTEN, NULL, NULL, error);
190 xfs_destroy_ioend(ioend);
194 * Allocate and initialise an IO completion structure.
195 * We need to track unwritten extent write completion here initially.
196 * We'll need to extend this for updating the ondisk inode size later
197 * (vs. incore size).
199 STATIC xfs_ioend_t *
200 xfs_alloc_ioend(
201 struct inode *inode,
202 unsigned int type)
204 xfs_ioend_t *ioend;
206 ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
209 * Set the count to 1 initially, which will prevent an I/O
210 * completion callback from happening before we have started
211 * all the I/O from calling the completion routine too early.
213 atomic_set(&ioend->io_remaining, 1);
214 ioend->io_uptodate = 1; /* cleared if any I/O fails */
215 ioend->io_list = NULL;
216 ioend->io_type = type;
217 ioend->io_vnode = vn_from_inode(inode);
218 ioend->io_buffer_head = NULL;
219 ioend->io_buffer_tail = NULL;
220 atomic_inc(&ioend->io_vnode->v_iocount);
221 ioend->io_offset = 0;
222 ioend->io_size = 0;
224 if (type == IOMAP_UNWRITTEN)
225 INIT_WORK(&ioend->io_work, xfs_end_bio_unwritten, ioend);
226 else if (type == IOMAP_DELAY)
227 INIT_WORK(&ioend->io_work, xfs_end_bio_delalloc, ioend);
228 else
229 INIT_WORK(&ioend->io_work, xfs_end_bio_written, ioend);
231 return ioend;
234 STATIC int
235 xfs_map_blocks(
236 struct inode *inode,
237 loff_t offset,
238 ssize_t count,
239 xfs_iomap_t *mapp,
240 int flags)
242 vnode_t *vp = vn_from_inode(inode);
243 int error, nmaps = 1;
245 VOP_BMAP(vp, offset, count, flags, mapp, &nmaps, error);
246 if (!error && (flags & (BMAPI_WRITE|BMAPI_ALLOCATE)))
247 VMODIFY(vp);
248 return -error;
251 STATIC inline int
252 xfs_iomap_valid(
253 xfs_iomap_t *iomapp,
254 loff_t offset)
256 return offset >= iomapp->iomap_offset &&
257 offset < iomapp->iomap_offset + iomapp->iomap_bsize;
261 * BIO completion handler for buffered IO.
263 STATIC int
264 xfs_end_bio(
265 struct bio *bio,
266 unsigned int bytes_done,
267 int error)
269 xfs_ioend_t *ioend = bio->bi_private;
271 if (bio->bi_size)
272 return 1;
274 ASSERT(ioend);
275 ASSERT(atomic_read(&bio->bi_cnt) >= 1);
277 /* Toss bio and pass work off to an xfsdatad thread */
278 if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
279 ioend->io_uptodate = 0;
280 bio->bi_private = NULL;
281 bio->bi_end_io = NULL;
283 bio_put(bio);
284 xfs_finish_ioend(ioend);
285 return 0;
288 STATIC void
289 xfs_submit_ioend_bio(
290 xfs_ioend_t *ioend,
291 struct bio *bio)
293 atomic_inc(&ioend->io_remaining);
295 bio->bi_private = ioend;
296 bio->bi_end_io = xfs_end_bio;
298 submit_bio(WRITE, bio);
299 ASSERT(!bio_flagged(bio, BIO_EOPNOTSUPP));
300 bio_put(bio);
303 STATIC struct bio *
304 xfs_alloc_ioend_bio(
305 struct buffer_head *bh)
307 struct bio *bio;
308 int nvecs = bio_get_nr_vecs(bh->b_bdev);
310 do {
311 bio = bio_alloc(GFP_NOIO, nvecs);
312 nvecs >>= 1;
313 } while (!bio);
315 ASSERT(bio->bi_private == NULL);
316 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
317 bio->bi_bdev = bh->b_bdev;
318 bio_get(bio);
319 return bio;
322 STATIC void
323 xfs_start_buffer_writeback(
324 struct buffer_head *bh)
326 ASSERT(buffer_mapped(bh));
327 ASSERT(buffer_locked(bh));
328 ASSERT(!buffer_delay(bh));
329 ASSERT(!buffer_unwritten(bh));
331 mark_buffer_async_write(bh);
332 set_buffer_uptodate(bh);
333 clear_buffer_dirty(bh);
336 STATIC void
337 xfs_start_page_writeback(
338 struct page *page,
339 struct writeback_control *wbc,
340 int clear_dirty,
341 int buffers)
343 ASSERT(PageLocked(page));
344 ASSERT(!PageWriteback(page));
345 set_page_writeback(page);
346 if (clear_dirty)
347 clear_page_dirty(page);
348 unlock_page(page);
349 if (!buffers) {
350 end_page_writeback(page);
351 wbc->pages_skipped++; /* We didn't write this page */
355 static inline int bio_add_buffer(struct bio *bio, struct buffer_head *bh)
357 return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
361 * Submit all of the bios for all of the ioends we have saved up, covering the
362 * initial writepage page and also any probed pages.
364 * Because we may have multiple ioends spanning a page, we need to start
365 * writeback on all the buffers before we submit them for I/O. If we mark the
366 * buffers as we got, then we can end up with a page that only has buffers
367 * marked async write and I/O complete on can occur before we mark the other
368 * buffers async write.
370 * The end result of this is that we trip a bug in end_page_writeback() because
371 * we call it twice for the one page as the code in end_buffer_async_write()
372 * assumes that all buffers on the page are started at the same time.
374 * The fix is two passes across the ioend list - one to start writeback on the
375 * bufferheads, and then the second one submit them for I/O.
377 STATIC void
378 xfs_submit_ioend(
379 xfs_ioend_t *ioend)
381 xfs_ioend_t *head = ioend;
382 xfs_ioend_t *next;
383 struct buffer_head *bh;
384 struct bio *bio;
385 sector_t lastblock = 0;
387 /* Pass 1 - start writeback */
388 do {
389 next = ioend->io_list;
390 for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
391 xfs_start_buffer_writeback(bh);
393 } while ((ioend = next) != NULL);
395 /* Pass 2 - submit I/O */
396 ioend = head;
397 do {
398 next = ioend->io_list;
399 bio = NULL;
401 for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
403 if (!bio) {
404 retry:
405 bio = xfs_alloc_ioend_bio(bh);
406 } else if (bh->b_blocknr != lastblock + 1) {
407 xfs_submit_ioend_bio(ioend, bio);
408 goto retry;
411 if (bio_add_buffer(bio, bh) != bh->b_size) {
412 xfs_submit_ioend_bio(ioend, bio);
413 goto retry;
416 lastblock = bh->b_blocknr;
418 if (bio)
419 xfs_submit_ioend_bio(ioend, bio);
420 xfs_finish_ioend(ioend);
421 } while ((ioend = next) != NULL);
425 * Cancel submission of all buffer_heads so far in this endio.
426 * Toss the endio too. Only ever called for the initial page
427 * in a writepage request, so only ever one page.
429 STATIC void
430 xfs_cancel_ioend(
431 xfs_ioend_t *ioend)
433 xfs_ioend_t *next;
434 struct buffer_head *bh, *next_bh;
436 do {
437 next = ioend->io_list;
438 bh = ioend->io_buffer_head;
439 do {
440 next_bh = bh->b_private;
441 clear_buffer_async_write(bh);
442 unlock_buffer(bh);
443 } while ((bh = next_bh) != NULL);
445 vn_iowake(ioend->io_vnode);
446 mempool_free(ioend, xfs_ioend_pool);
447 } while ((ioend = next) != NULL);
451 * Test to see if we've been building up a completion structure for
452 * earlier buffers -- if so, we try to append to this ioend if we
453 * can, otherwise we finish off any current ioend and start another.
454 * Return true if we've finished the given ioend.
456 STATIC void
457 xfs_add_to_ioend(
458 struct inode *inode,
459 struct buffer_head *bh,
460 xfs_off_t offset,
461 unsigned int type,
462 xfs_ioend_t **result,
463 int need_ioend)
465 xfs_ioend_t *ioend = *result;
467 if (!ioend || need_ioend || type != ioend->io_type) {
468 xfs_ioend_t *previous = *result;
470 ioend = xfs_alloc_ioend(inode, type);
471 ioend->io_offset = offset;
472 ioend->io_buffer_head = bh;
473 ioend->io_buffer_tail = bh;
474 if (previous)
475 previous->io_list = ioend;
476 *result = ioend;
477 } else {
478 ioend->io_buffer_tail->b_private = bh;
479 ioend->io_buffer_tail = bh;
482 bh->b_private = NULL;
483 ioend->io_size += bh->b_size;
486 STATIC void
487 xfs_map_buffer(
488 struct buffer_head *bh,
489 xfs_iomap_t *mp,
490 xfs_off_t offset,
491 uint block_bits)
493 sector_t bn;
495 ASSERT(mp->iomap_bn != IOMAP_DADDR_NULL);
497 bn = (mp->iomap_bn >> (block_bits - BBSHIFT)) +
498 ((offset - mp->iomap_offset) >> block_bits);
500 ASSERT(bn || (mp->iomap_flags & IOMAP_REALTIME));
502 bh->b_blocknr = bn;
503 set_buffer_mapped(bh);
506 STATIC void
507 xfs_map_at_offset(
508 struct buffer_head *bh,
509 loff_t offset,
510 int block_bits,
511 xfs_iomap_t *iomapp)
513 ASSERT(!(iomapp->iomap_flags & IOMAP_HOLE));
514 ASSERT(!(iomapp->iomap_flags & IOMAP_DELAY));
516 lock_buffer(bh);
517 xfs_map_buffer(bh, iomapp, offset, block_bits);
518 bh->b_bdev = iomapp->iomap_target->bt_bdev;
519 set_buffer_mapped(bh);
520 clear_buffer_delay(bh);
521 clear_buffer_unwritten(bh);
525 * Look for a page at index that is suitable for clustering.
527 STATIC unsigned int
528 xfs_probe_page(
529 struct page *page,
530 unsigned int pg_offset,
531 int mapped)
533 int ret = 0;
535 if (PageWriteback(page))
536 return 0;
538 if (page->mapping && PageDirty(page)) {
539 if (page_has_buffers(page)) {
540 struct buffer_head *bh, *head;
542 bh = head = page_buffers(page);
543 do {
544 if (!buffer_uptodate(bh))
545 break;
546 if (mapped != buffer_mapped(bh))
547 break;
548 ret += bh->b_size;
549 if (ret >= pg_offset)
550 break;
551 } while ((bh = bh->b_this_page) != head);
552 } else
553 ret = mapped ? 0 : PAGE_CACHE_SIZE;
556 return ret;
559 STATIC size_t
560 xfs_probe_cluster(
561 struct inode *inode,
562 struct page *startpage,
563 struct buffer_head *bh,
564 struct buffer_head *head,
565 int mapped)
567 struct pagevec pvec;
568 pgoff_t tindex, tlast, tloff;
569 size_t total = 0;
570 int done = 0, i;
572 /* First sum forwards in this page */
573 do {
574 if (!buffer_uptodate(bh) || (mapped != buffer_mapped(bh)))
575 return total;
576 total += bh->b_size;
577 } while ((bh = bh->b_this_page) != head);
579 /* if we reached the end of the page, sum forwards in following pages */
580 tlast = i_size_read(inode) >> PAGE_CACHE_SHIFT;
581 tindex = startpage->index + 1;
583 /* Prune this back to avoid pathological behavior */
584 tloff = min(tlast, startpage->index + 64);
586 pagevec_init(&pvec, 0);
587 while (!done && tindex <= tloff) {
588 unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
590 if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
591 break;
593 for (i = 0; i < pagevec_count(&pvec); i++) {
594 struct page *page = pvec.pages[i];
595 size_t pg_offset, len = 0;
597 if (tindex == tlast) {
598 pg_offset =
599 i_size_read(inode) & (PAGE_CACHE_SIZE - 1);
600 if (!pg_offset) {
601 done = 1;
602 break;
604 } else
605 pg_offset = PAGE_CACHE_SIZE;
607 if (page->index == tindex && !TestSetPageLocked(page)) {
608 len = xfs_probe_page(page, pg_offset, mapped);
609 unlock_page(page);
612 if (!len) {
613 done = 1;
614 break;
617 total += len;
618 tindex++;
621 pagevec_release(&pvec);
622 cond_resched();
625 return total;
629 * Test if a given page is suitable for writing as part of an unwritten
630 * or delayed allocate extent.
632 STATIC int
633 xfs_is_delayed_page(
634 struct page *page,
635 unsigned int type)
637 if (PageWriteback(page))
638 return 0;
640 if (page->mapping && page_has_buffers(page)) {
641 struct buffer_head *bh, *head;
642 int acceptable = 0;
644 bh = head = page_buffers(page);
645 do {
646 if (buffer_unwritten(bh))
647 acceptable = (type == IOMAP_UNWRITTEN);
648 else if (buffer_delay(bh))
649 acceptable = (type == IOMAP_DELAY);
650 else if (buffer_dirty(bh) && buffer_mapped(bh))
651 acceptable = (type == 0);
652 else
653 break;
654 } while ((bh = bh->b_this_page) != head);
656 if (acceptable)
657 return 1;
660 return 0;
664 * Allocate & map buffers for page given the extent map. Write it out.
665 * except for the original page of a writepage, this is called on
666 * delalloc/unwritten pages only, for the original page it is possible
667 * that the page has no mapping at all.
669 STATIC int
670 xfs_convert_page(
671 struct inode *inode,
672 struct page *page,
673 loff_t tindex,
674 xfs_iomap_t *mp,
675 xfs_ioend_t **ioendp,
676 struct writeback_control *wbc,
677 int startio,
678 int all_bh)
680 struct buffer_head *bh, *head;
681 xfs_off_t end_offset;
682 unsigned long p_offset;
683 unsigned int type;
684 int bbits = inode->i_blkbits;
685 int len, page_dirty;
686 int count = 0, done = 0, uptodate = 1;
687 xfs_off_t offset = page_offset(page);
689 if (page->index != tindex)
690 goto fail;
691 if (TestSetPageLocked(page))
692 goto fail;
693 if (PageWriteback(page))
694 goto fail_unlock_page;
695 if (page->mapping != inode->i_mapping)
696 goto fail_unlock_page;
697 if (!xfs_is_delayed_page(page, (*ioendp)->io_type))
698 goto fail_unlock_page;
701 * page_dirty is initially a count of buffers on the page before
702 * EOF and is decrememted as we move each into a cleanable state.
704 * Derivation:
706 * End offset is the highest offset that this page should represent.
707 * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
708 * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
709 * hence give us the correct page_dirty count. On any other page,
710 * it will be zero and in that case we need page_dirty to be the
711 * count of buffers on the page.
713 end_offset = min_t(unsigned long long,
714 (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
715 i_size_read(inode));
717 len = 1 << inode->i_blkbits;
718 p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
719 PAGE_CACHE_SIZE);
720 p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
721 page_dirty = p_offset / len;
723 bh = head = page_buffers(page);
724 do {
725 if (offset >= end_offset)
726 break;
727 if (!buffer_uptodate(bh))
728 uptodate = 0;
729 if (!(PageUptodate(page) || buffer_uptodate(bh))) {
730 done = 1;
731 continue;
734 if (buffer_unwritten(bh) || buffer_delay(bh)) {
735 if (buffer_unwritten(bh))
736 type = IOMAP_UNWRITTEN;
737 else
738 type = IOMAP_DELAY;
740 if (!xfs_iomap_valid(mp, offset)) {
741 done = 1;
742 continue;
745 ASSERT(!(mp->iomap_flags & IOMAP_HOLE));
746 ASSERT(!(mp->iomap_flags & IOMAP_DELAY));
748 xfs_map_at_offset(bh, offset, bbits, mp);
749 if (startio) {
750 xfs_add_to_ioend(inode, bh, offset,
751 type, ioendp, done);
752 } else {
753 set_buffer_dirty(bh);
754 unlock_buffer(bh);
755 mark_buffer_dirty(bh);
757 page_dirty--;
758 count++;
759 } else {
760 type = 0;
761 if (buffer_mapped(bh) && all_bh && startio) {
762 lock_buffer(bh);
763 xfs_add_to_ioend(inode, bh, offset,
764 type, ioendp, done);
765 count++;
766 page_dirty--;
767 } else {
768 done = 1;
771 } while (offset += len, (bh = bh->b_this_page) != head);
773 if (uptodate && bh == head)
774 SetPageUptodate(page);
776 if (startio) {
777 if (count) {
778 struct backing_dev_info *bdi;
780 bdi = inode->i_mapping->backing_dev_info;
781 wbc->nr_to_write--;
782 if (bdi_write_congested(bdi)) {
783 wbc->encountered_congestion = 1;
784 done = 1;
785 } else if (wbc->nr_to_write <= 0) {
786 done = 1;
789 xfs_start_page_writeback(page, wbc, !page_dirty, count);
792 return done;
793 fail_unlock_page:
794 unlock_page(page);
795 fail:
796 return 1;
800 * Convert & write out a cluster of pages in the same extent as defined
801 * by mp and following the start page.
803 STATIC void
804 xfs_cluster_write(
805 struct inode *inode,
806 pgoff_t tindex,
807 xfs_iomap_t *iomapp,
808 xfs_ioend_t **ioendp,
809 struct writeback_control *wbc,
810 int startio,
811 int all_bh,
812 pgoff_t tlast)
814 struct pagevec pvec;
815 int done = 0, i;
817 pagevec_init(&pvec, 0);
818 while (!done && tindex <= tlast) {
819 unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
821 if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
822 break;
824 for (i = 0; i < pagevec_count(&pvec); i++) {
825 done = xfs_convert_page(inode, pvec.pages[i], tindex++,
826 iomapp, ioendp, wbc, startio, all_bh);
827 if (done)
828 break;
831 pagevec_release(&pvec);
832 cond_resched();
837 * Calling this without startio set means we are being asked to make a dirty
838 * page ready for freeing it's buffers. When called with startio set then
839 * we are coming from writepage.
841 * When called with startio set it is important that we write the WHOLE
842 * page if possible.
843 * The bh->b_state's cannot know if any of the blocks or which block for
844 * that matter are dirty due to mmap writes, and therefore bh uptodate is
845 * only vaild if the page itself isn't completely uptodate. Some layers
846 * may clear the page dirty flag prior to calling write page, under the
847 * assumption the entire page will be written out; by not writing out the
848 * whole page the page can be reused before all valid dirty data is
849 * written out. Note: in the case of a page that has been dirty'd by
850 * mapwrite and but partially setup by block_prepare_write the
851 * bh->b_states's will not agree and only ones setup by BPW/BCW will have
852 * valid state, thus the whole page must be written out thing.
855 STATIC int
856 xfs_page_state_convert(
857 struct inode *inode,
858 struct page *page,
859 struct writeback_control *wbc,
860 int startio,
861 int unmapped) /* also implies page uptodate */
863 struct buffer_head *bh, *head;
864 xfs_iomap_t iomap;
865 xfs_ioend_t *ioend = NULL, *iohead = NULL;
866 loff_t offset;
867 unsigned long p_offset = 0;
868 unsigned int type;
869 __uint64_t end_offset;
870 pgoff_t end_index, last_index, tlast;
871 ssize_t size, len;
872 int flags, err, iomap_valid = 0, uptodate = 1;
873 int page_dirty, count = 0, trylock_flag = 0;
874 int all_bh = unmapped;
876 /* wait for other IO threads? */
877 if (startio && (wbc->sync_mode == WB_SYNC_NONE && wbc->nonblocking))
878 trylock_flag |= BMAPI_TRYLOCK;
880 /* Is this page beyond the end of the file? */
881 offset = i_size_read(inode);
882 end_index = offset >> PAGE_CACHE_SHIFT;
883 last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
884 if (page->index >= end_index) {
885 if ((page->index >= end_index + 1) ||
886 !(i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
887 if (startio)
888 unlock_page(page);
889 return 0;
894 * page_dirty is initially a count of buffers on the page before
895 * EOF and is decrememted as we move each into a cleanable state.
897 * Derivation:
899 * End offset is the highest offset that this page should represent.
900 * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
901 * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
902 * hence give us the correct page_dirty count. On any other page,
903 * it will be zero and in that case we need page_dirty to be the
904 * count of buffers on the page.
906 end_offset = min_t(unsigned long long,
907 (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT, offset);
908 len = 1 << inode->i_blkbits;
909 p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
910 PAGE_CACHE_SIZE);
911 p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
912 page_dirty = p_offset / len;
914 bh = head = page_buffers(page);
915 offset = page_offset(page);
916 flags = -1;
917 type = 0;
919 /* TODO: cleanup count and page_dirty */
921 do {
922 if (offset >= end_offset)
923 break;
924 if (!buffer_uptodate(bh))
925 uptodate = 0;
926 if (!(PageUptodate(page) || buffer_uptodate(bh)) && !startio) {
928 * the iomap is actually still valid, but the ioend
929 * isn't. shouldn't happen too often.
931 iomap_valid = 0;
932 continue;
935 if (iomap_valid)
936 iomap_valid = xfs_iomap_valid(&iomap, offset);
939 * First case, map an unwritten extent and prepare for
940 * extent state conversion transaction on completion.
942 * Second case, allocate space for a delalloc buffer.
943 * We can return EAGAIN here in the release page case.
945 * Third case, an unmapped buffer was found, and we are
946 * in a path where we need to write the whole page out.
948 if (buffer_unwritten(bh) || buffer_delay(bh) ||
949 ((buffer_uptodate(bh) || PageUptodate(page)) &&
950 !buffer_mapped(bh) && (unmapped || startio))) {
952 * Make sure we don't use a read-only iomap
954 if (flags == BMAPI_READ)
955 iomap_valid = 0;
957 if (buffer_unwritten(bh)) {
958 type = IOMAP_UNWRITTEN;
959 flags = BMAPI_WRITE|BMAPI_IGNSTATE;
960 } else if (buffer_delay(bh)) {
961 type = IOMAP_DELAY;
962 flags = BMAPI_ALLOCATE;
963 if (!startio)
964 flags |= trylock_flag;
965 } else {
966 type = IOMAP_NEW;
967 flags = BMAPI_WRITE|BMAPI_MMAP;
970 if (!iomap_valid) {
971 if (type == IOMAP_NEW) {
972 size = xfs_probe_cluster(inode,
973 page, bh, head, 0);
974 } else {
975 size = len;
978 err = xfs_map_blocks(inode, offset, size,
979 &iomap, flags);
980 if (err)
981 goto error;
982 iomap_valid = xfs_iomap_valid(&iomap, offset);
984 if (iomap_valid) {
985 xfs_map_at_offset(bh, offset,
986 inode->i_blkbits, &iomap);
987 if (startio) {
988 xfs_add_to_ioend(inode, bh, offset,
989 type, &ioend,
990 !iomap_valid);
991 } else {
992 set_buffer_dirty(bh);
993 unlock_buffer(bh);
994 mark_buffer_dirty(bh);
996 page_dirty--;
997 count++;
999 } else if (buffer_uptodate(bh) && startio) {
1001 * we got here because the buffer is already mapped.
1002 * That means it must already have extents allocated
1003 * underneath it. Map the extent by reading it.
1005 if (!iomap_valid || type != 0) {
1006 flags = BMAPI_READ;
1007 size = xfs_probe_cluster(inode, page, bh,
1008 head, 1);
1009 err = xfs_map_blocks(inode, offset, size,
1010 &iomap, flags);
1011 if (err)
1012 goto error;
1013 iomap_valid = xfs_iomap_valid(&iomap, offset);
1016 type = 0;
1017 if (!test_and_set_bit(BH_Lock, &bh->b_state)) {
1018 ASSERT(buffer_mapped(bh));
1019 if (iomap_valid)
1020 all_bh = 1;
1021 xfs_add_to_ioend(inode, bh, offset, type,
1022 &ioend, !iomap_valid);
1023 page_dirty--;
1024 count++;
1025 } else {
1026 iomap_valid = 0;
1028 } else if ((buffer_uptodate(bh) || PageUptodate(page)) &&
1029 (unmapped || startio)) {
1030 iomap_valid = 0;
1033 if (!iohead)
1034 iohead = ioend;
1036 } while (offset += len, ((bh = bh->b_this_page) != head));
1038 if (uptodate && bh == head)
1039 SetPageUptodate(page);
1041 if (startio)
1042 xfs_start_page_writeback(page, wbc, 1, count);
1044 if (ioend && iomap_valid) {
1045 offset = (iomap.iomap_offset + iomap.iomap_bsize - 1) >>
1046 PAGE_CACHE_SHIFT;
1047 tlast = min_t(pgoff_t, offset, last_index);
1048 xfs_cluster_write(inode, page->index + 1, &iomap, &ioend,
1049 wbc, startio, all_bh, tlast);
1052 if (iohead)
1053 xfs_submit_ioend(iohead);
1055 return page_dirty;
1057 error:
1058 if (iohead)
1059 xfs_cancel_ioend(iohead);
1062 * If it's delalloc and we have nowhere to put it,
1063 * throw it away, unless the lower layers told
1064 * us to try again.
1066 if (err != -EAGAIN) {
1067 if (!unmapped)
1068 block_invalidatepage(page, 0);
1069 ClearPageUptodate(page);
1071 return err;
1075 * writepage: Called from one of two places:
1077 * 1. we are flushing a delalloc buffer head.
1079 * 2. we are writing out a dirty page. Typically the page dirty
1080 * state is cleared before we get here. In this case is it
1081 * conceivable we have no buffer heads.
1083 * For delalloc space on the page we need to allocate space and
1084 * flush it. For unmapped buffer heads on the page we should
1085 * allocate space if the page is uptodate. For any other dirty
1086 * buffer heads on the page we should flush them.
1088 * If we detect that a transaction would be required to flush
1089 * the page, we have to check the process flags first, if we
1090 * are already in a transaction or disk I/O during allocations
1091 * is off, we need to fail the writepage and redirty the page.
1094 STATIC int
1095 xfs_vm_writepage(
1096 struct page *page,
1097 struct writeback_control *wbc)
1099 int error;
1100 int need_trans;
1101 int delalloc, unmapped, unwritten;
1102 struct inode *inode = page->mapping->host;
1104 xfs_page_trace(XFS_WRITEPAGE_ENTER, inode, page, 0);
1107 * We need a transaction if:
1108 * 1. There are delalloc buffers on the page
1109 * 2. The page is uptodate and we have unmapped buffers
1110 * 3. The page is uptodate and we have no buffers
1111 * 4. There are unwritten buffers on the page
1114 if (!page_has_buffers(page)) {
1115 unmapped = 1;
1116 need_trans = 1;
1117 } else {
1118 xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
1119 if (!PageUptodate(page))
1120 unmapped = 0;
1121 need_trans = delalloc + unmapped + unwritten;
1125 * If we need a transaction and the process flags say
1126 * we are already in a transaction, or no IO is allowed
1127 * then mark the page dirty again and leave the page
1128 * as is.
1130 if (PFLAGS_TEST_FSTRANS() && need_trans)
1131 goto out_fail;
1134 * Delay hooking up buffer heads until we have
1135 * made our go/no-go decision.
1137 if (!page_has_buffers(page))
1138 create_empty_buffers(page, 1 << inode->i_blkbits, 0);
1141 * Convert delayed allocate, unwritten or unmapped space
1142 * to real space and flush out to disk.
1144 error = xfs_page_state_convert(inode, page, wbc, 1, unmapped);
1145 if (error == -EAGAIN)
1146 goto out_fail;
1147 if (unlikely(error < 0))
1148 goto out_unlock;
1150 return 0;
1152 out_fail:
1153 redirty_page_for_writepage(wbc, page);
1154 unlock_page(page);
1155 return 0;
1156 out_unlock:
1157 unlock_page(page);
1158 return error;
1162 * Called to move a page into cleanable state - and from there
1163 * to be released. Possibly the page is already clean. We always
1164 * have buffer heads in this call.
1166 * Returns 0 if the page is ok to release, 1 otherwise.
1168 * Possible scenarios are:
1170 * 1. We are being called to release a page which has been written
1171 * to via regular I/O. buffer heads will be dirty and possibly
1172 * delalloc. If no delalloc buffer heads in this case then we
1173 * can just return zero.
1175 * 2. We are called to release a page which has been written via
1176 * mmap, all we need to do is ensure there is no delalloc
1177 * state in the buffer heads, if not we can let the caller
1178 * free them and we should come back later via writepage.
1180 STATIC int
1181 xfs_vm_releasepage(
1182 struct page *page,
1183 gfp_t gfp_mask)
1185 struct inode *inode = page->mapping->host;
1186 int dirty, delalloc, unmapped, unwritten;
1187 struct writeback_control wbc = {
1188 .sync_mode = WB_SYNC_ALL,
1189 .nr_to_write = 1,
1192 xfs_page_trace(XFS_RELEASEPAGE_ENTER, inode, page, gfp_mask);
1194 if (!page_has_buffers(page))
1195 return 0;
1197 xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
1198 if (!delalloc && !unwritten)
1199 goto free_buffers;
1201 if (!(gfp_mask & __GFP_FS))
1202 return 0;
1204 /* If we are already inside a transaction or the thread cannot
1205 * do I/O, we cannot release this page.
1207 if (PFLAGS_TEST_FSTRANS())
1208 return 0;
1211 * Convert delalloc space to real space, do not flush the
1212 * data out to disk, that will be done by the caller.
1213 * Never need to allocate space here - we will always
1214 * come back to writepage in that case.
1216 dirty = xfs_page_state_convert(inode, page, &wbc, 0, 0);
1217 if (dirty == 0 && !unwritten)
1218 goto free_buffers;
1219 return 0;
1221 free_buffers:
1222 return try_to_free_buffers(page);
1225 STATIC int
1226 __xfs_get_block(
1227 struct inode *inode,
1228 sector_t iblock,
1229 unsigned long blocks,
1230 struct buffer_head *bh_result,
1231 int create,
1232 int direct,
1233 bmapi_flags_t flags)
1235 vnode_t *vp = vn_from_inode(inode);
1236 xfs_iomap_t iomap;
1237 xfs_off_t offset;
1238 ssize_t size;
1239 int retpbbm = 1;
1240 int error;
1242 offset = (xfs_off_t)iblock << inode->i_blkbits;
1243 if (blocks)
1244 size = (ssize_t) min_t(xfs_off_t, LONG_MAX,
1245 (xfs_off_t)blocks << inode->i_blkbits);
1246 else
1247 size = 1 << inode->i_blkbits;
1249 VOP_BMAP(vp, offset, size,
1250 create ? flags : BMAPI_READ, &iomap, &retpbbm, error);
1251 if (error)
1252 return -error;
1254 if (retpbbm == 0)
1255 return 0;
1257 if (iomap.iomap_bn != IOMAP_DADDR_NULL) {
1259 * For unwritten extents do not report a disk address on
1260 * the read case (treat as if we're reading into a hole).
1262 if (create || !(iomap.iomap_flags & IOMAP_UNWRITTEN)) {
1263 xfs_map_buffer(bh_result, &iomap, offset,
1264 inode->i_blkbits);
1266 if (create && (iomap.iomap_flags & IOMAP_UNWRITTEN)) {
1267 if (direct)
1268 bh_result->b_private = inode;
1269 set_buffer_unwritten(bh_result);
1270 set_buffer_delay(bh_result);
1274 /* If this is a realtime file, data might be on a new device */
1275 bh_result->b_bdev = iomap.iomap_target->bt_bdev;
1277 /* If we previously allocated a block out beyond eof and
1278 * we are now coming back to use it then we will need to
1279 * flag it as new even if it has a disk address.
1281 if (create &&
1282 ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
1283 (offset >= i_size_read(inode)) || (iomap.iomap_flags & IOMAP_NEW)))
1284 set_buffer_new(bh_result);
1286 if (iomap.iomap_flags & IOMAP_DELAY) {
1287 BUG_ON(direct);
1288 if (create) {
1289 set_buffer_uptodate(bh_result);
1290 set_buffer_mapped(bh_result);
1291 set_buffer_delay(bh_result);
1295 if (blocks) {
1296 ASSERT(iomap.iomap_bsize - iomap.iomap_delta > 0);
1297 offset = min_t(xfs_off_t,
1298 iomap.iomap_bsize - iomap.iomap_delta,
1299 (xfs_off_t)blocks << inode->i_blkbits);
1300 bh_result->b_size = (u32) min_t(xfs_off_t, UINT_MAX, offset);
1303 return 0;
1307 xfs_get_block(
1308 struct inode *inode,
1309 sector_t iblock,
1310 struct buffer_head *bh_result,
1311 int create)
1313 return __xfs_get_block(inode, iblock, 0, bh_result,
1314 create, 0, BMAPI_WRITE);
1317 STATIC int
1318 xfs_get_blocks_direct(
1319 struct inode *inode,
1320 sector_t iblock,
1321 unsigned long max_blocks,
1322 struct buffer_head *bh_result,
1323 int create)
1325 return __xfs_get_block(inode, iblock, max_blocks, bh_result,
1326 create, 1, BMAPI_WRITE|BMAPI_DIRECT);
1329 STATIC void
1330 xfs_end_io_direct(
1331 struct kiocb *iocb,
1332 loff_t offset,
1333 ssize_t size,
1334 void *private)
1336 xfs_ioend_t *ioend = iocb->private;
1339 * Non-NULL private data means we need to issue a transaction to
1340 * convert a range from unwritten to written extents. This needs
1341 * to happen from process contect but aio+dio I/O completion
1342 * happens from irq context so we need to defer it to a workqueue.
1343 * This is not nessecary for synchronous direct I/O, but we do
1344 * it anyway to keep the code uniform and simpler.
1346 * The core direct I/O code might be changed to always call the
1347 * completion handler in the future, in which case all this can
1348 * go away.
1350 if (private && size > 0) {
1351 ioend->io_offset = offset;
1352 ioend->io_size = size;
1353 xfs_finish_ioend(ioend);
1354 } else {
1355 ASSERT(size >= 0);
1356 xfs_destroy_ioend(ioend);
1360 * blockdev_direct_IO can return an error even afer the I/O
1361 * completion handler was called. Thus we need to protect
1362 * against double-freeing.
1364 iocb->private = NULL;
1367 STATIC ssize_t
1368 xfs_vm_direct_IO(
1369 int rw,
1370 struct kiocb *iocb,
1371 const struct iovec *iov,
1372 loff_t offset,
1373 unsigned long nr_segs)
1375 struct file *file = iocb->ki_filp;
1376 struct inode *inode = file->f_mapping->host;
1377 vnode_t *vp = vn_from_inode(inode);
1378 xfs_iomap_t iomap;
1379 int maps = 1;
1380 int error;
1381 ssize_t ret;
1383 VOP_BMAP(vp, offset, 0, BMAPI_DEVICE, &iomap, &maps, error);
1384 if (error)
1385 return -error;
1387 iocb->private = xfs_alloc_ioend(inode, IOMAP_UNWRITTEN);
1389 ret = blockdev_direct_IO_own_locking(rw, iocb, inode,
1390 iomap.iomap_target->bt_bdev,
1391 iov, offset, nr_segs,
1392 xfs_get_blocks_direct,
1393 xfs_end_io_direct);
1395 if (unlikely(ret <= 0 && iocb->private))
1396 xfs_destroy_ioend(iocb->private);
1397 return ret;
1400 STATIC int
1401 xfs_vm_prepare_write(
1402 struct file *file,
1403 struct page *page,
1404 unsigned int from,
1405 unsigned int to)
1407 return block_prepare_write(page, from, to, xfs_get_block);
1410 STATIC sector_t
1411 xfs_vm_bmap(
1412 struct address_space *mapping,
1413 sector_t block)
1415 struct inode *inode = (struct inode *)mapping->host;
1416 vnode_t *vp = vn_from_inode(inode);
1417 int error;
1419 vn_trace_entry(vp, __FUNCTION__, (inst_t *)__return_address);
1421 VOP_RWLOCK(vp, VRWLOCK_READ);
1422 VOP_FLUSH_PAGES(vp, (xfs_off_t)0, -1, 0, FI_REMAPF, error);
1423 VOP_RWUNLOCK(vp, VRWLOCK_READ);
1424 return generic_block_bmap(mapping, block, xfs_get_block);
1427 STATIC int
1428 xfs_vm_readpage(
1429 struct file *unused,
1430 struct page *page)
1432 return mpage_readpage(page, xfs_get_block);
1435 STATIC int
1436 xfs_vm_readpages(
1437 struct file *unused,
1438 struct address_space *mapping,
1439 struct list_head *pages,
1440 unsigned nr_pages)
1442 return mpage_readpages(mapping, pages, nr_pages, xfs_get_block);
1445 STATIC void
1446 xfs_vm_invalidatepage(
1447 struct page *page,
1448 unsigned long offset)
1450 xfs_page_trace(XFS_INVALIDPAGE_ENTER,
1451 page->mapping->host, page, offset);
1452 block_invalidatepage(page, offset);
1455 struct address_space_operations xfs_address_space_operations = {
1456 .readpage = xfs_vm_readpage,
1457 .readpages = xfs_vm_readpages,
1458 .writepage = xfs_vm_writepage,
1459 .sync_page = block_sync_page,
1460 .releasepage = xfs_vm_releasepage,
1461 .invalidatepage = xfs_vm_invalidatepage,
1462 .prepare_write = xfs_vm_prepare_write,
1463 .commit_write = generic_commit_write,
1464 .bmap = xfs_vm_bmap,
1465 .direct_IO = xfs_vm_direct_IO,
1466 .migratepage = buffer_migrate_page,