Merge branch 'sh/for-2.6.33' of git://git.kernel.org/pub/scm/linux/kernel/git/lethal...
[linux-2.6/linux-2.6-openrd.git] / fs / buffer.c
blob6fa530256bfd7b118bbef4aebce7ceb79e145928
1 /*
2 * linux/fs/buffer.c
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
7 /*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/capability.h>
28 #include <linux/blkdev.h>
29 #include <linux/file.h>
30 #include <linux/quotaops.h>
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/writeback.h>
34 #include <linux/hash.h>
35 #include <linux/suspend.h>
36 #include <linux/buffer_head.h>
37 #include <linux/task_io_accounting_ops.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
47 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
49 inline void
50 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52 bh->b_end_io = handler;
53 bh->b_private = private;
55 EXPORT_SYMBOL(init_buffer);
57 static int sync_buffer(void *word)
59 struct block_device *bd;
60 struct buffer_head *bh
61 = container_of(word, struct buffer_head, b_state);
63 smp_mb();
64 bd = bh->b_bdev;
65 if (bd)
66 blk_run_address_space(bd->bd_inode->i_mapping);
67 io_schedule();
68 return 0;
71 void __lock_buffer(struct buffer_head *bh)
73 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74 TASK_UNINTERRUPTIBLE);
76 EXPORT_SYMBOL(__lock_buffer);
78 void unlock_buffer(struct buffer_head *bh)
80 clear_bit_unlock(BH_Lock, &bh->b_state);
81 smp_mb__after_clear_bit();
82 wake_up_bit(&bh->b_state, BH_Lock);
84 EXPORT_SYMBOL(unlock_buffer);
87 * Block until a buffer comes unlocked. This doesn't stop it
88 * from becoming locked again - you have to lock it yourself
89 * if you want to preserve its state.
91 void __wait_on_buffer(struct buffer_head * bh)
93 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
95 EXPORT_SYMBOL(__wait_on_buffer);
97 static void
98 __clear_page_buffers(struct page *page)
100 ClearPagePrivate(page);
101 set_page_private(page, 0);
102 page_cache_release(page);
106 static int quiet_error(struct buffer_head *bh)
108 if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
109 return 0;
110 return 1;
114 static void buffer_io_error(struct buffer_head *bh)
116 char b[BDEVNAME_SIZE];
117 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
118 bdevname(bh->b_bdev, b),
119 (unsigned long long)bh->b_blocknr);
123 * End-of-IO handler helper function which does not touch the bh after
124 * unlocking it.
125 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
126 * a race there is benign: unlock_buffer() only use the bh's address for
127 * hashing after unlocking the buffer, so it doesn't actually touch the bh
128 * itself.
130 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
132 if (uptodate) {
133 set_buffer_uptodate(bh);
134 } else {
135 /* This happens, due to failed READA attempts. */
136 clear_buffer_uptodate(bh);
138 unlock_buffer(bh);
142 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
143 * unlock the buffer. This is what ll_rw_block uses too.
145 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
147 __end_buffer_read_notouch(bh, uptodate);
148 put_bh(bh);
150 EXPORT_SYMBOL(end_buffer_read_sync);
152 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
154 char b[BDEVNAME_SIZE];
156 if (uptodate) {
157 set_buffer_uptodate(bh);
158 } else {
159 if (!buffer_eopnotsupp(bh) && !quiet_error(bh)) {
160 buffer_io_error(bh);
161 printk(KERN_WARNING "lost page write due to "
162 "I/O error on %s\n",
163 bdevname(bh->b_bdev, b));
165 set_buffer_write_io_error(bh);
166 clear_buffer_uptodate(bh);
168 unlock_buffer(bh);
169 put_bh(bh);
171 EXPORT_SYMBOL(end_buffer_write_sync);
174 * Various filesystems appear to want __find_get_block to be non-blocking.
175 * But it's the page lock which protects the buffers. To get around this,
176 * we get exclusion from try_to_free_buffers with the blockdev mapping's
177 * private_lock.
179 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
180 * may be quite high. This code could TryLock the page, and if that
181 * succeeds, there is no need to take private_lock. (But if
182 * private_lock is contended then so is mapping->tree_lock).
184 static struct buffer_head *
185 __find_get_block_slow(struct block_device *bdev, sector_t block)
187 struct inode *bd_inode = bdev->bd_inode;
188 struct address_space *bd_mapping = bd_inode->i_mapping;
189 struct buffer_head *ret = NULL;
190 pgoff_t index;
191 struct buffer_head *bh;
192 struct buffer_head *head;
193 struct page *page;
194 int all_mapped = 1;
196 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
197 page = find_get_page(bd_mapping, index);
198 if (!page)
199 goto out;
201 spin_lock(&bd_mapping->private_lock);
202 if (!page_has_buffers(page))
203 goto out_unlock;
204 head = page_buffers(page);
205 bh = head;
206 do {
207 if (!buffer_mapped(bh))
208 all_mapped = 0;
209 else if (bh->b_blocknr == block) {
210 ret = bh;
211 get_bh(bh);
212 goto out_unlock;
214 bh = bh->b_this_page;
215 } while (bh != head);
217 /* we might be here because some of the buffers on this page are
218 * not mapped. This is due to various races between
219 * file io on the block device and getblk. It gets dealt with
220 * elsewhere, don't buffer_error if we had some unmapped buffers
222 if (all_mapped) {
223 printk("__find_get_block_slow() failed. "
224 "block=%llu, b_blocknr=%llu\n",
225 (unsigned long long)block,
226 (unsigned long long)bh->b_blocknr);
227 printk("b_state=0x%08lx, b_size=%zu\n",
228 bh->b_state, bh->b_size);
229 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
231 out_unlock:
232 spin_unlock(&bd_mapping->private_lock);
233 page_cache_release(page);
234 out:
235 return ret;
238 /* If invalidate_buffers() will trash dirty buffers, it means some kind
239 of fs corruption is going on. Trashing dirty data always imply losing
240 information that was supposed to be just stored on the physical layer
241 by the user.
243 Thus invalidate_buffers in general usage is not allwowed to trash
244 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
245 be preserved. These buffers are simply skipped.
247 We also skip buffers which are still in use. For example this can
248 happen if a userspace program is reading the block device.
250 NOTE: In the case where the user removed a removable-media-disk even if
251 there's still dirty data not synced on disk (due a bug in the device driver
252 or due an error of the user), by not destroying the dirty buffers we could
253 generate corruption also on the next media inserted, thus a parameter is
254 necessary to handle this case in the most safe way possible (trying
255 to not corrupt also the new disk inserted with the data belonging to
256 the old now corrupted disk). Also for the ramdisk the natural thing
257 to do in order to release the ramdisk memory is to destroy dirty buffers.
259 These are two special cases. Normal usage imply the device driver
260 to issue a sync on the device (without waiting I/O completion) and
261 then an invalidate_buffers call that doesn't trash dirty buffers.
263 For handling cache coherency with the blkdev pagecache the 'update' case
264 is been introduced. It is needed to re-read from disk any pinned
265 buffer. NOTE: re-reading from disk is destructive so we can do it only
266 when we assume nobody is changing the buffercache under our I/O and when
267 we think the disk contains more recent information than the buffercache.
268 The update == 1 pass marks the buffers we need to update, the update == 2
269 pass does the actual I/O. */
270 void invalidate_bdev(struct block_device *bdev)
272 struct address_space *mapping = bdev->bd_inode->i_mapping;
274 if (mapping->nrpages == 0)
275 return;
277 invalidate_bh_lrus();
278 invalidate_mapping_pages(mapping, 0, -1);
280 EXPORT_SYMBOL(invalidate_bdev);
283 * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
285 static void free_more_memory(void)
287 struct zone *zone;
288 int nid;
290 wakeup_flusher_threads(1024);
291 yield();
293 for_each_online_node(nid) {
294 (void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
295 gfp_zone(GFP_NOFS), NULL,
296 &zone);
297 if (zone)
298 try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
299 GFP_NOFS, NULL);
304 * I/O completion handler for block_read_full_page() - pages
305 * which come unlocked at the end of I/O.
307 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
309 unsigned long flags;
310 struct buffer_head *first;
311 struct buffer_head *tmp;
312 struct page *page;
313 int page_uptodate = 1;
315 BUG_ON(!buffer_async_read(bh));
317 page = bh->b_page;
318 if (uptodate) {
319 set_buffer_uptodate(bh);
320 } else {
321 clear_buffer_uptodate(bh);
322 if (!quiet_error(bh))
323 buffer_io_error(bh);
324 SetPageError(page);
328 * Be _very_ careful from here on. Bad things can happen if
329 * two buffer heads end IO at almost the same time and both
330 * decide that the page is now completely done.
332 first = page_buffers(page);
333 local_irq_save(flags);
334 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
335 clear_buffer_async_read(bh);
336 unlock_buffer(bh);
337 tmp = bh;
338 do {
339 if (!buffer_uptodate(tmp))
340 page_uptodate = 0;
341 if (buffer_async_read(tmp)) {
342 BUG_ON(!buffer_locked(tmp));
343 goto still_busy;
345 tmp = tmp->b_this_page;
346 } while (tmp != bh);
347 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
348 local_irq_restore(flags);
351 * If none of the buffers had errors and they are all
352 * uptodate then we can set the page uptodate.
354 if (page_uptodate && !PageError(page))
355 SetPageUptodate(page);
356 unlock_page(page);
357 return;
359 still_busy:
360 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
361 local_irq_restore(flags);
362 return;
366 * Completion handler for block_write_full_page() - pages which are unlocked
367 * during I/O, and which have PageWriteback cleared upon I/O completion.
369 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
371 char b[BDEVNAME_SIZE];
372 unsigned long flags;
373 struct buffer_head *first;
374 struct buffer_head *tmp;
375 struct page *page;
377 BUG_ON(!buffer_async_write(bh));
379 page = bh->b_page;
380 if (uptodate) {
381 set_buffer_uptodate(bh);
382 } else {
383 if (!quiet_error(bh)) {
384 buffer_io_error(bh);
385 printk(KERN_WARNING "lost page write due to "
386 "I/O error on %s\n",
387 bdevname(bh->b_bdev, b));
389 set_bit(AS_EIO, &page->mapping->flags);
390 set_buffer_write_io_error(bh);
391 clear_buffer_uptodate(bh);
392 SetPageError(page);
395 first = page_buffers(page);
396 local_irq_save(flags);
397 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
399 clear_buffer_async_write(bh);
400 unlock_buffer(bh);
401 tmp = bh->b_this_page;
402 while (tmp != bh) {
403 if (buffer_async_write(tmp)) {
404 BUG_ON(!buffer_locked(tmp));
405 goto still_busy;
407 tmp = tmp->b_this_page;
409 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
410 local_irq_restore(flags);
411 end_page_writeback(page);
412 return;
414 still_busy:
415 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
416 local_irq_restore(flags);
417 return;
419 EXPORT_SYMBOL(end_buffer_async_write);
422 * If a page's buffers are under async readin (end_buffer_async_read
423 * completion) then there is a possibility that another thread of
424 * control could lock one of the buffers after it has completed
425 * but while some of the other buffers have not completed. This
426 * locked buffer would confuse end_buffer_async_read() into not unlocking
427 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
428 * that this buffer is not under async I/O.
430 * The page comes unlocked when it has no locked buffer_async buffers
431 * left.
433 * PageLocked prevents anyone starting new async I/O reads any of
434 * the buffers.
436 * PageWriteback is used to prevent simultaneous writeout of the same
437 * page.
439 * PageLocked prevents anyone from starting writeback of a page which is
440 * under read I/O (PageWriteback is only ever set against a locked page).
442 static void mark_buffer_async_read(struct buffer_head *bh)
444 bh->b_end_io = end_buffer_async_read;
445 set_buffer_async_read(bh);
448 static void mark_buffer_async_write_endio(struct buffer_head *bh,
449 bh_end_io_t *handler)
451 bh->b_end_io = handler;
452 set_buffer_async_write(bh);
455 void mark_buffer_async_write(struct buffer_head *bh)
457 mark_buffer_async_write_endio(bh, end_buffer_async_write);
459 EXPORT_SYMBOL(mark_buffer_async_write);
463 * fs/buffer.c contains helper functions for buffer-backed address space's
464 * fsync functions. A common requirement for buffer-based filesystems is
465 * that certain data from the backing blockdev needs to be written out for
466 * a successful fsync(). For example, ext2 indirect blocks need to be
467 * written back and waited upon before fsync() returns.
469 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
470 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
471 * management of a list of dependent buffers at ->i_mapping->private_list.
473 * Locking is a little subtle: try_to_free_buffers() will remove buffers
474 * from their controlling inode's queue when they are being freed. But
475 * try_to_free_buffers() will be operating against the *blockdev* mapping
476 * at the time, not against the S_ISREG file which depends on those buffers.
477 * So the locking for private_list is via the private_lock in the address_space
478 * which backs the buffers. Which is different from the address_space
479 * against which the buffers are listed. So for a particular address_space,
480 * mapping->private_lock does *not* protect mapping->private_list! In fact,
481 * mapping->private_list will always be protected by the backing blockdev's
482 * ->private_lock.
484 * Which introduces a requirement: all buffers on an address_space's
485 * ->private_list must be from the same address_space: the blockdev's.
487 * address_spaces which do not place buffers at ->private_list via these
488 * utility functions are free to use private_lock and private_list for
489 * whatever they want. The only requirement is that list_empty(private_list)
490 * be true at clear_inode() time.
492 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
493 * filesystems should do that. invalidate_inode_buffers() should just go
494 * BUG_ON(!list_empty).
496 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
497 * take an address_space, not an inode. And it should be called
498 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
499 * queued up.
501 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
502 * list if it is already on a list. Because if the buffer is on a list,
503 * it *must* already be on the right one. If not, the filesystem is being
504 * silly. This will save a ton of locking. But first we have to ensure
505 * that buffers are taken *off* the old inode's list when they are freed
506 * (presumably in truncate). That requires careful auditing of all
507 * filesystems (do it inside bforget()). It could also be done by bringing
508 * b_inode back.
512 * The buffer's backing address_space's private_lock must be held
514 static void __remove_assoc_queue(struct buffer_head *bh)
516 list_del_init(&bh->b_assoc_buffers);
517 WARN_ON(!bh->b_assoc_map);
518 if (buffer_write_io_error(bh))
519 set_bit(AS_EIO, &bh->b_assoc_map->flags);
520 bh->b_assoc_map = NULL;
523 int inode_has_buffers(struct inode *inode)
525 return !list_empty(&inode->i_data.private_list);
529 * osync is designed to support O_SYNC io. It waits synchronously for
530 * all already-submitted IO to complete, but does not queue any new
531 * writes to the disk.
533 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
534 * you dirty the buffers, and then use osync_inode_buffers to wait for
535 * completion. Any other dirty buffers which are not yet queued for
536 * write will not be flushed to disk by the osync.
538 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
540 struct buffer_head *bh;
541 struct list_head *p;
542 int err = 0;
544 spin_lock(lock);
545 repeat:
546 list_for_each_prev(p, list) {
547 bh = BH_ENTRY(p);
548 if (buffer_locked(bh)) {
549 get_bh(bh);
550 spin_unlock(lock);
551 wait_on_buffer(bh);
552 if (!buffer_uptodate(bh))
553 err = -EIO;
554 brelse(bh);
555 spin_lock(lock);
556 goto repeat;
559 spin_unlock(lock);
560 return err;
563 static void do_thaw_all(struct work_struct *work)
565 struct super_block *sb;
566 char b[BDEVNAME_SIZE];
568 spin_lock(&sb_lock);
569 restart:
570 list_for_each_entry(sb, &super_blocks, s_list) {
571 sb->s_count++;
572 spin_unlock(&sb_lock);
573 down_read(&sb->s_umount);
574 while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
575 printk(KERN_WARNING "Emergency Thaw on %s\n",
576 bdevname(sb->s_bdev, b));
577 up_read(&sb->s_umount);
578 spin_lock(&sb_lock);
579 if (__put_super_and_need_restart(sb))
580 goto restart;
582 spin_unlock(&sb_lock);
583 kfree(work);
584 printk(KERN_WARNING "Emergency Thaw complete\n");
588 * emergency_thaw_all -- forcibly thaw every frozen filesystem
590 * Used for emergency unfreeze of all filesystems via SysRq
592 void emergency_thaw_all(void)
594 struct work_struct *work;
596 work = kmalloc(sizeof(*work), GFP_ATOMIC);
597 if (work) {
598 INIT_WORK(work, do_thaw_all);
599 schedule_work(work);
604 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
605 * @mapping: the mapping which wants those buffers written
607 * Starts I/O against the buffers at mapping->private_list, and waits upon
608 * that I/O.
610 * Basically, this is a convenience function for fsync().
611 * @mapping is a file or directory which needs those buffers to be written for
612 * a successful fsync().
614 int sync_mapping_buffers(struct address_space *mapping)
616 struct address_space *buffer_mapping = mapping->assoc_mapping;
618 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
619 return 0;
621 return fsync_buffers_list(&buffer_mapping->private_lock,
622 &mapping->private_list);
624 EXPORT_SYMBOL(sync_mapping_buffers);
627 * Called when we've recently written block `bblock', and it is known that
628 * `bblock' was for a buffer_boundary() buffer. This means that the block at
629 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
630 * dirty, schedule it for IO. So that indirects merge nicely with their data.
632 void write_boundary_block(struct block_device *bdev,
633 sector_t bblock, unsigned blocksize)
635 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
636 if (bh) {
637 if (buffer_dirty(bh))
638 ll_rw_block(WRITE, 1, &bh);
639 put_bh(bh);
643 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
645 struct address_space *mapping = inode->i_mapping;
646 struct address_space *buffer_mapping = bh->b_page->mapping;
648 mark_buffer_dirty(bh);
649 if (!mapping->assoc_mapping) {
650 mapping->assoc_mapping = buffer_mapping;
651 } else {
652 BUG_ON(mapping->assoc_mapping != buffer_mapping);
654 if (!bh->b_assoc_map) {
655 spin_lock(&buffer_mapping->private_lock);
656 list_move_tail(&bh->b_assoc_buffers,
657 &mapping->private_list);
658 bh->b_assoc_map = mapping;
659 spin_unlock(&buffer_mapping->private_lock);
662 EXPORT_SYMBOL(mark_buffer_dirty_inode);
665 * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
666 * dirty.
668 * If warn is true, then emit a warning if the page is not uptodate and has
669 * not been truncated.
671 static void __set_page_dirty(struct page *page,
672 struct address_space *mapping, int warn)
674 spin_lock_irq(&mapping->tree_lock);
675 if (page->mapping) { /* Race with truncate? */
676 WARN_ON_ONCE(warn && !PageUptodate(page));
677 account_page_dirtied(page, mapping);
678 radix_tree_tag_set(&mapping->page_tree,
679 page_index(page), PAGECACHE_TAG_DIRTY);
681 spin_unlock_irq(&mapping->tree_lock);
682 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
686 * Add a page to the dirty page list.
688 * It is a sad fact of life that this function is called from several places
689 * deeply under spinlocking. It may not sleep.
691 * If the page has buffers, the uptodate buffers are set dirty, to preserve
692 * dirty-state coherency between the page and the buffers. It the page does
693 * not have buffers then when they are later attached they will all be set
694 * dirty.
696 * The buffers are dirtied before the page is dirtied. There's a small race
697 * window in which a writepage caller may see the page cleanness but not the
698 * buffer dirtiness. That's fine. If this code were to set the page dirty
699 * before the buffers, a concurrent writepage caller could clear the page dirty
700 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
701 * page on the dirty page list.
703 * We use private_lock to lock against try_to_free_buffers while using the
704 * page's buffer list. Also use this to protect against clean buffers being
705 * added to the page after it was set dirty.
707 * FIXME: may need to call ->reservepage here as well. That's rather up to the
708 * address_space though.
710 int __set_page_dirty_buffers(struct page *page)
712 int newly_dirty;
713 struct address_space *mapping = page_mapping(page);
715 if (unlikely(!mapping))
716 return !TestSetPageDirty(page);
718 spin_lock(&mapping->private_lock);
719 if (page_has_buffers(page)) {
720 struct buffer_head *head = page_buffers(page);
721 struct buffer_head *bh = head;
723 do {
724 set_buffer_dirty(bh);
725 bh = bh->b_this_page;
726 } while (bh != head);
728 newly_dirty = !TestSetPageDirty(page);
729 spin_unlock(&mapping->private_lock);
731 if (newly_dirty)
732 __set_page_dirty(page, mapping, 1);
733 return newly_dirty;
735 EXPORT_SYMBOL(__set_page_dirty_buffers);
738 * Write out and wait upon a list of buffers.
740 * We have conflicting pressures: we want to make sure that all
741 * initially dirty buffers get waited on, but that any subsequently
742 * dirtied buffers don't. After all, we don't want fsync to last
743 * forever if somebody is actively writing to the file.
745 * Do this in two main stages: first we copy dirty buffers to a
746 * temporary inode list, queueing the writes as we go. Then we clean
747 * up, waiting for those writes to complete.
749 * During this second stage, any subsequent updates to the file may end
750 * up refiling the buffer on the original inode's dirty list again, so
751 * there is a chance we will end up with a buffer queued for write but
752 * not yet completed on that list. So, as a final cleanup we go through
753 * the osync code to catch these locked, dirty buffers without requeuing
754 * any newly dirty buffers for write.
756 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
758 struct buffer_head *bh;
759 struct list_head tmp;
760 struct address_space *mapping, *prev_mapping = NULL;
761 int err = 0, err2;
763 INIT_LIST_HEAD(&tmp);
765 spin_lock(lock);
766 while (!list_empty(list)) {
767 bh = BH_ENTRY(list->next);
768 mapping = bh->b_assoc_map;
769 __remove_assoc_queue(bh);
770 /* Avoid race with mark_buffer_dirty_inode() which does
771 * a lockless check and we rely on seeing the dirty bit */
772 smp_mb();
773 if (buffer_dirty(bh) || buffer_locked(bh)) {
774 list_add(&bh->b_assoc_buffers, &tmp);
775 bh->b_assoc_map = mapping;
776 if (buffer_dirty(bh)) {
777 get_bh(bh);
778 spin_unlock(lock);
780 * Ensure any pending I/O completes so that
781 * ll_rw_block() actually writes the current
782 * contents - it is a noop if I/O is still in
783 * flight on potentially older contents.
785 ll_rw_block(SWRITE_SYNC_PLUG, 1, &bh);
788 * Kick off IO for the previous mapping. Note
789 * that we will not run the very last mapping,
790 * wait_on_buffer() will do that for us
791 * through sync_buffer().
793 if (prev_mapping && prev_mapping != mapping)
794 blk_run_address_space(prev_mapping);
795 prev_mapping = mapping;
797 brelse(bh);
798 spin_lock(lock);
803 while (!list_empty(&tmp)) {
804 bh = BH_ENTRY(tmp.prev);
805 get_bh(bh);
806 mapping = bh->b_assoc_map;
807 __remove_assoc_queue(bh);
808 /* Avoid race with mark_buffer_dirty_inode() which does
809 * a lockless check and we rely on seeing the dirty bit */
810 smp_mb();
811 if (buffer_dirty(bh)) {
812 list_add(&bh->b_assoc_buffers,
813 &mapping->private_list);
814 bh->b_assoc_map = mapping;
816 spin_unlock(lock);
817 wait_on_buffer(bh);
818 if (!buffer_uptodate(bh))
819 err = -EIO;
820 brelse(bh);
821 spin_lock(lock);
824 spin_unlock(lock);
825 err2 = osync_buffers_list(lock, list);
826 if (err)
827 return err;
828 else
829 return err2;
833 * Invalidate any and all dirty buffers on a given inode. We are
834 * probably unmounting the fs, but that doesn't mean we have already
835 * done a sync(). Just drop the buffers from the inode list.
837 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
838 * assumes that all the buffers are against the blockdev. Not true
839 * for reiserfs.
841 void invalidate_inode_buffers(struct inode *inode)
843 if (inode_has_buffers(inode)) {
844 struct address_space *mapping = &inode->i_data;
845 struct list_head *list = &mapping->private_list;
846 struct address_space *buffer_mapping = mapping->assoc_mapping;
848 spin_lock(&buffer_mapping->private_lock);
849 while (!list_empty(list))
850 __remove_assoc_queue(BH_ENTRY(list->next));
851 spin_unlock(&buffer_mapping->private_lock);
854 EXPORT_SYMBOL(invalidate_inode_buffers);
857 * Remove any clean buffers from the inode's buffer list. This is called
858 * when we're trying to free the inode itself. Those buffers can pin it.
860 * Returns true if all buffers were removed.
862 int remove_inode_buffers(struct inode *inode)
864 int ret = 1;
866 if (inode_has_buffers(inode)) {
867 struct address_space *mapping = &inode->i_data;
868 struct list_head *list = &mapping->private_list;
869 struct address_space *buffer_mapping = mapping->assoc_mapping;
871 spin_lock(&buffer_mapping->private_lock);
872 while (!list_empty(list)) {
873 struct buffer_head *bh = BH_ENTRY(list->next);
874 if (buffer_dirty(bh)) {
875 ret = 0;
876 break;
878 __remove_assoc_queue(bh);
880 spin_unlock(&buffer_mapping->private_lock);
882 return ret;
886 * Create the appropriate buffers when given a page for data area and
887 * the size of each buffer.. Use the bh->b_this_page linked list to
888 * follow the buffers created. Return NULL if unable to create more
889 * buffers.
891 * The retry flag is used to differentiate async IO (paging, swapping)
892 * which may not fail from ordinary buffer allocations.
894 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
895 int retry)
897 struct buffer_head *bh, *head;
898 long offset;
900 try_again:
901 head = NULL;
902 offset = PAGE_SIZE;
903 while ((offset -= size) >= 0) {
904 bh = alloc_buffer_head(GFP_NOFS);
905 if (!bh)
906 goto no_grow;
908 bh->b_bdev = NULL;
909 bh->b_this_page = head;
910 bh->b_blocknr = -1;
911 head = bh;
913 bh->b_state = 0;
914 atomic_set(&bh->b_count, 0);
915 bh->b_private = NULL;
916 bh->b_size = size;
918 /* Link the buffer to its page */
919 set_bh_page(bh, page, offset);
921 init_buffer(bh, NULL, NULL);
923 return head;
925 * In case anything failed, we just free everything we got.
927 no_grow:
928 if (head) {
929 do {
930 bh = head;
931 head = head->b_this_page;
932 free_buffer_head(bh);
933 } while (head);
937 * Return failure for non-async IO requests. Async IO requests
938 * are not allowed to fail, so we have to wait until buffer heads
939 * become available. But we don't want tasks sleeping with
940 * partially complete buffers, so all were released above.
942 if (!retry)
943 return NULL;
945 /* We're _really_ low on memory. Now we just
946 * wait for old buffer heads to become free due to
947 * finishing IO. Since this is an async request and
948 * the reserve list is empty, we're sure there are
949 * async buffer heads in use.
951 free_more_memory();
952 goto try_again;
954 EXPORT_SYMBOL_GPL(alloc_page_buffers);
956 static inline void
957 link_dev_buffers(struct page *page, struct buffer_head *head)
959 struct buffer_head *bh, *tail;
961 bh = head;
962 do {
963 tail = bh;
964 bh = bh->b_this_page;
965 } while (bh);
966 tail->b_this_page = head;
967 attach_page_buffers(page, head);
971 * Initialise the state of a blockdev page's buffers.
973 static void
974 init_page_buffers(struct page *page, struct block_device *bdev,
975 sector_t block, int size)
977 struct buffer_head *head = page_buffers(page);
978 struct buffer_head *bh = head;
979 int uptodate = PageUptodate(page);
981 do {
982 if (!buffer_mapped(bh)) {
983 init_buffer(bh, NULL, NULL);
984 bh->b_bdev = bdev;
985 bh->b_blocknr = block;
986 if (uptodate)
987 set_buffer_uptodate(bh);
988 set_buffer_mapped(bh);
990 block++;
991 bh = bh->b_this_page;
992 } while (bh != head);
996 * Create the page-cache page that contains the requested block.
998 * This is user purely for blockdev mappings.
1000 static struct page *
1001 grow_dev_page(struct block_device *bdev, sector_t block,
1002 pgoff_t index, int size)
1004 struct inode *inode = bdev->bd_inode;
1005 struct page *page;
1006 struct buffer_head *bh;
1008 page = find_or_create_page(inode->i_mapping, index,
1009 (mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE);
1010 if (!page)
1011 return NULL;
1013 BUG_ON(!PageLocked(page));
1015 if (page_has_buffers(page)) {
1016 bh = page_buffers(page);
1017 if (bh->b_size == size) {
1018 init_page_buffers(page, bdev, block, size);
1019 return page;
1021 if (!try_to_free_buffers(page))
1022 goto failed;
1026 * Allocate some buffers for this page
1028 bh = alloc_page_buffers(page, size, 0);
1029 if (!bh)
1030 goto failed;
1033 * Link the page to the buffers and initialise them. Take the
1034 * lock to be atomic wrt __find_get_block(), which does not
1035 * run under the page lock.
1037 spin_lock(&inode->i_mapping->private_lock);
1038 link_dev_buffers(page, bh);
1039 init_page_buffers(page, bdev, block, size);
1040 spin_unlock(&inode->i_mapping->private_lock);
1041 return page;
1043 failed:
1044 BUG();
1045 unlock_page(page);
1046 page_cache_release(page);
1047 return NULL;
1051 * Create buffers for the specified block device block's page. If
1052 * that page was dirty, the buffers are set dirty also.
1054 static int
1055 grow_buffers(struct block_device *bdev, sector_t block, int size)
1057 struct page *page;
1058 pgoff_t index;
1059 int sizebits;
1061 sizebits = -1;
1062 do {
1063 sizebits++;
1064 } while ((size << sizebits) < PAGE_SIZE);
1066 index = block >> sizebits;
1069 * Check for a block which wants to lie outside our maximum possible
1070 * pagecache index. (this comparison is done using sector_t types).
1072 if (unlikely(index != block >> sizebits)) {
1073 char b[BDEVNAME_SIZE];
1075 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1076 "device %s\n",
1077 __func__, (unsigned long long)block,
1078 bdevname(bdev, b));
1079 return -EIO;
1081 block = index << sizebits;
1082 /* Create a page with the proper size buffers.. */
1083 page = grow_dev_page(bdev, block, index, size);
1084 if (!page)
1085 return 0;
1086 unlock_page(page);
1087 page_cache_release(page);
1088 return 1;
1091 static struct buffer_head *
1092 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1094 /* Size must be multiple of hard sectorsize */
1095 if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1096 (size < 512 || size > PAGE_SIZE))) {
1097 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1098 size);
1099 printk(KERN_ERR "logical block size: %d\n",
1100 bdev_logical_block_size(bdev));
1102 dump_stack();
1103 return NULL;
1106 for (;;) {
1107 struct buffer_head * bh;
1108 int ret;
1110 bh = __find_get_block(bdev, block, size);
1111 if (bh)
1112 return bh;
1114 ret = grow_buffers(bdev, block, size);
1115 if (ret < 0)
1116 return NULL;
1117 if (ret == 0)
1118 free_more_memory();
1123 * The relationship between dirty buffers and dirty pages:
1125 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1126 * the page is tagged dirty in its radix tree.
1128 * At all times, the dirtiness of the buffers represents the dirtiness of
1129 * subsections of the page. If the page has buffers, the page dirty bit is
1130 * merely a hint about the true dirty state.
1132 * When a page is set dirty in its entirety, all its buffers are marked dirty
1133 * (if the page has buffers).
1135 * When a buffer is marked dirty, its page is dirtied, but the page's other
1136 * buffers are not.
1138 * Also. When blockdev buffers are explicitly read with bread(), they
1139 * individually become uptodate. But their backing page remains not
1140 * uptodate - even if all of its buffers are uptodate. A subsequent
1141 * block_read_full_page() against that page will discover all the uptodate
1142 * buffers, will set the page uptodate and will perform no I/O.
1146 * mark_buffer_dirty - mark a buffer_head as needing writeout
1147 * @bh: the buffer_head to mark dirty
1149 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1150 * backing page dirty, then tag the page as dirty in its address_space's radix
1151 * tree and then attach the address_space's inode to its superblock's dirty
1152 * inode list.
1154 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1155 * mapping->tree_lock and the global inode_lock.
1157 void mark_buffer_dirty(struct buffer_head *bh)
1159 WARN_ON_ONCE(!buffer_uptodate(bh));
1162 * Very *carefully* optimize the it-is-already-dirty case.
1164 * Don't let the final "is it dirty" escape to before we
1165 * perhaps modified the buffer.
1167 if (buffer_dirty(bh)) {
1168 smp_mb();
1169 if (buffer_dirty(bh))
1170 return;
1173 if (!test_set_buffer_dirty(bh)) {
1174 struct page *page = bh->b_page;
1175 if (!TestSetPageDirty(page)) {
1176 struct address_space *mapping = page_mapping(page);
1177 if (mapping)
1178 __set_page_dirty(page, mapping, 0);
1182 EXPORT_SYMBOL(mark_buffer_dirty);
1185 * Decrement a buffer_head's reference count. If all buffers against a page
1186 * have zero reference count, are clean and unlocked, and if the page is clean
1187 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1188 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1189 * a page but it ends up not being freed, and buffers may later be reattached).
1191 void __brelse(struct buffer_head * buf)
1193 if (atomic_read(&buf->b_count)) {
1194 put_bh(buf);
1195 return;
1197 WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1199 EXPORT_SYMBOL(__brelse);
1202 * bforget() is like brelse(), except it discards any
1203 * potentially dirty data.
1205 void __bforget(struct buffer_head *bh)
1207 clear_buffer_dirty(bh);
1208 if (bh->b_assoc_map) {
1209 struct address_space *buffer_mapping = bh->b_page->mapping;
1211 spin_lock(&buffer_mapping->private_lock);
1212 list_del_init(&bh->b_assoc_buffers);
1213 bh->b_assoc_map = NULL;
1214 spin_unlock(&buffer_mapping->private_lock);
1216 __brelse(bh);
1218 EXPORT_SYMBOL(__bforget);
1220 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1222 lock_buffer(bh);
1223 if (buffer_uptodate(bh)) {
1224 unlock_buffer(bh);
1225 return bh;
1226 } else {
1227 get_bh(bh);
1228 bh->b_end_io = end_buffer_read_sync;
1229 submit_bh(READ, bh);
1230 wait_on_buffer(bh);
1231 if (buffer_uptodate(bh))
1232 return bh;
1234 brelse(bh);
1235 return NULL;
1239 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1240 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1241 * refcount elevated by one when they're in an LRU. A buffer can only appear
1242 * once in a particular CPU's LRU. A single buffer can be present in multiple
1243 * CPU's LRUs at the same time.
1245 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1246 * sb_find_get_block().
1248 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1249 * a local interrupt disable for that.
1252 #define BH_LRU_SIZE 8
1254 struct bh_lru {
1255 struct buffer_head *bhs[BH_LRU_SIZE];
1258 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1260 #ifdef CONFIG_SMP
1261 #define bh_lru_lock() local_irq_disable()
1262 #define bh_lru_unlock() local_irq_enable()
1263 #else
1264 #define bh_lru_lock() preempt_disable()
1265 #define bh_lru_unlock() preempt_enable()
1266 #endif
1268 static inline void check_irqs_on(void)
1270 #ifdef irqs_disabled
1271 BUG_ON(irqs_disabled());
1272 #endif
1276 * The LRU management algorithm is dopey-but-simple. Sorry.
1278 static void bh_lru_install(struct buffer_head *bh)
1280 struct buffer_head *evictee = NULL;
1281 struct bh_lru *lru;
1283 check_irqs_on();
1284 bh_lru_lock();
1285 lru = &__get_cpu_var(bh_lrus);
1286 if (lru->bhs[0] != bh) {
1287 struct buffer_head *bhs[BH_LRU_SIZE];
1288 int in;
1289 int out = 0;
1291 get_bh(bh);
1292 bhs[out++] = bh;
1293 for (in = 0; in < BH_LRU_SIZE; in++) {
1294 struct buffer_head *bh2 = lru->bhs[in];
1296 if (bh2 == bh) {
1297 __brelse(bh2);
1298 } else {
1299 if (out >= BH_LRU_SIZE) {
1300 BUG_ON(evictee != NULL);
1301 evictee = bh2;
1302 } else {
1303 bhs[out++] = bh2;
1307 while (out < BH_LRU_SIZE)
1308 bhs[out++] = NULL;
1309 memcpy(lru->bhs, bhs, sizeof(bhs));
1311 bh_lru_unlock();
1313 if (evictee)
1314 __brelse(evictee);
1318 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1320 static struct buffer_head *
1321 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1323 struct buffer_head *ret = NULL;
1324 struct bh_lru *lru;
1325 unsigned int i;
1327 check_irqs_on();
1328 bh_lru_lock();
1329 lru = &__get_cpu_var(bh_lrus);
1330 for (i = 0; i < BH_LRU_SIZE; i++) {
1331 struct buffer_head *bh = lru->bhs[i];
1333 if (bh && bh->b_bdev == bdev &&
1334 bh->b_blocknr == block && bh->b_size == size) {
1335 if (i) {
1336 while (i) {
1337 lru->bhs[i] = lru->bhs[i - 1];
1338 i--;
1340 lru->bhs[0] = bh;
1342 get_bh(bh);
1343 ret = bh;
1344 break;
1347 bh_lru_unlock();
1348 return ret;
1352 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1353 * it in the LRU and mark it as accessed. If it is not present then return
1354 * NULL
1356 struct buffer_head *
1357 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1359 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1361 if (bh == NULL) {
1362 bh = __find_get_block_slow(bdev, block);
1363 if (bh)
1364 bh_lru_install(bh);
1366 if (bh)
1367 touch_buffer(bh);
1368 return bh;
1370 EXPORT_SYMBOL(__find_get_block);
1373 * __getblk will locate (and, if necessary, create) the buffer_head
1374 * which corresponds to the passed block_device, block and size. The
1375 * returned buffer has its reference count incremented.
1377 * __getblk() cannot fail - it just keeps trying. If you pass it an
1378 * illegal block number, __getblk() will happily return a buffer_head
1379 * which represents the non-existent block. Very weird.
1381 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1382 * attempt is failing. FIXME, perhaps?
1384 struct buffer_head *
1385 __getblk(struct block_device *bdev, sector_t block, unsigned size)
1387 struct buffer_head *bh = __find_get_block(bdev, block, size);
1389 might_sleep();
1390 if (bh == NULL)
1391 bh = __getblk_slow(bdev, block, size);
1392 return bh;
1394 EXPORT_SYMBOL(__getblk);
1397 * Do async read-ahead on a buffer..
1399 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1401 struct buffer_head *bh = __getblk(bdev, block, size);
1402 if (likely(bh)) {
1403 ll_rw_block(READA, 1, &bh);
1404 brelse(bh);
1407 EXPORT_SYMBOL(__breadahead);
1410 * __bread() - reads a specified block and returns the bh
1411 * @bdev: the block_device to read from
1412 * @block: number of block
1413 * @size: size (in bytes) to read
1415 * Reads a specified block, and returns buffer head that contains it.
1416 * It returns NULL if the block was unreadable.
1418 struct buffer_head *
1419 __bread(struct block_device *bdev, sector_t block, unsigned size)
1421 struct buffer_head *bh = __getblk(bdev, block, size);
1423 if (likely(bh) && !buffer_uptodate(bh))
1424 bh = __bread_slow(bh);
1425 return bh;
1427 EXPORT_SYMBOL(__bread);
1430 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1431 * This doesn't race because it runs in each cpu either in irq
1432 * or with preempt disabled.
1434 static void invalidate_bh_lru(void *arg)
1436 struct bh_lru *b = &get_cpu_var(bh_lrus);
1437 int i;
1439 for (i = 0; i < BH_LRU_SIZE; i++) {
1440 brelse(b->bhs[i]);
1441 b->bhs[i] = NULL;
1443 put_cpu_var(bh_lrus);
1446 void invalidate_bh_lrus(void)
1448 on_each_cpu(invalidate_bh_lru, NULL, 1);
1450 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1452 void set_bh_page(struct buffer_head *bh,
1453 struct page *page, unsigned long offset)
1455 bh->b_page = page;
1456 BUG_ON(offset >= PAGE_SIZE);
1457 if (PageHighMem(page))
1459 * This catches illegal uses and preserves the offset:
1461 bh->b_data = (char *)(0 + offset);
1462 else
1463 bh->b_data = page_address(page) + offset;
1465 EXPORT_SYMBOL(set_bh_page);
1468 * Called when truncating a buffer on a page completely.
1470 static void discard_buffer(struct buffer_head * bh)
1472 lock_buffer(bh);
1473 clear_buffer_dirty(bh);
1474 bh->b_bdev = NULL;
1475 clear_buffer_mapped(bh);
1476 clear_buffer_req(bh);
1477 clear_buffer_new(bh);
1478 clear_buffer_delay(bh);
1479 clear_buffer_unwritten(bh);
1480 unlock_buffer(bh);
1484 * block_invalidatepage - invalidate part of all of a buffer-backed page
1486 * @page: the page which is affected
1487 * @offset: the index of the truncation point
1489 * block_invalidatepage() is called when all or part of the page has become
1490 * invalidatedby a truncate operation.
1492 * block_invalidatepage() does not have to release all buffers, but it must
1493 * ensure that no dirty buffer is left outside @offset and that no I/O
1494 * is underway against any of the blocks which are outside the truncation
1495 * point. Because the caller is about to free (and possibly reuse) those
1496 * blocks on-disk.
1498 void block_invalidatepage(struct page *page, unsigned long offset)
1500 struct buffer_head *head, *bh, *next;
1501 unsigned int curr_off = 0;
1503 BUG_ON(!PageLocked(page));
1504 if (!page_has_buffers(page))
1505 goto out;
1507 head = page_buffers(page);
1508 bh = head;
1509 do {
1510 unsigned int next_off = curr_off + bh->b_size;
1511 next = bh->b_this_page;
1514 * is this block fully invalidated?
1516 if (offset <= curr_off)
1517 discard_buffer(bh);
1518 curr_off = next_off;
1519 bh = next;
1520 } while (bh != head);
1523 * We release buffers only if the entire page is being invalidated.
1524 * The get_block cached value has been unconditionally invalidated,
1525 * so real IO is not possible anymore.
1527 if (offset == 0)
1528 try_to_release_page(page, 0);
1529 out:
1530 return;
1532 EXPORT_SYMBOL(block_invalidatepage);
1535 * We attach and possibly dirty the buffers atomically wrt
1536 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1537 * is already excluded via the page lock.
1539 void create_empty_buffers(struct page *page,
1540 unsigned long blocksize, unsigned long b_state)
1542 struct buffer_head *bh, *head, *tail;
1544 head = alloc_page_buffers(page, blocksize, 1);
1545 bh = head;
1546 do {
1547 bh->b_state |= b_state;
1548 tail = bh;
1549 bh = bh->b_this_page;
1550 } while (bh);
1551 tail->b_this_page = head;
1553 spin_lock(&page->mapping->private_lock);
1554 if (PageUptodate(page) || PageDirty(page)) {
1555 bh = head;
1556 do {
1557 if (PageDirty(page))
1558 set_buffer_dirty(bh);
1559 if (PageUptodate(page))
1560 set_buffer_uptodate(bh);
1561 bh = bh->b_this_page;
1562 } while (bh != head);
1564 attach_page_buffers(page, head);
1565 spin_unlock(&page->mapping->private_lock);
1567 EXPORT_SYMBOL(create_empty_buffers);
1570 * We are taking a block for data and we don't want any output from any
1571 * buffer-cache aliases starting from return from that function and
1572 * until the moment when something will explicitly mark the buffer
1573 * dirty (hopefully that will not happen until we will free that block ;-)
1574 * We don't even need to mark it not-uptodate - nobody can expect
1575 * anything from a newly allocated buffer anyway. We used to used
1576 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1577 * don't want to mark the alias unmapped, for example - it would confuse
1578 * anyone who might pick it with bread() afterwards...
1580 * Also.. Note that bforget() doesn't lock the buffer. So there can
1581 * be writeout I/O going on against recently-freed buffers. We don't
1582 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1583 * only if we really need to. That happens here.
1585 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1587 struct buffer_head *old_bh;
1589 might_sleep();
1591 old_bh = __find_get_block_slow(bdev, block);
1592 if (old_bh) {
1593 clear_buffer_dirty(old_bh);
1594 wait_on_buffer(old_bh);
1595 clear_buffer_req(old_bh);
1596 __brelse(old_bh);
1599 EXPORT_SYMBOL(unmap_underlying_metadata);
1602 * NOTE! All mapped/uptodate combinations are valid:
1604 * Mapped Uptodate Meaning
1606 * No No "unknown" - must do get_block()
1607 * No Yes "hole" - zero-filled
1608 * Yes No "allocated" - allocated on disk, not read in
1609 * Yes Yes "valid" - allocated and up-to-date in memory.
1611 * "Dirty" is valid only with the last case (mapped+uptodate).
1615 * While block_write_full_page is writing back the dirty buffers under
1616 * the page lock, whoever dirtied the buffers may decide to clean them
1617 * again at any time. We handle that by only looking at the buffer
1618 * state inside lock_buffer().
1620 * If block_write_full_page() is called for regular writeback
1621 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1622 * locked buffer. This only can happen if someone has written the buffer
1623 * directly, with submit_bh(). At the address_space level PageWriteback
1624 * prevents this contention from occurring.
1626 * If block_write_full_page() is called with wbc->sync_mode ==
1627 * WB_SYNC_ALL, the writes are posted using WRITE_SYNC_PLUG; this
1628 * causes the writes to be flagged as synchronous writes, but the
1629 * block device queue will NOT be unplugged, since usually many pages
1630 * will be pushed to the out before the higher-level caller actually
1631 * waits for the writes to be completed. The various wait functions,
1632 * such as wait_on_writeback_range() will ultimately call sync_page()
1633 * which will ultimately call blk_run_backing_dev(), which will end up
1634 * unplugging the device queue.
1636 static int __block_write_full_page(struct inode *inode, struct page *page,
1637 get_block_t *get_block, struct writeback_control *wbc,
1638 bh_end_io_t *handler)
1640 int err;
1641 sector_t block;
1642 sector_t last_block;
1643 struct buffer_head *bh, *head;
1644 const unsigned blocksize = 1 << inode->i_blkbits;
1645 int nr_underway = 0;
1646 int write_op = (wbc->sync_mode == WB_SYNC_ALL ?
1647 WRITE_SYNC_PLUG : WRITE);
1649 BUG_ON(!PageLocked(page));
1651 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1653 if (!page_has_buffers(page)) {
1654 create_empty_buffers(page, blocksize,
1655 (1 << BH_Dirty)|(1 << BH_Uptodate));
1659 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1660 * here, and the (potentially unmapped) buffers may become dirty at
1661 * any time. If a buffer becomes dirty here after we've inspected it
1662 * then we just miss that fact, and the page stays dirty.
1664 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1665 * handle that here by just cleaning them.
1668 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1669 head = page_buffers(page);
1670 bh = head;
1673 * Get all the dirty buffers mapped to disk addresses and
1674 * handle any aliases from the underlying blockdev's mapping.
1676 do {
1677 if (block > last_block) {
1679 * mapped buffers outside i_size will occur, because
1680 * this page can be outside i_size when there is a
1681 * truncate in progress.
1684 * The buffer was zeroed by block_write_full_page()
1686 clear_buffer_dirty(bh);
1687 set_buffer_uptodate(bh);
1688 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1689 buffer_dirty(bh)) {
1690 WARN_ON(bh->b_size != blocksize);
1691 err = get_block(inode, block, bh, 1);
1692 if (err)
1693 goto recover;
1694 clear_buffer_delay(bh);
1695 if (buffer_new(bh)) {
1696 /* blockdev mappings never come here */
1697 clear_buffer_new(bh);
1698 unmap_underlying_metadata(bh->b_bdev,
1699 bh->b_blocknr);
1702 bh = bh->b_this_page;
1703 block++;
1704 } while (bh != head);
1706 do {
1707 if (!buffer_mapped(bh))
1708 continue;
1710 * If it's a fully non-blocking write attempt and we cannot
1711 * lock the buffer then redirty the page. Note that this can
1712 * potentially cause a busy-wait loop from writeback threads
1713 * and kswapd activity, but those code paths have their own
1714 * higher-level throttling.
1716 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1717 lock_buffer(bh);
1718 } else if (!trylock_buffer(bh)) {
1719 redirty_page_for_writepage(wbc, page);
1720 continue;
1722 if (test_clear_buffer_dirty(bh)) {
1723 mark_buffer_async_write_endio(bh, handler);
1724 } else {
1725 unlock_buffer(bh);
1727 } while ((bh = bh->b_this_page) != head);
1730 * The page and its buffers are protected by PageWriteback(), so we can
1731 * drop the bh refcounts early.
1733 BUG_ON(PageWriteback(page));
1734 set_page_writeback(page);
1736 do {
1737 struct buffer_head *next = bh->b_this_page;
1738 if (buffer_async_write(bh)) {
1739 submit_bh(write_op, bh);
1740 nr_underway++;
1742 bh = next;
1743 } while (bh != head);
1744 unlock_page(page);
1746 err = 0;
1747 done:
1748 if (nr_underway == 0) {
1750 * The page was marked dirty, but the buffers were
1751 * clean. Someone wrote them back by hand with
1752 * ll_rw_block/submit_bh. A rare case.
1754 end_page_writeback(page);
1757 * The page and buffer_heads can be released at any time from
1758 * here on.
1761 return err;
1763 recover:
1765 * ENOSPC, or some other error. We may already have added some
1766 * blocks to the file, so we need to write these out to avoid
1767 * exposing stale data.
1768 * The page is currently locked and not marked for writeback
1770 bh = head;
1771 /* Recovery: lock and submit the mapped buffers */
1772 do {
1773 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1774 !buffer_delay(bh)) {
1775 lock_buffer(bh);
1776 mark_buffer_async_write_endio(bh, handler);
1777 } else {
1779 * The buffer may have been set dirty during
1780 * attachment to a dirty page.
1782 clear_buffer_dirty(bh);
1784 } while ((bh = bh->b_this_page) != head);
1785 SetPageError(page);
1786 BUG_ON(PageWriteback(page));
1787 mapping_set_error(page->mapping, err);
1788 set_page_writeback(page);
1789 do {
1790 struct buffer_head *next = bh->b_this_page;
1791 if (buffer_async_write(bh)) {
1792 clear_buffer_dirty(bh);
1793 submit_bh(write_op, bh);
1794 nr_underway++;
1796 bh = next;
1797 } while (bh != head);
1798 unlock_page(page);
1799 goto done;
1803 * If a page has any new buffers, zero them out here, and mark them uptodate
1804 * and dirty so they'll be written out (in order to prevent uninitialised
1805 * block data from leaking). And clear the new bit.
1807 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1809 unsigned int block_start, block_end;
1810 struct buffer_head *head, *bh;
1812 BUG_ON(!PageLocked(page));
1813 if (!page_has_buffers(page))
1814 return;
1816 bh = head = page_buffers(page);
1817 block_start = 0;
1818 do {
1819 block_end = block_start + bh->b_size;
1821 if (buffer_new(bh)) {
1822 if (block_end > from && block_start < to) {
1823 if (!PageUptodate(page)) {
1824 unsigned start, size;
1826 start = max(from, block_start);
1827 size = min(to, block_end) - start;
1829 zero_user(page, start, size);
1830 set_buffer_uptodate(bh);
1833 clear_buffer_new(bh);
1834 mark_buffer_dirty(bh);
1838 block_start = block_end;
1839 bh = bh->b_this_page;
1840 } while (bh != head);
1842 EXPORT_SYMBOL(page_zero_new_buffers);
1844 static int __block_prepare_write(struct inode *inode, struct page *page,
1845 unsigned from, unsigned to, get_block_t *get_block)
1847 unsigned block_start, block_end;
1848 sector_t block;
1849 int err = 0;
1850 unsigned blocksize, bbits;
1851 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1853 BUG_ON(!PageLocked(page));
1854 BUG_ON(from > PAGE_CACHE_SIZE);
1855 BUG_ON(to > PAGE_CACHE_SIZE);
1856 BUG_ON(from > to);
1858 blocksize = 1 << inode->i_blkbits;
1859 if (!page_has_buffers(page))
1860 create_empty_buffers(page, blocksize, 0);
1861 head = page_buffers(page);
1863 bbits = inode->i_blkbits;
1864 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1866 for(bh = head, block_start = 0; bh != head || !block_start;
1867 block++, block_start=block_end, bh = bh->b_this_page) {
1868 block_end = block_start + blocksize;
1869 if (block_end <= from || block_start >= to) {
1870 if (PageUptodate(page)) {
1871 if (!buffer_uptodate(bh))
1872 set_buffer_uptodate(bh);
1874 continue;
1876 if (buffer_new(bh))
1877 clear_buffer_new(bh);
1878 if (!buffer_mapped(bh)) {
1879 WARN_ON(bh->b_size != blocksize);
1880 err = get_block(inode, block, bh, 1);
1881 if (err)
1882 break;
1883 if (buffer_new(bh)) {
1884 unmap_underlying_metadata(bh->b_bdev,
1885 bh->b_blocknr);
1886 if (PageUptodate(page)) {
1887 clear_buffer_new(bh);
1888 set_buffer_uptodate(bh);
1889 mark_buffer_dirty(bh);
1890 continue;
1892 if (block_end > to || block_start < from)
1893 zero_user_segments(page,
1894 to, block_end,
1895 block_start, from);
1896 continue;
1899 if (PageUptodate(page)) {
1900 if (!buffer_uptodate(bh))
1901 set_buffer_uptodate(bh);
1902 continue;
1904 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1905 !buffer_unwritten(bh) &&
1906 (block_start < from || block_end > to)) {
1907 ll_rw_block(READ, 1, &bh);
1908 *wait_bh++=bh;
1912 * If we issued read requests - let them complete.
1914 while(wait_bh > wait) {
1915 wait_on_buffer(*--wait_bh);
1916 if (!buffer_uptodate(*wait_bh))
1917 err = -EIO;
1919 if (unlikely(err))
1920 page_zero_new_buffers(page, from, to);
1921 return err;
1924 static int __block_commit_write(struct inode *inode, struct page *page,
1925 unsigned from, unsigned to)
1927 unsigned block_start, block_end;
1928 int partial = 0;
1929 unsigned blocksize;
1930 struct buffer_head *bh, *head;
1932 blocksize = 1 << inode->i_blkbits;
1934 for(bh = head = page_buffers(page), block_start = 0;
1935 bh != head || !block_start;
1936 block_start=block_end, bh = bh->b_this_page) {
1937 block_end = block_start + blocksize;
1938 if (block_end <= from || block_start >= to) {
1939 if (!buffer_uptodate(bh))
1940 partial = 1;
1941 } else {
1942 set_buffer_uptodate(bh);
1943 mark_buffer_dirty(bh);
1945 clear_buffer_new(bh);
1949 * If this is a partial write which happened to make all buffers
1950 * uptodate then we can optimize away a bogus readpage() for
1951 * the next read(). Here we 'discover' whether the page went
1952 * uptodate as a result of this (potentially partial) write.
1954 if (!partial)
1955 SetPageUptodate(page);
1956 return 0;
1960 * block_write_begin takes care of the basic task of block allocation and
1961 * bringing partial write blocks uptodate first.
1963 * If *pagep is not NULL, then block_write_begin uses the locked page
1964 * at *pagep rather than allocating its own. In this case, the page will
1965 * not be unlocked or deallocated on failure.
1967 int block_write_begin(struct file *file, struct address_space *mapping,
1968 loff_t pos, unsigned len, unsigned flags,
1969 struct page **pagep, void **fsdata,
1970 get_block_t *get_block)
1972 struct inode *inode = mapping->host;
1973 int status = 0;
1974 struct page *page;
1975 pgoff_t index;
1976 unsigned start, end;
1977 int ownpage = 0;
1979 index = pos >> PAGE_CACHE_SHIFT;
1980 start = pos & (PAGE_CACHE_SIZE - 1);
1981 end = start + len;
1983 page = *pagep;
1984 if (page == NULL) {
1985 ownpage = 1;
1986 page = grab_cache_page_write_begin(mapping, index, flags);
1987 if (!page) {
1988 status = -ENOMEM;
1989 goto out;
1991 *pagep = page;
1992 } else
1993 BUG_ON(!PageLocked(page));
1995 status = __block_prepare_write(inode, page, start, end, get_block);
1996 if (unlikely(status)) {
1997 ClearPageUptodate(page);
1999 if (ownpage) {
2000 unlock_page(page);
2001 page_cache_release(page);
2002 *pagep = NULL;
2005 * prepare_write() may have instantiated a few blocks
2006 * outside i_size. Trim these off again. Don't need
2007 * i_size_read because we hold i_mutex.
2009 if (pos + len > inode->i_size)
2010 vmtruncate(inode, inode->i_size);
2014 out:
2015 return status;
2017 EXPORT_SYMBOL(block_write_begin);
2019 int block_write_end(struct file *file, struct address_space *mapping,
2020 loff_t pos, unsigned len, unsigned copied,
2021 struct page *page, void *fsdata)
2023 struct inode *inode = mapping->host;
2024 unsigned start;
2026 start = pos & (PAGE_CACHE_SIZE - 1);
2028 if (unlikely(copied < len)) {
2030 * The buffers that were written will now be uptodate, so we
2031 * don't have to worry about a readpage reading them and
2032 * overwriting a partial write. However if we have encountered
2033 * a short write and only partially written into a buffer, it
2034 * will not be marked uptodate, so a readpage might come in and
2035 * destroy our partial write.
2037 * Do the simplest thing, and just treat any short write to a
2038 * non uptodate page as a zero-length write, and force the
2039 * caller to redo the whole thing.
2041 if (!PageUptodate(page))
2042 copied = 0;
2044 page_zero_new_buffers(page, start+copied, start+len);
2046 flush_dcache_page(page);
2048 /* This could be a short (even 0-length) commit */
2049 __block_commit_write(inode, page, start, start+copied);
2051 return copied;
2053 EXPORT_SYMBOL(block_write_end);
2055 int generic_write_end(struct file *file, struct address_space *mapping,
2056 loff_t pos, unsigned len, unsigned copied,
2057 struct page *page, void *fsdata)
2059 struct inode *inode = mapping->host;
2060 int i_size_changed = 0;
2062 copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2065 * No need to use i_size_read() here, the i_size
2066 * cannot change under us because we hold i_mutex.
2068 * But it's important to update i_size while still holding page lock:
2069 * page writeout could otherwise come in and zero beyond i_size.
2071 if (pos+copied > inode->i_size) {
2072 i_size_write(inode, pos+copied);
2073 i_size_changed = 1;
2076 unlock_page(page);
2077 page_cache_release(page);
2080 * Don't mark the inode dirty under page lock. First, it unnecessarily
2081 * makes the holding time of page lock longer. Second, it forces lock
2082 * ordering of page lock and transaction start for journaling
2083 * filesystems.
2085 if (i_size_changed)
2086 mark_inode_dirty(inode);
2088 return copied;
2090 EXPORT_SYMBOL(generic_write_end);
2093 * block_is_partially_uptodate checks whether buffers within a page are
2094 * uptodate or not.
2096 * Returns true if all buffers which correspond to a file portion
2097 * we want to read are uptodate.
2099 int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
2100 unsigned long from)
2102 struct inode *inode = page->mapping->host;
2103 unsigned block_start, block_end, blocksize;
2104 unsigned to;
2105 struct buffer_head *bh, *head;
2106 int ret = 1;
2108 if (!page_has_buffers(page))
2109 return 0;
2111 blocksize = 1 << inode->i_blkbits;
2112 to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
2113 to = from + to;
2114 if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
2115 return 0;
2117 head = page_buffers(page);
2118 bh = head;
2119 block_start = 0;
2120 do {
2121 block_end = block_start + blocksize;
2122 if (block_end > from && block_start < to) {
2123 if (!buffer_uptodate(bh)) {
2124 ret = 0;
2125 break;
2127 if (block_end >= to)
2128 break;
2130 block_start = block_end;
2131 bh = bh->b_this_page;
2132 } while (bh != head);
2134 return ret;
2136 EXPORT_SYMBOL(block_is_partially_uptodate);
2139 * Generic "read page" function for block devices that have the normal
2140 * get_block functionality. This is most of the block device filesystems.
2141 * Reads the page asynchronously --- the unlock_buffer() and
2142 * set/clear_buffer_uptodate() functions propagate buffer state into the
2143 * page struct once IO has completed.
2145 int block_read_full_page(struct page *page, get_block_t *get_block)
2147 struct inode *inode = page->mapping->host;
2148 sector_t iblock, lblock;
2149 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2150 unsigned int blocksize;
2151 int nr, i;
2152 int fully_mapped = 1;
2154 BUG_ON(!PageLocked(page));
2155 blocksize = 1 << inode->i_blkbits;
2156 if (!page_has_buffers(page))
2157 create_empty_buffers(page, blocksize, 0);
2158 head = page_buffers(page);
2160 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2161 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2162 bh = head;
2163 nr = 0;
2164 i = 0;
2166 do {
2167 if (buffer_uptodate(bh))
2168 continue;
2170 if (!buffer_mapped(bh)) {
2171 int err = 0;
2173 fully_mapped = 0;
2174 if (iblock < lblock) {
2175 WARN_ON(bh->b_size != blocksize);
2176 err = get_block(inode, iblock, bh, 0);
2177 if (err)
2178 SetPageError(page);
2180 if (!buffer_mapped(bh)) {
2181 zero_user(page, i * blocksize, blocksize);
2182 if (!err)
2183 set_buffer_uptodate(bh);
2184 continue;
2187 * get_block() might have updated the buffer
2188 * synchronously
2190 if (buffer_uptodate(bh))
2191 continue;
2193 arr[nr++] = bh;
2194 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2196 if (fully_mapped)
2197 SetPageMappedToDisk(page);
2199 if (!nr) {
2201 * All buffers are uptodate - we can set the page uptodate
2202 * as well. But not if get_block() returned an error.
2204 if (!PageError(page))
2205 SetPageUptodate(page);
2206 unlock_page(page);
2207 return 0;
2210 /* Stage two: lock the buffers */
2211 for (i = 0; i < nr; i++) {
2212 bh = arr[i];
2213 lock_buffer(bh);
2214 mark_buffer_async_read(bh);
2218 * Stage 3: start the IO. Check for uptodateness
2219 * inside the buffer lock in case another process reading
2220 * the underlying blockdev brought it uptodate (the sct fix).
2222 for (i = 0; i < nr; i++) {
2223 bh = arr[i];
2224 if (buffer_uptodate(bh))
2225 end_buffer_async_read(bh, 1);
2226 else
2227 submit_bh(READ, bh);
2229 return 0;
2231 EXPORT_SYMBOL(block_read_full_page);
2233 /* utility function for filesystems that need to do work on expanding
2234 * truncates. Uses filesystem pagecache writes to allow the filesystem to
2235 * deal with the hole.
2237 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2239 struct address_space *mapping = inode->i_mapping;
2240 struct page *page;
2241 void *fsdata;
2242 int err;
2244 err = inode_newsize_ok(inode, size);
2245 if (err)
2246 goto out;
2248 err = pagecache_write_begin(NULL, mapping, size, 0,
2249 AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
2250 &page, &fsdata);
2251 if (err)
2252 goto out;
2254 err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2255 BUG_ON(err > 0);
2257 out:
2258 return err;
2260 EXPORT_SYMBOL(generic_cont_expand_simple);
2262 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2263 loff_t pos, loff_t *bytes)
2265 struct inode *inode = mapping->host;
2266 unsigned blocksize = 1 << inode->i_blkbits;
2267 struct page *page;
2268 void *fsdata;
2269 pgoff_t index, curidx;
2270 loff_t curpos;
2271 unsigned zerofrom, offset, len;
2272 int err = 0;
2274 index = pos >> PAGE_CACHE_SHIFT;
2275 offset = pos & ~PAGE_CACHE_MASK;
2277 while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
2278 zerofrom = curpos & ~PAGE_CACHE_MASK;
2279 if (zerofrom & (blocksize-1)) {
2280 *bytes |= (blocksize-1);
2281 (*bytes)++;
2283 len = PAGE_CACHE_SIZE - zerofrom;
2285 err = pagecache_write_begin(file, mapping, curpos, len,
2286 AOP_FLAG_UNINTERRUPTIBLE,
2287 &page, &fsdata);
2288 if (err)
2289 goto out;
2290 zero_user(page, zerofrom, len);
2291 err = pagecache_write_end(file, mapping, curpos, len, len,
2292 page, fsdata);
2293 if (err < 0)
2294 goto out;
2295 BUG_ON(err != len);
2296 err = 0;
2298 balance_dirty_pages_ratelimited(mapping);
2301 /* page covers the boundary, find the boundary offset */
2302 if (index == curidx) {
2303 zerofrom = curpos & ~PAGE_CACHE_MASK;
2304 /* if we will expand the thing last block will be filled */
2305 if (offset <= zerofrom) {
2306 goto out;
2308 if (zerofrom & (blocksize-1)) {
2309 *bytes |= (blocksize-1);
2310 (*bytes)++;
2312 len = offset - zerofrom;
2314 err = pagecache_write_begin(file, mapping, curpos, len,
2315 AOP_FLAG_UNINTERRUPTIBLE,
2316 &page, &fsdata);
2317 if (err)
2318 goto out;
2319 zero_user(page, zerofrom, len);
2320 err = pagecache_write_end(file, mapping, curpos, len, len,
2321 page, fsdata);
2322 if (err < 0)
2323 goto out;
2324 BUG_ON(err != len);
2325 err = 0;
2327 out:
2328 return err;
2332 * For moronic filesystems that do not allow holes in file.
2333 * We may have to extend the file.
2335 int cont_write_begin(struct file *file, struct address_space *mapping,
2336 loff_t pos, unsigned len, unsigned flags,
2337 struct page **pagep, void **fsdata,
2338 get_block_t *get_block, loff_t *bytes)
2340 struct inode *inode = mapping->host;
2341 unsigned blocksize = 1 << inode->i_blkbits;
2342 unsigned zerofrom;
2343 int err;
2345 err = cont_expand_zero(file, mapping, pos, bytes);
2346 if (err)
2347 goto out;
2349 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2350 if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2351 *bytes |= (blocksize-1);
2352 (*bytes)++;
2355 *pagep = NULL;
2356 err = block_write_begin(file, mapping, pos, len,
2357 flags, pagep, fsdata, get_block);
2358 out:
2359 return err;
2361 EXPORT_SYMBOL(cont_write_begin);
2363 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2364 get_block_t *get_block)
2366 struct inode *inode = page->mapping->host;
2367 int err = __block_prepare_write(inode, page, from, to, get_block);
2368 if (err)
2369 ClearPageUptodate(page);
2370 return err;
2372 EXPORT_SYMBOL(block_prepare_write);
2374 int block_commit_write(struct page *page, unsigned from, unsigned to)
2376 struct inode *inode = page->mapping->host;
2377 __block_commit_write(inode,page,from,to);
2378 return 0;
2380 EXPORT_SYMBOL(block_commit_write);
2383 * block_page_mkwrite() is not allowed to change the file size as it gets
2384 * called from a page fault handler when a page is first dirtied. Hence we must
2385 * be careful to check for EOF conditions here. We set the page up correctly
2386 * for a written page which means we get ENOSPC checking when writing into
2387 * holes and correct delalloc and unwritten extent mapping on filesystems that
2388 * support these features.
2390 * We are not allowed to take the i_mutex here so we have to play games to
2391 * protect against truncate races as the page could now be beyond EOF. Because
2392 * vmtruncate() writes the inode size before removing pages, once we have the
2393 * page lock we can determine safely if the page is beyond EOF. If it is not
2394 * beyond EOF, then the page is guaranteed safe against truncation until we
2395 * unlock the page.
2398 block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2399 get_block_t get_block)
2401 struct page *page = vmf->page;
2402 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
2403 unsigned long end;
2404 loff_t size;
2405 int ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
2407 lock_page(page);
2408 size = i_size_read(inode);
2409 if ((page->mapping != inode->i_mapping) ||
2410 (page_offset(page) > size)) {
2411 /* page got truncated out from underneath us */
2412 unlock_page(page);
2413 goto out;
2416 /* page is wholly or partially inside EOF */
2417 if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
2418 end = size & ~PAGE_CACHE_MASK;
2419 else
2420 end = PAGE_CACHE_SIZE;
2422 ret = block_prepare_write(page, 0, end, get_block);
2423 if (!ret)
2424 ret = block_commit_write(page, 0, end);
2426 if (unlikely(ret)) {
2427 unlock_page(page);
2428 if (ret == -ENOMEM)
2429 ret = VM_FAULT_OOM;
2430 else /* -ENOSPC, -EIO, etc */
2431 ret = VM_FAULT_SIGBUS;
2432 } else
2433 ret = VM_FAULT_LOCKED;
2435 out:
2436 return ret;
2438 EXPORT_SYMBOL(block_page_mkwrite);
2441 * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2442 * immediately, while under the page lock. So it needs a special end_io
2443 * handler which does not touch the bh after unlocking it.
2445 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2447 __end_buffer_read_notouch(bh, uptodate);
2451 * Attach the singly-linked list of buffers created by nobh_write_begin, to
2452 * the page (converting it to circular linked list and taking care of page
2453 * dirty races).
2455 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2457 struct buffer_head *bh;
2459 BUG_ON(!PageLocked(page));
2461 spin_lock(&page->mapping->private_lock);
2462 bh = head;
2463 do {
2464 if (PageDirty(page))
2465 set_buffer_dirty(bh);
2466 if (!bh->b_this_page)
2467 bh->b_this_page = head;
2468 bh = bh->b_this_page;
2469 } while (bh != head);
2470 attach_page_buffers(page, head);
2471 spin_unlock(&page->mapping->private_lock);
2475 * On entry, the page is fully not uptodate.
2476 * On exit the page is fully uptodate in the areas outside (from,to)
2478 int nobh_write_begin(struct file *file, struct address_space *mapping,
2479 loff_t pos, unsigned len, unsigned flags,
2480 struct page **pagep, void **fsdata,
2481 get_block_t *get_block)
2483 struct inode *inode = mapping->host;
2484 const unsigned blkbits = inode->i_blkbits;
2485 const unsigned blocksize = 1 << blkbits;
2486 struct buffer_head *head, *bh;
2487 struct page *page;
2488 pgoff_t index;
2489 unsigned from, to;
2490 unsigned block_in_page;
2491 unsigned block_start, block_end;
2492 sector_t block_in_file;
2493 int nr_reads = 0;
2494 int ret = 0;
2495 int is_mapped_to_disk = 1;
2497 index = pos >> PAGE_CACHE_SHIFT;
2498 from = pos & (PAGE_CACHE_SIZE - 1);
2499 to = from + len;
2501 page = grab_cache_page_write_begin(mapping, index, flags);
2502 if (!page)
2503 return -ENOMEM;
2504 *pagep = page;
2505 *fsdata = NULL;
2507 if (page_has_buffers(page)) {
2508 unlock_page(page);
2509 page_cache_release(page);
2510 *pagep = NULL;
2511 return block_write_begin(file, mapping, pos, len, flags, pagep,
2512 fsdata, get_block);
2515 if (PageMappedToDisk(page))
2516 return 0;
2519 * Allocate buffers so that we can keep track of state, and potentially
2520 * attach them to the page if an error occurs. In the common case of
2521 * no error, they will just be freed again without ever being attached
2522 * to the page (which is all OK, because we're under the page lock).
2524 * Be careful: the buffer linked list is a NULL terminated one, rather
2525 * than the circular one we're used to.
2527 head = alloc_page_buffers(page, blocksize, 0);
2528 if (!head) {
2529 ret = -ENOMEM;
2530 goto out_release;
2533 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2536 * We loop across all blocks in the page, whether or not they are
2537 * part of the affected region. This is so we can discover if the
2538 * page is fully mapped-to-disk.
2540 for (block_start = 0, block_in_page = 0, bh = head;
2541 block_start < PAGE_CACHE_SIZE;
2542 block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2543 int create;
2545 block_end = block_start + blocksize;
2546 bh->b_state = 0;
2547 create = 1;
2548 if (block_start >= to)
2549 create = 0;
2550 ret = get_block(inode, block_in_file + block_in_page,
2551 bh, create);
2552 if (ret)
2553 goto failed;
2554 if (!buffer_mapped(bh))
2555 is_mapped_to_disk = 0;
2556 if (buffer_new(bh))
2557 unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
2558 if (PageUptodate(page)) {
2559 set_buffer_uptodate(bh);
2560 continue;
2562 if (buffer_new(bh) || !buffer_mapped(bh)) {
2563 zero_user_segments(page, block_start, from,
2564 to, block_end);
2565 continue;
2567 if (buffer_uptodate(bh))
2568 continue; /* reiserfs does this */
2569 if (block_start < from || block_end > to) {
2570 lock_buffer(bh);
2571 bh->b_end_io = end_buffer_read_nobh;
2572 submit_bh(READ, bh);
2573 nr_reads++;
2577 if (nr_reads) {
2579 * The page is locked, so these buffers are protected from
2580 * any VM or truncate activity. Hence we don't need to care
2581 * for the buffer_head refcounts.
2583 for (bh = head; bh; bh = bh->b_this_page) {
2584 wait_on_buffer(bh);
2585 if (!buffer_uptodate(bh))
2586 ret = -EIO;
2588 if (ret)
2589 goto failed;
2592 if (is_mapped_to_disk)
2593 SetPageMappedToDisk(page);
2595 *fsdata = head; /* to be released by nobh_write_end */
2597 return 0;
2599 failed:
2600 BUG_ON(!ret);
2602 * Error recovery is a bit difficult. We need to zero out blocks that
2603 * were newly allocated, and dirty them to ensure they get written out.
2604 * Buffers need to be attached to the page at this point, otherwise
2605 * the handling of potential IO errors during writeout would be hard
2606 * (could try doing synchronous writeout, but what if that fails too?)
2608 attach_nobh_buffers(page, head);
2609 page_zero_new_buffers(page, from, to);
2611 out_release:
2612 unlock_page(page);
2613 page_cache_release(page);
2614 *pagep = NULL;
2616 if (pos + len > inode->i_size)
2617 vmtruncate(inode, inode->i_size);
2619 return ret;
2621 EXPORT_SYMBOL(nobh_write_begin);
2623 int nobh_write_end(struct file *file, struct address_space *mapping,
2624 loff_t pos, unsigned len, unsigned copied,
2625 struct page *page, void *fsdata)
2627 struct inode *inode = page->mapping->host;
2628 struct buffer_head *head = fsdata;
2629 struct buffer_head *bh;
2630 BUG_ON(fsdata != NULL && page_has_buffers(page));
2632 if (unlikely(copied < len) && head)
2633 attach_nobh_buffers(page, head);
2634 if (page_has_buffers(page))
2635 return generic_write_end(file, mapping, pos, len,
2636 copied, page, fsdata);
2638 SetPageUptodate(page);
2639 set_page_dirty(page);
2640 if (pos+copied > inode->i_size) {
2641 i_size_write(inode, pos+copied);
2642 mark_inode_dirty(inode);
2645 unlock_page(page);
2646 page_cache_release(page);
2648 while (head) {
2649 bh = head;
2650 head = head->b_this_page;
2651 free_buffer_head(bh);
2654 return copied;
2656 EXPORT_SYMBOL(nobh_write_end);
2659 * nobh_writepage() - based on block_full_write_page() except
2660 * that it tries to operate without attaching bufferheads to
2661 * the page.
2663 int nobh_writepage(struct page *page, get_block_t *get_block,
2664 struct writeback_control *wbc)
2666 struct inode * const inode = page->mapping->host;
2667 loff_t i_size = i_size_read(inode);
2668 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2669 unsigned offset;
2670 int ret;
2672 /* Is the page fully inside i_size? */
2673 if (page->index < end_index)
2674 goto out;
2676 /* Is the page fully outside i_size? (truncate in progress) */
2677 offset = i_size & (PAGE_CACHE_SIZE-1);
2678 if (page->index >= end_index+1 || !offset) {
2680 * The page may have dirty, unmapped buffers. For example,
2681 * they may have been added in ext3_writepage(). Make them
2682 * freeable here, so the page does not leak.
2684 #if 0
2685 /* Not really sure about this - do we need this ? */
2686 if (page->mapping->a_ops->invalidatepage)
2687 page->mapping->a_ops->invalidatepage(page, offset);
2688 #endif
2689 unlock_page(page);
2690 return 0; /* don't care */
2694 * The page straddles i_size. It must be zeroed out on each and every
2695 * writepage invocation because it may be mmapped. "A file is mapped
2696 * in multiples of the page size. For a file that is not a multiple of
2697 * the page size, the remaining memory is zeroed when mapped, and
2698 * writes to that region are not written out to the file."
2700 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2701 out:
2702 ret = mpage_writepage(page, get_block, wbc);
2703 if (ret == -EAGAIN)
2704 ret = __block_write_full_page(inode, page, get_block, wbc,
2705 end_buffer_async_write);
2706 return ret;
2708 EXPORT_SYMBOL(nobh_writepage);
2710 int nobh_truncate_page(struct address_space *mapping,
2711 loff_t from, get_block_t *get_block)
2713 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2714 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2715 unsigned blocksize;
2716 sector_t iblock;
2717 unsigned length, pos;
2718 struct inode *inode = mapping->host;
2719 struct page *page;
2720 struct buffer_head map_bh;
2721 int err;
2723 blocksize = 1 << inode->i_blkbits;
2724 length = offset & (blocksize - 1);
2726 /* Block boundary? Nothing to do */
2727 if (!length)
2728 return 0;
2730 length = blocksize - length;
2731 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2733 page = grab_cache_page(mapping, index);
2734 err = -ENOMEM;
2735 if (!page)
2736 goto out;
2738 if (page_has_buffers(page)) {
2739 has_buffers:
2740 unlock_page(page);
2741 page_cache_release(page);
2742 return block_truncate_page(mapping, from, get_block);
2745 /* Find the buffer that contains "offset" */
2746 pos = blocksize;
2747 while (offset >= pos) {
2748 iblock++;
2749 pos += blocksize;
2752 map_bh.b_size = blocksize;
2753 map_bh.b_state = 0;
2754 err = get_block(inode, iblock, &map_bh, 0);
2755 if (err)
2756 goto unlock;
2757 /* unmapped? It's a hole - nothing to do */
2758 if (!buffer_mapped(&map_bh))
2759 goto unlock;
2761 /* Ok, it's mapped. Make sure it's up-to-date */
2762 if (!PageUptodate(page)) {
2763 err = mapping->a_ops->readpage(NULL, page);
2764 if (err) {
2765 page_cache_release(page);
2766 goto out;
2768 lock_page(page);
2769 if (!PageUptodate(page)) {
2770 err = -EIO;
2771 goto unlock;
2773 if (page_has_buffers(page))
2774 goto has_buffers;
2776 zero_user(page, offset, length);
2777 set_page_dirty(page);
2778 err = 0;
2780 unlock:
2781 unlock_page(page);
2782 page_cache_release(page);
2783 out:
2784 return err;
2786 EXPORT_SYMBOL(nobh_truncate_page);
2788 int block_truncate_page(struct address_space *mapping,
2789 loff_t from, get_block_t *get_block)
2791 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2792 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2793 unsigned blocksize;
2794 sector_t iblock;
2795 unsigned length, pos;
2796 struct inode *inode = mapping->host;
2797 struct page *page;
2798 struct buffer_head *bh;
2799 int err;
2801 blocksize = 1 << inode->i_blkbits;
2802 length = offset & (blocksize - 1);
2804 /* Block boundary? Nothing to do */
2805 if (!length)
2806 return 0;
2808 length = blocksize - length;
2809 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2811 page = grab_cache_page(mapping, index);
2812 err = -ENOMEM;
2813 if (!page)
2814 goto out;
2816 if (!page_has_buffers(page))
2817 create_empty_buffers(page, blocksize, 0);
2819 /* Find the buffer that contains "offset" */
2820 bh = page_buffers(page);
2821 pos = blocksize;
2822 while (offset >= pos) {
2823 bh = bh->b_this_page;
2824 iblock++;
2825 pos += blocksize;
2828 err = 0;
2829 if (!buffer_mapped(bh)) {
2830 WARN_ON(bh->b_size != blocksize);
2831 err = get_block(inode, iblock, bh, 0);
2832 if (err)
2833 goto unlock;
2834 /* unmapped? It's a hole - nothing to do */
2835 if (!buffer_mapped(bh))
2836 goto unlock;
2839 /* Ok, it's mapped. Make sure it's up-to-date */
2840 if (PageUptodate(page))
2841 set_buffer_uptodate(bh);
2843 if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2844 err = -EIO;
2845 ll_rw_block(READ, 1, &bh);
2846 wait_on_buffer(bh);
2847 /* Uhhuh. Read error. Complain and punt. */
2848 if (!buffer_uptodate(bh))
2849 goto unlock;
2852 zero_user(page, offset, length);
2853 mark_buffer_dirty(bh);
2854 err = 0;
2856 unlock:
2857 unlock_page(page);
2858 page_cache_release(page);
2859 out:
2860 return err;
2862 EXPORT_SYMBOL(block_truncate_page);
2865 * The generic ->writepage function for buffer-backed address_spaces
2866 * this form passes in the end_io handler used to finish the IO.
2868 int block_write_full_page_endio(struct page *page, get_block_t *get_block,
2869 struct writeback_control *wbc, bh_end_io_t *handler)
2871 struct inode * const inode = page->mapping->host;
2872 loff_t i_size = i_size_read(inode);
2873 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2874 unsigned offset;
2876 /* Is the page fully inside i_size? */
2877 if (page->index < end_index)
2878 return __block_write_full_page(inode, page, get_block, wbc,
2879 handler);
2881 /* Is the page fully outside i_size? (truncate in progress) */
2882 offset = i_size & (PAGE_CACHE_SIZE-1);
2883 if (page->index >= end_index+1 || !offset) {
2885 * The page may have dirty, unmapped buffers. For example,
2886 * they may have been added in ext3_writepage(). Make them
2887 * freeable here, so the page does not leak.
2889 do_invalidatepage(page, 0);
2890 unlock_page(page);
2891 return 0; /* don't care */
2895 * The page straddles i_size. It must be zeroed out on each and every
2896 * writepage invokation because it may be mmapped. "A file is mapped
2897 * in multiples of the page size. For a file that is not a multiple of
2898 * the page size, the remaining memory is zeroed when mapped, and
2899 * writes to that region are not written out to the file."
2901 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2902 return __block_write_full_page(inode, page, get_block, wbc, handler);
2904 EXPORT_SYMBOL(block_write_full_page_endio);
2907 * The generic ->writepage function for buffer-backed address_spaces
2909 int block_write_full_page(struct page *page, get_block_t *get_block,
2910 struct writeback_control *wbc)
2912 return block_write_full_page_endio(page, get_block, wbc,
2913 end_buffer_async_write);
2915 EXPORT_SYMBOL(block_write_full_page);
2917 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2918 get_block_t *get_block)
2920 struct buffer_head tmp;
2921 struct inode *inode = mapping->host;
2922 tmp.b_state = 0;
2923 tmp.b_blocknr = 0;
2924 tmp.b_size = 1 << inode->i_blkbits;
2925 get_block(inode, block, &tmp, 0);
2926 return tmp.b_blocknr;
2928 EXPORT_SYMBOL(generic_block_bmap);
2930 static void end_bio_bh_io_sync(struct bio *bio, int err)
2932 struct buffer_head *bh = bio->bi_private;
2934 if (err == -EOPNOTSUPP) {
2935 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2936 set_bit(BH_Eopnotsupp, &bh->b_state);
2939 if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
2940 set_bit(BH_Quiet, &bh->b_state);
2942 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2943 bio_put(bio);
2946 int submit_bh(int rw, struct buffer_head * bh)
2948 struct bio *bio;
2949 int ret = 0;
2951 BUG_ON(!buffer_locked(bh));
2952 BUG_ON(!buffer_mapped(bh));
2953 BUG_ON(!bh->b_end_io);
2954 BUG_ON(buffer_delay(bh));
2955 BUG_ON(buffer_unwritten(bh));
2958 * Mask in barrier bit for a write (could be either a WRITE or a
2959 * WRITE_SYNC
2961 if (buffer_ordered(bh) && (rw & WRITE))
2962 rw |= WRITE_BARRIER;
2965 * Only clear out a write error when rewriting
2967 if (test_set_buffer_req(bh) && (rw & WRITE))
2968 clear_buffer_write_io_error(bh);
2971 * from here on down, it's all bio -- do the initial mapping,
2972 * submit_bio -> generic_make_request may further map this bio around
2974 bio = bio_alloc(GFP_NOIO, 1);
2976 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2977 bio->bi_bdev = bh->b_bdev;
2978 bio->bi_io_vec[0].bv_page = bh->b_page;
2979 bio->bi_io_vec[0].bv_len = bh->b_size;
2980 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2982 bio->bi_vcnt = 1;
2983 bio->bi_idx = 0;
2984 bio->bi_size = bh->b_size;
2986 bio->bi_end_io = end_bio_bh_io_sync;
2987 bio->bi_private = bh;
2989 bio_get(bio);
2990 submit_bio(rw, bio);
2992 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2993 ret = -EOPNOTSUPP;
2995 bio_put(bio);
2996 return ret;
2998 EXPORT_SYMBOL(submit_bh);
3001 * ll_rw_block: low-level access to block devices (DEPRECATED)
3002 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
3003 * @nr: number of &struct buffer_heads in the array
3004 * @bhs: array of pointers to &struct buffer_head
3006 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3007 * requests an I/O operation on them, either a %READ or a %WRITE. The third
3008 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
3009 * are sent to disk. The fourth %READA option is described in the documentation
3010 * for generic_make_request() which ll_rw_block() calls.
3012 * This function drops any buffer that it cannot get a lock on (with the
3013 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
3014 * clean when doing a write request, and any buffer that appears to be
3015 * up-to-date when doing read request. Further it marks as clean buffers that
3016 * are processed for writing (the buffer cache won't assume that they are
3017 * actually clean until the buffer gets unlocked).
3019 * ll_rw_block sets b_end_io to simple completion handler that marks
3020 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
3021 * any waiters.
3023 * All of the buffers must be for the same device, and must also be a
3024 * multiple of the current approved size for the device.
3026 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
3028 int i;
3030 for (i = 0; i < nr; i++) {
3031 struct buffer_head *bh = bhs[i];
3033 if (rw == SWRITE || rw == SWRITE_SYNC || rw == SWRITE_SYNC_PLUG)
3034 lock_buffer(bh);
3035 else if (!trylock_buffer(bh))
3036 continue;
3038 if (rw == WRITE || rw == SWRITE || rw == SWRITE_SYNC ||
3039 rw == SWRITE_SYNC_PLUG) {
3040 if (test_clear_buffer_dirty(bh)) {
3041 bh->b_end_io = end_buffer_write_sync;
3042 get_bh(bh);
3043 if (rw == SWRITE_SYNC)
3044 submit_bh(WRITE_SYNC, bh);
3045 else
3046 submit_bh(WRITE, bh);
3047 continue;
3049 } else {
3050 if (!buffer_uptodate(bh)) {
3051 bh->b_end_io = end_buffer_read_sync;
3052 get_bh(bh);
3053 submit_bh(rw, bh);
3054 continue;
3057 unlock_buffer(bh);
3060 EXPORT_SYMBOL(ll_rw_block);
3063 * For a data-integrity writeout, we need to wait upon any in-progress I/O
3064 * and then start new I/O and then wait upon it. The caller must have a ref on
3065 * the buffer_head.
3067 int sync_dirty_buffer(struct buffer_head *bh)
3069 int ret = 0;
3071 WARN_ON(atomic_read(&bh->b_count) < 1);
3072 lock_buffer(bh);
3073 if (test_clear_buffer_dirty(bh)) {
3074 get_bh(bh);
3075 bh->b_end_io = end_buffer_write_sync;
3076 ret = submit_bh(WRITE_SYNC, bh);
3077 wait_on_buffer(bh);
3078 if (buffer_eopnotsupp(bh)) {
3079 clear_buffer_eopnotsupp(bh);
3080 ret = -EOPNOTSUPP;
3082 if (!ret && !buffer_uptodate(bh))
3083 ret = -EIO;
3084 } else {
3085 unlock_buffer(bh);
3087 return ret;
3089 EXPORT_SYMBOL(sync_dirty_buffer);
3092 * try_to_free_buffers() checks if all the buffers on this particular page
3093 * are unused, and releases them if so.
3095 * Exclusion against try_to_free_buffers may be obtained by either
3096 * locking the page or by holding its mapping's private_lock.
3098 * If the page is dirty but all the buffers are clean then we need to
3099 * be sure to mark the page clean as well. This is because the page
3100 * may be against a block device, and a later reattachment of buffers
3101 * to a dirty page will set *all* buffers dirty. Which would corrupt
3102 * filesystem data on the same device.
3104 * The same applies to regular filesystem pages: if all the buffers are
3105 * clean then we set the page clean and proceed. To do that, we require
3106 * total exclusion from __set_page_dirty_buffers(). That is obtained with
3107 * private_lock.
3109 * try_to_free_buffers() is non-blocking.
3111 static inline int buffer_busy(struct buffer_head *bh)
3113 return atomic_read(&bh->b_count) |
3114 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3117 static int
3118 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3120 struct buffer_head *head = page_buffers(page);
3121 struct buffer_head *bh;
3123 bh = head;
3124 do {
3125 if (buffer_write_io_error(bh) && page->mapping)
3126 set_bit(AS_EIO, &page->mapping->flags);
3127 if (buffer_busy(bh))
3128 goto failed;
3129 bh = bh->b_this_page;
3130 } while (bh != head);
3132 do {
3133 struct buffer_head *next = bh->b_this_page;
3135 if (bh->b_assoc_map)
3136 __remove_assoc_queue(bh);
3137 bh = next;
3138 } while (bh != head);
3139 *buffers_to_free = head;
3140 __clear_page_buffers(page);
3141 return 1;
3142 failed:
3143 return 0;
3146 int try_to_free_buffers(struct page *page)
3148 struct address_space * const mapping = page->mapping;
3149 struct buffer_head *buffers_to_free = NULL;
3150 int ret = 0;
3152 BUG_ON(!PageLocked(page));
3153 if (PageWriteback(page))
3154 return 0;
3156 if (mapping == NULL) { /* can this still happen? */
3157 ret = drop_buffers(page, &buffers_to_free);
3158 goto out;
3161 spin_lock(&mapping->private_lock);
3162 ret = drop_buffers(page, &buffers_to_free);
3165 * If the filesystem writes its buffers by hand (eg ext3)
3166 * then we can have clean buffers against a dirty page. We
3167 * clean the page here; otherwise the VM will never notice
3168 * that the filesystem did any IO at all.
3170 * Also, during truncate, discard_buffer will have marked all
3171 * the page's buffers clean. We discover that here and clean
3172 * the page also.
3174 * private_lock must be held over this entire operation in order
3175 * to synchronise against __set_page_dirty_buffers and prevent the
3176 * dirty bit from being lost.
3178 if (ret)
3179 cancel_dirty_page(page, PAGE_CACHE_SIZE);
3180 spin_unlock(&mapping->private_lock);
3181 out:
3182 if (buffers_to_free) {
3183 struct buffer_head *bh = buffers_to_free;
3185 do {
3186 struct buffer_head *next = bh->b_this_page;
3187 free_buffer_head(bh);
3188 bh = next;
3189 } while (bh != buffers_to_free);
3191 return ret;
3193 EXPORT_SYMBOL(try_to_free_buffers);
3195 void block_sync_page(struct page *page)
3197 struct address_space *mapping;
3199 smp_mb();
3200 mapping = page_mapping(page);
3201 if (mapping)
3202 blk_run_backing_dev(mapping->backing_dev_info, page);
3204 EXPORT_SYMBOL(block_sync_page);
3207 * There are no bdflush tunables left. But distributions are
3208 * still running obsolete flush daemons, so we terminate them here.
3210 * Use of bdflush() is deprecated and will be removed in a future kernel.
3211 * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3213 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3215 static int msg_count;
3217 if (!capable(CAP_SYS_ADMIN))
3218 return -EPERM;
3220 if (msg_count < 5) {
3221 msg_count++;
3222 printk(KERN_INFO
3223 "warning: process `%s' used the obsolete bdflush"
3224 " system call\n", current->comm);
3225 printk(KERN_INFO "Fix your initscripts?\n");
3228 if (func == 1)
3229 do_exit(0);
3230 return 0;
3234 * Buffer-head allocation
3236 static struct kmem_cache *bh_cachep;
3239 * Once the number of bh's in the machine exceeds this level, we start
3240 * stripping them in writeback.
3242 static int max_buffer_heads;
3244 int buffer_heads_over_limit;
3246 struct bh_accounting {
3247 int nr; /* Number of live bh's */
3248 int ratelimit; /* Limit cacheline bouncing */
3251 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3253 static void recalc_bh_state(void)
3255 int i;
3256 int tot = 0;
3258 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3259 return;
3260 __get_cpu_var(bh_accounting).ratelimit = 0;
3261 for_each_online_cpu(i)
3262 tot += per_cpu(bh_accounting, i).nr;
3263 buffer_heads_over_limit = (tot > max_buffer_heads);
3266 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3268 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3269 if (ret) {
3270 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3271 get_cpu_var(bh_accounting).nr++;
3272 recalc_bh_state();
3273 put_cpu_var(bh_accounting);
3275 return ret;
3277 EXPORT_SYMBOL(alloc_buffer_head);
3279 void free_buffer_head(struct buffer_head *bh)
3281 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3282 kmem_cache_free(bh_cachep, bh);
3283 get_cpu_var(bh_accounting).nr--;
3284 recalc_bh_state();
3285 put_cpu_var(bh_accounting);
3287 EXPORT_SYMBOL(free_buffer_head);
3289 static void buffer_exit_cpu(int cpu)
3291 int i;
3292 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3294 for (i = 0; i < BH_LRU_SIZE; i++) {
3295 brelse(b->bhs[i]);
3296 b->bhs[i] = NULL;
3298 get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
3299 per_cpu(bh_accounting, cpu).nr = 0;
3300 put_cpu_var(bh_accounting);
3303 static int buffer_cpu_notify(struct notifier_block *self,
3304 unsigned long action, void *hcpu)
3306 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
3307 buffer_exit_cpu((unsigned long)hcpu);
3308 return NOTIFY_OK;
3312 * bh_uptodate_or_lock - Test whether the buffer is uptodate
3313 * @bh: struct buffer_head
3315 * Return true if the buffer is up-to-date and false,
3316 * with the buffer locked, if not.
3318 int bh_uptodate_or_lock(struct buffer_head *bh)
3320 if (!buffer_uptodate(bh)) {
3321 lock_buffer(bh);
3322 if (!buffer_uptodate(bh))
3323 return 0;
3324 unlock_buffer(bh);
3326 return 1;
3328 EXPORT_SYMBOL(bh_uptodate_or_lock);
3331 * bh_submit_read - Submit a locked buffer for reading
3332 * @bh: struct buffer_head
3334 * Returns zero on success and -EIO on error.
3336 int bh_submit_read(struct buffer_head *bh)
3338 BUG_ON(!buffer_locked(bh));
3340 if (buffer_uptodate(bh)) {
3341 unlock_buffer(bh);
3342 return 0;
3345 get_bh(bh);
3346 bh->b_end_io = end_buffer_read_sync;
3347 submit_bh(READ, bh);
3348 wait_on_buffer(bh);
3349 if (buffer_uptodate(bh))
3350 return 0;
3351 return -EIO;
3353 EXPORT_SYMBOL(bh_submit_read);
3355 static void
3356 init_buffer_head(void *data)
3358 struct buffer_head *bh = data;
3360 memset(bh, 0, sizeof(*bh));
3361 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3364 void __init buffer_init(void)
3366 int nrpages;
3368 bh_cachep = kmem_cache_create("buffer_head",
3369 sizeof(struct buffer_head), 0,
3370 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3371 SLAB_MEM_SPREAD),
3372 init_buffer_head);
3375 * Limit the bh occupancy to 10% of ZONE_NORMAL
3377 nrpages = (nr_free_buffer_pages() * 10) / 100;
3378 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3379 hotcpu_notifier(buffer_cpu_notify, 0);