4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
21 #include <linux/config.h>
22 #include <linux/kernel.h>
23 #include <linux/syscalls.h>
26 #include <linux/percpu.h>
27 #include <linux/slab.h>
28 #include <linux/smp_lock.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/module.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
44 static int fsync_buffers_list(spinlock_t
*lock
, struct list_head
*list
);
45 static void invalidate_bh_lrus(void);
47 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
50 init_buffer(struct buffer_head
*bh
, bh_end_io_t
*handler
, void *private)
52 bh
->b_end_io
= handler
;
53 bh
->b_private
= private;
56 static int sync_buffer(void *word
)
58 struct block_device
*bd
;
59 struct buffer_head
*bh
60 = container_of(word
, struct buffer_head
, b_state
);
65 blk_run_address_space(bd
->bd_inode
->i_mapping
);
70 void fastcall
__lock_buffer(struct buffer_head
*bh
)
72 wait_on_bit_lock(&bh
->b_state
, BH_Lock
, sync_buffer
,
73 TASK_UNINTERRUPTIBLE
);
75 EXPORT_SYMBOL(__lock_buffer
);
77 void fastcall
unlock_buffer(struct buffer_head
*bh
)
79 clear_buffer_locked(bh
);
80 smp_mb__after_clear_bit();
81 wake_up_bit(&bh
->b_state
, BH_Lock
);
85 * Block until a buffer comes unlocked. This doesn't stop it
86 * from becoming locked again - you have to lock it yourself
87 * if you want to preserve its state.
89 void __wait_on_buffer(struct buffer_head
* bh
)
91 wait_on_bit(&bh
->b_state
, BH_Lock
, sync_buffer
, TASK_UNINTERRUPTIBLE
);
95 __clear_page_buffers(struct page
*page
)
97 ClearPagePrivate(page
);
99 page_cache_release(page
);
102 static void buffer_io_error(struct buffer_head
*bh
)
104 char b
[BDEVNAME_SIZE
];
106 printk(KERN_ERR
"Buffer I/O error on device %s, logical block %Lu\n",
107 bdevname(bh
->b_bdev
, b
),
108 (unsigned long long)bh
->b_blocknr
);
112 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
113 * unlock the buffer. This is what ll_rw_block uses too.
115 void end_buffer_read_sync(struct buffer_head
*bh
, int uptodate
)
118 set_buffer_uptodate(bh
);
120 /* This happens, due to failed READA attempts. */
121 clear_buffer_uptodate(bh
);
127 void end_buffer_write_sync(struct buffer_head
*bh
, int uptodate
)
129 char b
[BDEVNAME_SIZE
];
132 set_buffer_uptodate(bh
);
134 if (!buffer_eopnotsupp(bh
) && printk_ratelimit()) {
136 printk(KERN_WARNING
"lost page write due to "
138 bdevname(bh
->b_bdev
, b
));
140 set_buffer_write_io_error(bh
);
141 clear_buffer_uptodate(bh
);
148 * Write out and wait upon all the dirty data associated with a block
149 * device via its mapping. Does not take the superblock lock.
151 int sync_blockdev(struct block_device
*bdev
)
158 ret
= filemap_fdatawrite(bdev
->bd_inode
->i_mapping
);
159 err
= filemap_fdatawait(bdev
->bd_inode
->i_mapping
);
165 EXPORT_SYMBOL(sync_blockdev
);
168 * Write out and wait upon all dirty data associated with this
169 * superblock. Filesystem data as well as the underlying block
170 * device. Takes the superblock lock.
172 int fsync_super(struct super_block
*sb
)
174 sync_inodes_sb(sb
, 0);
177 if (sb
->s_dirt
&& sb
->s_op
->write_super
)
178 sb
->s_op
->write_super(sb
);
180 if (sb
->s_op
->sync_fs
)
181 sb
->s_op
->sync_fs(sb
, 1);
182 sync_blockdev(sb
->s_bdev
);
183 sync_inodes_sb(sb
, 1);
185 return sync_blockdev(sb
->s_bdev
);
189 * Write out and wait upon all dirty data associated with this
190 * device. Filesystem data as well as the underlying block
191 * device. Takes the superblock lock.
193 int fsync_bdev(struct block_device
*bdev
)
195 struct super_block
*sb
= get_super(bdev
);
197 int res
= fsync_super(sb
);
201 return sync_blockdev(bdev
);
205 * freeze_bdev -- lock a filesystem and force it into a consistent state
206 * @bdev: blockdevice to lock
208 * This takes the block device bd_mount_sem to make sure no new mounts
209 * happen on bdev until thaw_bdev() is called.
210 * If a superblock is found on this device, we take the s_umount semaphore
211 * on it to make sure nobody unmounts until the snapshot creation is done.
213 struct super_block
*freeze_bdev(struct block_device
*bdev
)
215 struct super_block
*sb
;
217 down(&bdev
->bd_mount_sem
);
218 sb
= get_super(bdev
);
219 if (sb
&& !(sb
->s_flags
& MS_RDONLY
)) {
220 sb
->s_frozen
= SB_FREEZE_WRITE
;
223 sync_inodes_sb(sb
, 0);
227 if (sb
->s_dirt
&& sb
->s_op
->write_super
)
228 sb
->s_op
->write_super(sb
);
231 if (sb
->s_op
->sync_fs
)
232 sb
->s_op
->sync_fs(sb
, 1);
234 sync_blockdev(sb
->s_bdev
);
235 sync_inodes_sb(sb
, 1);
237 sb
->s_frozen
= SB_FREEZE_TRANS
;
240 sync_blockdev(sb
->s_bdev
);
242 if (sb
->s_op
->write_super_lockfs
)
243 sb
->s_op
->write_super_lockfs(sb
);
247 return sb
; /* thaw_bdev releases s->s_umount and bd_mount_sem */
249 EXPORT_SYMBOL(freeze_bdev
);
252 * thaw_bdev -- unlock filesystem
253 * @bdev: blockdevice to unlock
254 * @sb: associated superblock
256 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
258 void thaw_bdev(struct block_device
*bdev
, struct super_block
*sb
)
261 BUG_ON(sb
->s_bdev
!= bdev
);
263 if (sb
->s_op
->unlockfs
)
264 sb
->s_op
->unlockfs(sb
);
265 sb
->s_frozen
= SB_UNFROZEN
;
267 wake_up(&sb
->s_wait_unfrozen
);
271 up(&bdev
->bd_mount_sem
);
273 EXPORT_SYMBOL(thaw_bdev
);
276 * sync everything. Start out by waking pdflush, because that writes back
277 * all queues in parallel.
279 static void do_sync(unsigned long wait
)
282 sync_inodes(0); /* All mappings, inodes and their blockdevs */
284 sync_supers(); /* Write the superblocks */
285 sync_filesystems(0); /* Start syncing the filesystems */
286 sync_filesystems(wait
); /* Waitingly sync the filesystems */
287 sync_inodes(wait
); /* Mappings, inodes and blockdevs, again. */
289 printk("Emergency Sync complete\n");
290 if (unlikely(laptop_mode
))
291 laptop_sync_completion();
294 asmlinkage
long sys_sync(void)
300 void emergency_sync(void)
302 pdflush_operation(do_sync
, 0);
306 * Generic function to fsync a file.
308 * filp may be NULL if called via the msync of a vma.
311 int file_fsync(struct file
*filp
, struct dentry
*dentry
, int datasync
)
313 struct inode
* inode
= dentry
->d_inode
;
314 struct super_block
* sb
;
317 /* sync the inode to buffers */
318 ret
= write_inode_now(inode
, 0);
320 /* sync the superblock to buffers */
323 if (sb
->s_op
->write_super
)
324 sb
->s_op
->write_super(sb
);
327 /* .. finally sync the buffers to disk */
328 err
= sync_blockdev(sb
->s_bdev
);
334 asmlinkage
long sys_fsync(unsigned int fd
)
337 struct address_space
*mapping
;
345 mapping
= file
->f_mapping
;
348 if (!file
->f_op
|| !file
->f_op
->fsync
) {
349 /* Why? We can still call filemap_fdatawrite */
353 current
->flags
|= PF_SYNCWRITE
;
354 ret
= filemap_fdatawrite(mapping
);
357 * We need to protect against concurrent writers,
358 * which could cause livelocks in fsync_buffers_list
360 down(&mapping
->host
->i_sem
);
361 err
= file
->f_op
->fsync(file
, file
->f_dentry
, 0);
364 up(&mapping
->host
->i_sem
);
365 err
= filemap_fdatawait(mapping
);
368 current
->flags
&= ~PF_SYNCWRITE
;
376 asmlinkage
long sys_fdatasync(unsigned int fd
)
379 struct address_space
*mapping
;
388 if (!file
->f_op
|| !file
->f_op
->fsync
)
391 mapping
= file
->f_mapping
;
393 current
->flags
|= PF_SYNCWRITE
;
394 ret
= filemap_fdatawrite(mapping
);
395 down(&mapping
->host
->i_sem
);
396 err
= file
->f_op
->fsync(file
, file
->f_dentry
, 1);
399 up(&mapping
->host
->i_sem
);
400 err
= filemap_fdatawait(mapping
);
403 current
->flags
&= ~PF_SYNCWRITE
;
412 * Various filesystems appear to want __find_get_block to be non-blocking.
413 * But it's the page lock which protects the buffers. To get around this,
414 * we get exclusion from try_to_free_buffers with the blockdev mapping's
417 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
418 * may be quite high. This code could TryLock the page, and if that
419 * succeeds, there is no need to take private_lock. (But if
420 * private_lock is contended then so is mapping->tree_lock).
422 static struct buffer_head
*
423 __find_get_block_slow(struct block_device
*bdev
, sector_t block
, int unused
)
425 struct inode
*bd_inode
= bdev
->bd_inode
;
426 struct address_space
*bd_mapping
= bd_inode
->i_mapping
;
427 struct buffer_head
*ret
= NULL
;
429 struct buffer_head
*bh
;
430 struct buffer_head
*head
;
434 index
= block
>> (PAGE_CACHE_SHIFT
- bd_inode
->i_blkbits
);
435 page
= find_get_page(bd_mapping
, index
);
439 spin_lock(&bd_mapping
->private_lock
);
440 if (!page_has_buffers(page
))
442 head
= page_buffers(page
);
445 if (bh
->b_blocknr
== block
) {
450 if (!buffer_mapped(bh
))
452 bh
= bh
->b_this_page
;
453 } while (bh
!= head
);
455 /* we might be here because some of the buffers on this page are
456 * not mapped. This is due to various races between
457 * file io on the block device and getblk. It gets dealt with
458 * elsewhere, don't buffer_error if we had some unmapped buffers
461 printk("__find_get_block_slow() failed. "
462 "block=%llu, b_blocknr=%llu\n",
463 (unsigned long long)block
, (unsigned long long)bh
->b_blocknr
);
464 printk("b_state=0x%08lx, b_size=%u\n", bh
->b_state
, bh
->b_size
);
465 printk("device blocksize: %d\n", 1 << bd_inode
->i_blkbits
);
468 spin_unlock(&bd_mapping
->private_lock
);
469 page_cache_release(page
);
474 /* If invalidate_buffers() will trash dirty buffers, it means some kind
475 of fs corruption is going on. Trashing dirty data always imply losing
476 information that was supposed to be just stored on the physical layer
479 Thus invalidate_buffers in general usage is not allwowed to trash
480 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
481 be preserved. These buffers are simply skipped.
483 We also skip buffers which are still in use. For example this can
484 happen if a userspace program is reading the block device.
486 NOTE: In the case where the user removed a removable-media-disk even if
487 there's still dirty data not synced on disk (due a bug in the device driver
488 or due an error of the user), by not destroying the dirty buffers we could
489 generate corruption also on the next media inserted, thus a parameter is
490 necessary to handle this case in the most safe way possible (trying
491 to not corrupt also the new disk inserted with the data belonging to
492 the old now corrupted disk). Also for the ramdisk the natural thing
493 to do in order to release the ramdisk memory is to destroy dirty buffers.
495 These are two special cases. Normal usage imply the device driver
496 to issue a sync on the device (without waiting I/O completion) and
497 then an invalidate_buffers call that doesn't trash dirty buffers.
499 For handling cache coherency with the blkdev pagecache the 'update' case
500 is been introduced. It is needed to re-read from disk any pinned
501 buffer. NOTE: re-reading from disk is destructive so we can do it only
502 when we assume nobody is changing the buffercache under our I/O and when
503 we think the disk contains more recent information than the buffercache.
504 The update == 1 pass marks the buffers we need to update, the update == 2
505 pass does the actual I/O. */
506 void invalidate_bdev(struct block_device
*bdev
, int destroy_dirty_buffers
)
508 invalidate_bh_lrus();
510 * FIXME: what about destroy_dirty_buffers?
511 * We really want to use invalidate_inode_pages2() for
512 * that, but not until that's cleaned up.
514 invalidate_inode_pages(bdev
->bd_inode
->i_mapping
);
518 * Kick pdflush then try to free up some ZONE_NORMAL memory.
520 static void free_more_memory(void)
525 wakeup_bdflush(1024);
528 for_each_pgdat(pgdat
) {
529 zones
= pgdat
->node_zonelists
[GFP_NOFS
&GFP_ZONEMASK
].zones
;
531 try_to_free_pages(zones
, GFP_NOFS
, 0);
536 * I/O completion handler for block_read_full_page() - pages
537 * which come unlocked at the end of I/O.
539 static void end_buffer_async_read(struct buffer_head
*bh
, int uptodate
)
541 static DEFINE_SPINLOCK(page_uptodate_lock
);
543 struct buffer_head
*tmp
;
545 int page_uptodate
= 1;
547 BUG_ON(!buffer_async_read(bh
));
551 set_buffer_uptodate(bh
);
553 clear_buffer_uptodate(bh
);
554 if (printk_ratelimit())
560 * Be _very_ careful from here on. Bad things can happen if
561 * two buffer heads end IO at almost the same time and both
562 * decide that the page is now completely done.
564 spin_lock_irqsave(&page_uptodate_lock
, flags
);
565 clear_buffer_async_read(bh
);
569 if (!buffer_uptodate(tmp
))
571 if (buffer_async_read(tmp
)) {
572 BUG_ON(!buffer_locked(tmp
));
575 tmp
= tmp
->b_this_page
;
577 spin_unlock_irqrestore(&page_uptodate_lock
, flags
);
580 * If none of the buffers had errors and they are all
581 * uptodate then we can set the page uptodate.
583 if (page_uptodate
&& !PageError(page
))
584 SetPageUptodate(page
);
589 spin_unlock_irqrestore(&page_uptodate_lock
, flags
);
594 * Completion handler for block_write_full_page() - pages which are unlocked
595 * during I/O, and which have PageWriteback cleared upon I/O completion.
597 void end_buffer_async_write(struct buffer_head
*bh
, int uptodate
)
599 char b
[BDEVNAME_SIZE
];
600 static DEFINE_SPINLOCK(page_uptodate_lock
);
602 struct buffer_head
*tmp
;
605 BUG_ON(!buffer_async_write(bh
));
609 set_buffer_uptodate(bh
);
611 if (printk_ratelimit()) {
613 printk(KERN_WARNING
"lost page write due to "
615 bdevname(bh
->b_bdev
, b
));
617 set_bit(AS_EIO
, &page
->mapping
->flags
);
618 clear_buffer_uptodate(bh
);
622 spin_lock_irqsave(&page_uptodate_lock
, flags
);
623 clear_buffer_async_write(bh
);
625 tmp
= bh
->b_this_page
;
627 if (buffer_async_write(tmp
)) {
628 BUG_ON(!buffer_locked(tmp
));
631 tmp
= tmp
->b_this_page
;
633 spin_unlock_irqrestore(&page_uptodate_lock
, flags
);
634 end_page_writeback(page
);
638 spin_unlock_irqrestore(&page_uptodate_lock
, flags
);
643 * If a page's buffers are under async readin (end_buffer_async_read
644 * completion) then there is a possibility that another thread of
645 * control could lock one of the buffers after it has completed
646 * but while some of the other buffers have not completed. This
647 * locked buffer would confuse end_buffer_async_read() into not unlocking
648 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
649 * that this buffer is not under async I/O.
651 * The page comes unlocked when it has no locked buffer_async buffers
654 * PageLocked prevents anyone starting new async I/O reads any of
657 * PageWriteback is used to prevent simultaneous writeout of the same
660 * PageLocked prevents anyone from starting writeback of a page which is
661 * under read I/O (PageWriteback is only ever set against a locked page).
663 static void mark_buffer_async_read(struct buffer_head
*bh
)
665 bh
->b_end_io
= end_buffer_async_read
;
666 set_buffer_async_read(bh
);
669 void mark_buffer_async_write(struct buffer_head
*bh
)
671 bh
->b_end_io
= end_buffer_async_write
;
672 set_buffer_async_write(bh
);
674 EXPORT_SYMBOL(mark_buffer_async_write
);
678 * fs/buffer.c contains helper functions for buffer-backed address space's
679 * fsync functions. A common requirement for buffer-based filesystems is
680 * that certain data from the backing blockdev needs to be written out for
681 * a successful fsync(). For example, ext2 indirect blocks need to be
682 * written back and waited upon before fsync() returns.
684 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
685 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
686 * management of a list of dependent buffers at ->i_mapping->private_list.
688 * Locking is a little subtle: try_to_free_buffers() will remove buffers
689 * from their controlling inode's queue when they are being freed. But
690 * try_to_free_buffers() will be operating against the *blockdev* mapping
691 * at the time, not against the S_ISREG file which depends on those buffers.
692 * So the locking for private_list is via the private_lock in the address_space
693 * which backs the buffers. Which is different from the address_space
694 * against which the buffers are listed. So for a particular address_space,
695 * mapping->private_lock does *not* protect mapping->private_list! In fact,
696 * mapping->private_list will always be protected by the backing blockdev's
699 * Which introduces a requirement: all buffers on an address_space's
700 * ->private_list must be from the same address_space: the blockdev's.
702 * address_spaces which do not place buffers at ->private_list via these
703 * utility functions are free to use private_lock and private_list for
704 * whatever they want. The only requirement is that list_empty(private_list)
705 * be true at clear_inode() time.
707 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
708 * filesystems should do that. invalidate_inode_buffers() should just go
709 * BUG_ON(!list_empty).
711 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
712 * take an address_space, not an inode. And it should be called
713 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
716 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
717 * list if it is already on a list. Because if the buffer is on a list,
718 * it *must* already be on the right one. If not, the filesystem is being
719 * silly. This will save a ton of locking. But first we have to ensure
720 * that buffers are taken *off* the old inode's list when they are freed
721 * (presumably in truncate). That requires careful auditing of all
722 * filesystems (do it inside bforget()). It could also be done by bringing
727 * The buffer's backing address_space's private_lock must be held
729 static inline void __remove_assoc_queue(struct buffer_head
*bh
)
731 list_del_init(&bh
->b_assoc_buffers
);
734 int inode_has_buffers(struct inode
*inode
)
736 return !list_empty(&inode
->i_data
.private_list
);
740 * osync is designed to support O_SYNC io. It waits synchronously for
741 * all already-submitted IO to complete, but does not queue any new
742 * writes to the disk.
744 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
745 * you dirty the buffers, and then use osync_inode_buffers to wait for
746 * completion. Any other dirty buffers which are not yet queued for
747 * write will not be flushed to disk by the osync.
749 static int osync_buffers_list(spinlock_t
*lock
, struct list_head
*list
)
751 struct buffer_head
*bh
;
757 list_for_each_prev(p
, list
) {
759 if (buffer_locked(bh
)) {
763 if (!buffer_uptodate(bh
))
775 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
777 * @buffer_mapping - the mapping which backs the buffers' data
778 * @mapping - the mapping which wants those buffers written
780 * Starts I/O against the buffers at mapping->private_list, and waits upon
783 * Basically, this is a convenience function for fsync(). @buffer_mapping is
784 * the blockdev which "owns" the buffers and @mapping is a file or directory
785 * which needs those buffers to be written for a successful fsync().
787 int sync_mapping_buffers(struct address_space
*mapping
)
789 struct address_space
*buffer_mapping
= mapping
->assoc_mapping
;
791 if (buffer_mapping
== NULL
|| list_empty(&mapping
->private_list
))
794 return fsync_buffers_list(&buffer_mapping
->private_lock
,
795 &mapping
->private_list
);
797 EXPORT_SYMBOL(sync_mapping_buffers
);
800 * Called when we've recently written block `bblock', and it is known that
801 * `bblock' was for a buffer_boundary() buffer. This means that the block at
802 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
803 * dirty, schedule it for IO. So that indirects merge nicely with their data.
805 void write_boundary_block(struct block_device
*bdev
,
806 sector_t bblock
, unsigned blocksize
)
808 struct buffer_head
*bh
= __find_get_block(bdev
, bblock
+ 1, blocksize
);
810 if (buffer_dirty(bh
))
811 ll_rw_block(WRITE
, 1, &bh
);
816 void mark_buffer_dirty_inode(struct buffer_head
*bh
, struct inode
*inode
)
818 struct address_space
*mapping
= inode
->i_mapping
;
819 struct address_space
*buffer_mapping
= bh
->b_page
->mapping
;
821 mark_buffer_dirty(bh
);
822 if (!mapping
->assoc_mapping
) {
823 mapping
->assoc_mapping
= buffer_mapping
;
825 if (mapping
->assoc_mapping
!= buffer_mapping
)
828 if (list_empty(&bh
->b_assoc_buffers
)) {
829 spin_lock(&buffer_mapping
->private_lock
);
830 list_move_tail(&bh
->b_assoc_buffers
,
831 &mapping
->private_list
);
832 spin_unlock(&buffer_mapping
->private_lock
);
835 EXPORT_SYMBOL(mark_buffer_dirty_inode
);
838 * Add a page to the dirty page list.
840 * It is a sad fact of life that this function is called from several places
841 * deeply under spinlocking. It may not sleep.
843 * If the page has buffers, the uptodate buffers are set dirty, to preserve
844 * dirty-state coherency between the page and the buffers. It the page does
845 * not have buffers then when they are later attached they will all be set
848 * The buffers are dirtied before the page is dirtied. There's a small race
849 * window in which a writepage caller may see the page cleanness but not the
850 * buffer dirtiness. That's fine. If this code were to set the page dirty
851 * before the buffers, a concurrent writepage caller could clear the page dirty
852 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
853 * page on the dirty page list.
855 * We use private_lock to lock against try_to_free_buffers while using the
856 * page's buffer list. Also use this to protect against clean buffers being
857 * added to the page after it was set dirty.
859 * FIXME: may need to call ->reservepage here as well. That's rather up to the
860 * address_space though.
862 int __set_page_dirty_buffers(struct page
*page
)
864 struct address_space
* const mapping
= page
->mapping
;
866 spin_lock(&mapping
->private_lock
);
867 if (page_has_buffers(page
)) {
868 struct buffer_head
*head
= page_buffers(page
);
869 struct buffer_head
*bh
= head
;
872 set_buffer_dirty(bh
);
873 bh
= bh
->b_this_page
;
874 } while (bh
!= head
);
876 spin_unlock(&mapping
->private_lock
);
878 if (!TestSetPageDirty(page
)) {
879 write_lock_irq(&mapping
->tree_lock
);
880 if (page
->mapping
) { /* Race with truncate? */
881 if (mapping_cap_account_dirty(mapping
))
882 inc_page_state(nr_dirty
);
883 radix_tree_tag_set(&mapping
->page_tree
,
885 PAGECACHE_TAG_DIRTY
);
887 write_unlock_irq(&mapping
->tree_lock
);
888 __mark_inode_dirty(mapping
->host
, I_DIRTY_PAGES
);
893 EXPORT_SYMBOL(__set_page_dirty_buffers
);
896 * Write out and wait upon a list of buffers.
898 * We have conflicting pressures: we want to make sure that all
899 * initially dirty buffers get waited on, but that any subsequently
900 * dirtied buffers don't. After all, we don't want fsync to last
901 * forever if somebody is actively writing to the file.
903 * Do this in two main stages: first we copy dirty buffers to a
904 * temporary inode list, queueing the writes as we go. Then we clean
905 * up, waiting for those writes to complete.
907 * During this second stage, any subsequent updates to the file may end
908 * up refiling the buffer on the original inode's dirty list again, so
909 * there is a chance we will end up with a buffer queued for write but
910 * not yet completed on that list. So, as a final cleanup we go through
911 * the osync code to catch these locked, dirty buffers without requeuing
912 * any newly dirty buffers for write.
914 static int fsync_buffers_list(spinlock_t
*lock
, struct list_head
*list
)
916 struct buffer_head
*bh
;
917 struct list_head tmp
;
920 INIT_LIST_HEAD(&tmp
);
923 while (!list_empty(list
)) {
924 bh
= BH_ENTRY(list
->next
);
925 list_del_init(&bh
->b_assoc_buffers
);
926 if (buffer_dirty(bh
) || buffer_locked(bh
)) {
927 list_add(&bh
->b_assoc_buffers
, &tmp
);
928 if (buffer_dirty(bh
)) {
932 * Ensure any pending I/O completes so that
933 * ll_rw_block() actually writes the current
934 * contents - it is a noop if I/O is still in
935 * flight on potentially older contents.
938 ll_rw_block(WRITE
, 1, &bh
);
945 while (!list_empty(&tmp
)) {
946 bh
= BH_ENTRY(tmp
.prev
);
947 __remove_assoc_queue(bh
);
951 if (!buffer_uptodate(bh
))
958 err2
= osync_buffers_list(lock
, list
);
966 * Invalidate any and all dirty buffers on a given inode. We are
967 * probably unmounting the fs, but that doesn't mean we have already
968 * done a sync(). Just drop the buffers from the inode list.
970 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
971 * assumes that all the buffers are against the blockdev. Not true
974 void invalidate_inode_buffers(struct inode
*inode
)
976 if (inode_has_buffers(inode
)) {
977 struct address_space
*mapping
= &inode
->i_data
;
978 struct list_head
*list
= &mapping
->private_list
;
979 struct address_space
*buffer_mapping
= mapping
->assoc_mapping
;
981 spin_lock(&buffer_mapping
->private_lock
);
982 while (!list_empty(list
))
983 __remove_assoc_queue(BH_ENTRY(list
->next
));
984 spin_unlock(&buffer_mapping
->private_lock
);
989 * Remove any clean buffers from the inode's buffer list. This is called
990 * when we're trying to free the inode itself. Those buffers can pin it.
992 * Returns true if all buffers were removed.
994 int remove_inode_buffers(struct inode
*inode
)
998 if (inode_has_buffers(inode
)) {
999 struct address_space
*mapping
= &inode
->i_data
;
1000 struct list_head
*list
= &mapping
->private_list
;
1001 struct address_space
*buffer_mapping
= mapping
->assoc_mapping
;
1003 spin_lock(&buffer_mapping
->private_lock
);
1004 while (!list_empty(list
)) {
1005 struct buffer_head
*bh
= BH_ENTRY(list
->next
);
1006 if (buffer_dirty(bh
)) {
1010 __remove_assoc_queue(bh
);
1012 spin_unlock(&buffer_mapping
->private_lock
);
1018 * Create the appropriate buffers when given a page for data area and
1019 * the size of each buffer.. Use the bh->b_this_page linked list to
1020 * follow the buffers created. Return NULL if unable to create more
1023 * The retry flag is used to differentiate async IO (paging, swapping)
1024 * which may not fail from ordinary buffer allocations.
1026 struct buffer_head
*alloc_page_buffers(struct page
*page
, unsigned long size
,
1029 struct buffer_head
*bh
, *head
;
1035 while ((offset
-= size
) >= 0) {
1036 bh
= alloc_buffer_head(GFP_NOFS
);
1041 bh
->b_this_page
= head
;
1046 atomic_set(&bh
->b_count
, 0);
1049 /* Link the buffer to its page */
1050 set_bh_page(bh
, page
, offset
);
1052 bh
->b_end_io
= NULL
;
1056 * In case anything failed, we just free everything we got.
1062 head
= head
->b_this_page
;
1063 free_buffer_head(bh
);
1068 * Return failure for non-async IO requests. Async IO requests
1069 * are not allowed to fail, so we have to wait until buffer heads
1070 * become available. But we don't want tasks sleeping with
1071 * partially complete buffers, so all were released above.
1076 /* We're _really_ low on memory. Now we just
1077 * wait for old buffer heads to become free due to
1078 * finishing IO. Since this is an async request and
1079 * the reserve list is empty, we're sure there are
1080 * async buffer heads in use.
1085 EXPORT_SYMBOL_GPL(alloc_page_buffers
);
1088 link_dev_buffers(struct page
*page
, struct buffer_head
*head
)
1090 struct buffer_head
*bh
, *tail
;
1095 bh
= bh
->b_this_page
;
1097 tail
->b_this_page
= head
;
1098 attach_page_buffers(page
, head
);
1102 * Initialise the state of a blockdev page's buffers.
1105 init_page_buffers(struct page
*page
, struct block_device
*bdev
,
1106 sector_t block
, int size
)
1108 struct buffer_head
*head
= page_buffers(page
);
1109 struct buffer_head
*bh
= head
;
1110 int uptodate
= PageUptodate(page
);
1113 if (!buffer_mapped(bh
)) {
1114 init_buffer(bh
, NULL
, NULL
);
1116 bh
->b_blocknr
= block
;
1118 set_buffer_uptodate(bh
);
1119 set_buffer_mapped(bh
);
1122 bh
= bh
->b_this_page
;
1123 } while (bh
!= head
);
1127 * Create the page-cache page that contains the requested block.
1129 * This is user purely for blockdev mappings.
1131 static struct page
*
1132 grow_dev_page(struct block_device
*bdev
, sector_t block
,
1133 pgoff_t index
, int size
)
1135 struct inode
*inode
= bdev
->bd_inode
;
1137 struct buffer_head
*bh
;
1139 page
= find_or_create_page(inode
->i_mapping
, index
, GFP_NOFS
);
1143 if (!PageLocked(page
))
1146 if (page_has_buffers(page
)) {
1147 bh
= page_buffers(page
);
1148 if (bh
->b_size
== size
) {
1149 init_page_buffers(page
, bdev
, block
, size
);
1152 if (!try_to_free_buffers(page
))
1157 * Allocate some buffers for this page
1159 bh
= alloc_page_buffers(page
, size
, 0);
1164 * Link the page to the buffers and initialise them. Take the
1165 * lock to be atomic wrt __find_get_block(), which does not
1166 * run under the page lock.
1168 spin_lock(&inode
->i_mapping
->private_lock
);
1169 link_dev_buffers(page
, bh
);
1170 init_page_buffers(page
, bdev
, block
, size
);
1171 spin_unlock(&inode
->i_mapping
->private_lock
);
1177 page_cache_release(page
);
1182 * Create buffers for the specified block device block's page. If
1183 * that page was dirty, the buffers are set dirty also.
1185 * Except that's a bug. Attaching dirty buffers to a dirty
1186 * blockdev's page can result in filesystem corruption, because
1187 * some of those buffers may be aliases of filesystem data.
1188 * grow_dev_page() will go BUG() if this happens.
1191 grow_buffers(struct block_device
*bdev
, sector_t block
, int size
)
1200 } while ((size
<< sizebits
) < PAGE_SIZE
);
1202 index
= block
>> sizebits
;
1203 block
= index
<< sizebits
;
1205 /* Create a page with the proper size buffers.. */
1206 page
= grow_dev_page(bdev
, block
, index
, size
);
1210 page_cache_release(page
);
1214 struct buffer_head
*
1215 __getblk_slow(struct block_device
*bdev
, sector_t block
, int size
)
1217 /* Size must be multiple of hard sectorsize */
1218 if (unlikely(size
& (bdev_hardsect_size(bdev
)-1) ||
1219 (size
< 512 || size
> PAGE_SIZE
))) {
1220 printk(KERN_ERR
"getblk(): invalid block size %d requested\n",
1222 printk(KERN_ERR
"hardsect size: %d\n",
1223 bdev_hardsect_size(bdev
));
1230 struct buffer_head
* bh
;
1232 bh
= __find_get_block(bdev
, block
, size
);
1236 if (!grow_buffers(bdev
, block
, size
))
1242 * The relationship between dirty buffers and dirty pages:
1244 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1245 * the page is tagged dirty in its radix tree.
1247 * At all times, the dirtiness of the buffers represents the dirtiness of
1248 * subsections of the page. If the page has buffers, the page dirty bit is
1249 * merely a hint about the true dirty state.
1251 * When a page is set dirty in its entirety, all its buffers are marked dirty
1252 * (if the page has buffers).
1254 * When a buffer is marked dirty, its page is dirtied, but the page's other
1257 * Also. When blockdev buffers are explicitly read with bread(), they
1258 * individually become uptodate. But their backing page remains not
1259 * uptodate - even if all of its buffers are uptodate. A subsequent
1260 * block_read_full_page() against that page will discover all the uptodate
1261 * buffers, will set the page uptodate and will perform no I/O.
1265 * mark_buffer_dirty - mark a buffer_head as needing writeout
1267 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1268 * backing page dirty, then tag the page as dirty in its address_space's radix
1269 * tree and then attach the address_space's inode to its superblock's dirty
1272 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1273 * mapping->tree_lock and the global inode_lock.
1275 void fastcall
mark_buffer_dirty(struct buffer_head
*bh
)
1277 if (!buffer_dirty(bh
) && !test_set_buffer_dirty(bh
))
1278 __set_page_dirty_nobuffers(bh
->b_page
);
1282 * Decrement a buffer_head's reference count. If all buffers against a page
1283 * have zero reference count, are clean and unlocked, and if the page is clean
1284 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1285 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1286 * a page but it ends up not being freed, and buffers may later be reattached).
1288 void __brelse(struct buffer_head
* buf
)
1290 if (atomic_read(&buf
->b_count
)) {
1294 printk(KERN_ERR
"VFS: brelse: Trying to free free buffer\n");
1299 * bforget() is like brelse(), except it discards any
1300 * potentially dirty data.
1302 void __bforget(struct buffer_head
*bh
)
1304 clear_buffer_dirty(bh
);
1305 if (!list_empty(&bh
->b_assoc_buffers
)) {
1306 struct address_space
*buffer_mapping
= bh
->b_page
->mapping
;
1308 spin_lock(&buffer_mapping
->private_lock
);
1309 list_del_init(&bh
->b_assoc_buffers
);
1310 spin_unlock(&buffer_mapping
->private_lock
);
1315 static struct buffer_head
*__bread_slow(struct buffer_head
*bh
)
1318 if (buffer_uptodate(bh
)) {
1323 bh
->b_end_io
= end_buffer_read_sync
;
1324 submit_bh(READ
, bh
);
1326 if (buffer_uptodate(bh
))
1334 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1335 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1336 * refcount elevated by one when they're in an LRU. A buffer can only appear
1337 * once in a particular CPU's LRU. A single buffer can be present in multiple
1338 * CPU's LRUs at the same time.
1340 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1341 * sb_find_get_block().
1343 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1344 * a local interrupt disable for that.
1347 #define BH_LRU_SIZE 8
1350 struct buffer_head
*bhs
[BH_LRU_SIZE
];
1353 static DEFINE_PER_CPU(struct bh_lru
, bh_lrus
) = {{ NULL
}};
1356 #define bh_lru_lock() local_irq_disable()
1357 #define bh_lru_unlock() local_irq_enable()
1359 #define bh_lru_lock() preempt_disable()
1360 #define bh_lru_unlock() preempt_enable()
1363 static inline void check_irqs_on(void)
1365 #ifdef irqs_disabled
1366 BUG_ON(irqs_disabled());
1371 * The LRU management algorithm is dopey-but-simple. Sorry.
1373 static void bh_lru_install(struct buffer_head
*bh
)
1375 struct buffer_head
*evictee
= NULL
;
1380 lru
= &__get_cpu_var(bh_lrus
);
1381 if (lru
->bhs
[0] != bh
) {
1382 struct buffer_head
*bhs
[BH_LRU_SIZE
];
1388 for (in
= 0; in
< BH_LRU_SIZE
; in
++) {
1389 struct buffer_head
*bh2
= lru
->bhs
[in
];
1394 if (out
>= BH_LRU_SIZE
) {
1395 BUG_ON(evictee
!= NULL
);
1402 while (out
< BH_LRU_SIZE
)
1404 memcpy(lru
->bhs
, bhs
, sizeof(bhs
));
1413 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1415 static inline struct buffer_head
*
1416 lookup_bh_lru(struct block_device
*bdev
, sector_t block
, int size
)
1418 struct buffer_head
*ret
= NULL
;
1424 lru
= &__get_cpu_var(bh_lrus
);
1425 for (i
= 0; i
< BH_LRU_SIZE
; i
++) {
1426 struct buffer_head
*bh
= lru
->bhs
[i
];
1428 if (bh
&& bh
->b_bdev
== bdev
&&
1429 bh
->b_blocknr
== block
&& bh
->b_size
== size
) {
1432 lru
->bhs
[i
] = lru
->bhs
[i
- 1];
1447 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1448 * it in the LRU and mark it as accessed. If it is not present then return
1451 struct buffer_head
*
1452 __find_get_block(struct block_device
*bdev
, sector_t block
, int size
)
1454 struct buffer_head
*bh
= lookup_bh_lru(bdev
, block
, size
);
1457 bh
= __find_get_block_slow(bdev
, block
, size
);
1465 EXPORT_SYMBOL(__find_get_block
);
1468 * __getblk will locate (and, if necessary, create) the buffer_head
1469 * which corresponds to the passed block_device, block and size. The
1470 * returned buffer has its reference count incremented.
1472 * __getblk() cannot fail - it just keeps trying. If you pass it an
1473 * illegal block number, __getblk() will happily return a buffer_head
1474 * which represents the non-existent block. Very weird.
1476 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1477 * attempt is failing. FIXME, perhaps?
1479 struct buffer_head
*
1480 __getblk(struct block_device
*bdev
, sector_t block
, int size
)
1482 struct buffer_head
*bh
= __find_get_block(bdev
, block
, size
);
1486 bh
= __getblk_slow(bdev
, block
, size
);
1489 EXPORT_SYMBOL(__getblk
);
1492 * Do async read-ahead on a buffer..
1494 void __breadahead(struct block_device
*bdev
, sector_t block
, int size
)
1496 struct buffer_head
*bh
= __getblk(bdev
, block
, size
);
1497 ll_rw_block(READA
, 1, &bh
);
1500 EXPORT_SYMBOL(__breadahead
);
1503 * __bread() - reads a specified block and returns the bh
1504 * @block: number of block
1505 * @size: size (in bytes) to read
1507 * Reads a specified block, and returns buffer head that contains it.
1508 * It returns NULL if the block was unreadable.
1510 struct buffer_head
*
1511 __bread(struct block_device
*bdev
, sector_t block
, int size
)
1513 struct buffer_head
*bh
= __getblk(bdev
, block
, size
);
1515 if (!buffer_uptodate(bh
))
1516 bh
= __bread_slow(bh
);
1519 EXPORT_SYMBOL(__bread
);
1522 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1523 * This doesn't race because it runs in each cpu either in irq
1524 * or with preempt disabled.
1526 static void invalidate_bh_lru(void *arg
)
1528 struct bh_lru
*b
= &get_cpu_var(bh_lrus
);
1531 for (i
= 0; i
< BH_LRU_SIZE
; i
++) {
1535 put_cpu_var(bh_lrus
);
1538 static void invalidate_bh_lrus(void)
1540 on_each_cpu(invalidate_bh_lru
, NULL
, 1, 1);
1543 void set_bh_page(struct buffer_head
*bh
,
1544 struct page
*page
, unsigned long offset
)
1547 if (offset
>= PAGE_SIZE
)
1549 if (PageHighMem(page
))
1551 * This catches illegal uses and preserves the offset:
1553 bh
->b_data
= (char *)(0 + offset
);
1555 bh
->b_data
= page_address(page
) + offset
;
1557 EXPORT_SYMBOL(set_bh_page
);
1560 * Called when truncating a buffer on a page completely.
1562 static inline void discard_buffer(struct buffer_head
* bh
)
1565 clear_buffer_dirty(bh
);
1567 clear_buffer_mapped(bh
);
1568 clear_buffer_req(bh
);
1569 clear_buffer_new(bh
);
1570 clear_buffer_delay(bh
);
1575 * try_to_release_page() - release old fs-specific metadata on a page
1577 * @page: the page which the kernel is trying to free
1578 * @gfp_mask: memory allocation flags (and I/O mode)
1580 * The address_space is to try to release any data against the page
1581 * (presumably at page->private). If the release was successful, return `1'.
1582 * Otherwise return zero.
1584 * The @gfp_mask argument specifies whether I/O may be performed to release
1585 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1587 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1589 int try_to_release_page(struct page
*page
, int gfp_mask
)
1591 struct address_space
* const mapping
= page
->mapping
;
1593 BUG_ON(!PageLocked(page
));
1594 if (PageWriteback(page
))
1597 if (mapping
&& mapping
->a_ops
->releasepage
)
1598 return mapping
->a_ops
->releasepage(page
, gfp_mask
);
1599 return try_to_free_buffers(page
);
1601 EXPORT_SYMBOL(try_to_release_page
);
1604 * block_invalidatepage - invalidate part of all of a buffer-backed page
1606 * @page: the page which is affected
1607 * @offset: the index of the truncation point
1609 * block_invalidatepage() is called when all or part of the page has become
1610 * invalidatedby a truncate operation.
1612 * block_invalidatepage() does not have to release all buffers, but it must
1613 * ensure that no dirty buffer is left outside @offset and that no I/O
1614 * is underway against any of the blocks which are outside the truncation
1615 * point. Because the caller is about to free (and possibly reuse) those
1618 int block_invalidatepage(struct page
*page
, unsigned long offset
)
1620 struct buffer_head
*head
, *bh
, *next
;
1621 unsigned int curr_off
= 0;
1624 BUG_ON(!PageLocked(page
));
1625 if (!page_has_buffers(page
))
1628 head
= page_buffers(page
);
1631 unsigned int next_off
= curr_off
+ bh
->b_size
;
1632 next
= bh
->b_this_page
;
1635 * is this block fully invalidated?
1637 if (offset
<= curr_off
)
1639 curr_off
= next_off
;
1641 } while (bh
!= head
);
1644 * We release buffers only if the entire page is being invalidated.
1645 * The get_block cached value has been unconditionally invalidated,
1646 * so real IO is not possible anymore.
1649 ret
= try_to_release_page(page
, 0);
1653 EXPORT_SYMBOL(block_invalidatepage
);
1656 * We attach and possibly dirty the buffers atomically wrt
1657 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1658 * is already excluded via the page lock.
1660 void create_empty_buffers(struct page
*page
,
1661 unsigned long blocksize
, unsigned long b_state
)
1663 struct buffer_head
*bh
, *head
, *tail
;
1665 head
= alloc_page_buffers(page
, blocksize
, 1);
1668 bh
->b_state
|= b_state
;
1670 bh
= bh
->b_this_page
;
1672 tail
->b_this_page
= head
;
1674 spin_lock(&page
->mapping
->private_lock
);
1675 if (PageUptodate(page
) || PageDirty(page
)) {
1678 if (PageDirty(page
))
1679 set_buffer_dirty(bh
);
1680 if (PageUptodate(page
))
1681 set_buffer_uptodate(bh
);
1682 bh
= bh
->b_this_page
;
1683 } while (bh
!= head
);
1685 attach_page_buffers(page
, head
);
1686 spin_unlock(&page
->mapping
->private_lock
);
1688 EXPORT_SYMBOL(create_empty_buffers
);
1691 * We are taking a block for data and we don't want any output from any
1692 * buffer-cache aliases starting from return from that function and
1693 * until the moment when something will explicitly mark the buffer
1694 * dirty (hopefully that will not happen until we will free that block ;-)
1695 * We don't even need to mark it not-uptodate - nobody can expect
1696 * anything from a newly allocated buffer anyway. We used to used
1697 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1698 * don't want to mark the alias unmapped, for example - it would confuse
1699 * anyone who might pick it with bread() afterwards...
1701 * Also.. Note that bforget() doesn't lock the buffer. So there can
1702 * be writeout I/O going on against recently-freed buffers. We don't
1703 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1704 * only if we really need to. That happens here.
1706 void unmap_underlying_metadata(struct block_device
*bdev
, sector_t block
)
1708 struct buffer_head
*old_bh
;
1712 old_bh
= __find_get_block_slow(bdev
, block
, 0);
1714 clear_buffer_dirty(old_bh
);
1715 wait_on_buffer(old_bh
);
1716 clear_buffer_req(old_bh
);
1720 EXPORT_SYMBOL(unmap_underlying_metadata
);
1723 * NOTE! All mapped/uptodate combinations are valid:
1725 * Mapped Uptodate Meaning
1727 * No No "unknown" - must do get_block()
1728 * No Yes "hole" - zero-filled
1729 * Yes No "allocated" - allocated on disk, not read in
1730 * Yes Yes "valid" - allocated and up-to-date in memory.
1732 * "Dirty" is valid only with the last case (mapped+uptodate).
1736 * While block_write_full_page is writing back the dirty buffers under
1737 * the page lock, whoever dirtied the buffers may decide to clean them
1738 * again at any time. We handle that by only looking at the buffer
1739 * state inside lock_buffer().
1741 * If block_write_full_page() is called for regular writeback
1742 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1743 * locked buffer. This only can happen if someone has written the buffer
1744 * directly, with submit_bh(). At the address_space level PageWriteback
1745 * prevents this contention from occurring.
1747 static int __block_write_full_page(struct inode
*inode
, struct page
*page
,
1748 get_block_t
*get_block
, struct writeback_control
*wbc
)
1752 sector_t last_block
;
1753 struct buffer_head
*bh
, *head
;
1754 int nr_underway
= 0;
1756 BUG_ON(!PageLocked(page
));
1758 last_block
= (i_size_read(inode
) - 1) >> inode
->i_blkbits
;
1760 if (!page_has_buffers(page
)) {
1761 create_empty_buffers(page
, 1 << inode
->i_blkbits
,
1762 (1 << BH_Dirty
)|(1 << BH_Uptodate
));
1766 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1767 * here, and the (potentially unmapped) buffers may become dirty at
1768 * any time. If a buffer becomes dirty here after we've inspected it
1769 * then we just miss that fact, and the page stays dirty.
1771 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1772 * handle that here by just cleaning them.
1775 block
= page
->index
<< (PAGE_CACHE_SHIFT
- inode
->i_blkbits
);
1776 head
= page_buffers(page
);
1780 * Get all the dirty buffers mapped to disk addresses and
1781 * handle any aliases from the underlying blockdev's mapping.
1784 if (block
> last_block
) {
1786 * mapped buffers outside i_size will occur, because
1787 * this page can be outside i_size when there is a
1788 * truncate in progress.
1791 * The buffer was zeroed by block_write_full_page()
1793 clear_buffer_dirty(bh
);
1794 set_buffer_uptodate(bh
);
1795 } else if (!buffer_mapped(bh
) && buffer_dirty(bh
)) {
1796 err
= get_block(inode
, block
, bh
, 1);
1799 if (buffer_new(bh
)) {
1800 /* blockdev mappings never come here */
1801 clear_buffer_new(bh
);
1802 unmap_underlying_metadata(bh
->b_bdev
,
1806 bh
= bh
->b_this_page
;
1808 } while (bh
!= head
);
1812 if (!buffer_mapped(bh
))
1815 * If it's a fully non-blocking write attempt and we cannot
1816 * lock the buffer then redirty the page. Note that this can
1817 * potentially cause a busy-wait loop from pdflush and kswapd
1818 * activity, but those code paths have their own higher-level
1821 if (wbc
->sync_mode
!= WB_SYNC_NONE
|| !wbc
->nonblocking
) {
1823 } else if (test_set_buffer_locked(bh
)) {
1824 redirty_page_for_writepage(wbc
, page
);
1827 if (test_clear_buffer_dirty(bh
)) {
1828 mark_buffer_async_write(bh
);
1832 } while ((bh
= bh
->b_this_page
) != head
);
1835 * The page and its buffers are protected by PageWriteback(), so we can
1836 * drop the bh refcounts early.
1838 BUG_ON(PageWriteback(page
));
1839 set_page_writeback(page
);
1843 struct buffer_head
*next
= bh
->b_this_page
;
1844 if (buffer_async_write(bh
)) {
1845 submit_bh(WRITE
, bh
);
1850 } while (bh
!= head
);
1854 if (nr_underway
== 0) {
1856 * The page was marked dirty, but the buffers were
1857 * clean. Someone wrote them back by hand with
1858 * ll_rw_block/submit_bh. A rare case.
1862 if (!buffer_uptodate(bh
)) {
1866 bh
= bh
->b_this_page
;
1867 } while (bh
!= head
);
1869 SetPageUptodate(page
);
1870 end_page_writeback(page
);
1872 * The page and buffer_heads can be released at any time from
1875 wbc
->pages_skipped
++; /* We didn't write this page */
1881 * ENOSPC, or some other error. We may already have added some
1882 * blocks to the file, so we need to write these out to avoid
1883 * exposing stale data.
1884 * The page is currently locked and not marked for writeback
1887 /* Recovery: lock and submit the mapped buffers */
1890 if (buffer_mapped(bh
) && buffer_dirty(bh
)) {
1892 mark_buffer_async_write(bh
);
1895 * The buffer may have been set dirty during
1896 * attachment to a dirty page.
1898 clear_buffer_dirty(bh
);
1900 } while ((bh
= bh
->b_this_page
) != head
);
1902 BUG_ON(PageWriteback(page
));
1903 set_page_writeback(page
);
1906 struct buffer_head
*next
= bh
->b_this_page
;
1907 if (buffer_async_write(bh
)) {
1908 clear_buffer_dirty(bh
);
1909 submit_bh(WRITE
, bh
);
1914 } while (bh
!= head
);
1918 static int __block_prepare_write(struct inode
*inode
, struct page
*page
,
1919 unsigned from
, unsigned to
, get_block_t
*get_block
)
1921 unsigned block_start
, block_end
;
1924 unsigned blocksize
, bbits
;
1925 struct buffer_head
*bh
, *head
, *wait
[2], **wait_bh
=wait
;
1927 BUG_ON(!PageLocked(page
));
1928 BUG_ON(from
> PAGE_CACHE_SIZE
);
1929 BUG_ON(to
> PAGE_CACHE_SIZE
);
1932 blocksize
= 1 << inode
->i_blkbits
;
1933 if (!page_has_buffers(page
))
1934 create_empty_buffers(page
, blocksize
, 0);
1935 head
= page_buffers(page
);
1937 bbits
= inode
->i_blkbits
;
1938 block
= (sector_t
)page
->index
<< (PAGE_CACHE_SHIFT
- bbits
);
1940 for(bh
= head
, block_start
= 0; bh
!= head
|| !block_start
;
1941 block
++, block_start
=block_end
, bh
= bh
->b_this_page
) {
1942 block_end
= block_start
+ blocksize
;
1943 if (block_end
<= from
|| block_start
>= to
) {
1944 if (PageUptodate(page
)) {
1945 if (!buffer_uptodate(bh
))
1946 set_buffer_uptodate(bh
);
1951 clear_buffer_new(bh
);
1952 if (!buffer_mapped(bh
)) {
1953 err
= get_block(inode
, block
, bh
, 1);
1956 if (buffer_new(bh
)) {
1957 clear_buffer_new(bh
);
1958 unmap_underlying_metadata(bh
->b_bdev
,
1960 if (PageUptodate(page
)) {
1961 set_buffer_uptodate(bh
);
1964 if (block_end
> to
|| block_start
< from
) {
1967 kaddr
= kmap_atomic(page
, KM_USER0
);
1971 if (block_start
< from
)
1972 memset(kaddr
+block_start
,
1973 0, from
-block_start
);
1974 flush_dcache_page(page
);
1975 kunmap_atomic(kaddr
, KM_USER0
);
1980 if (PageUptodate(page
)) {
1981 if (!buffer_uptodate(bh
))
1982 set_buffer_uptodate(bh
);
1985 if (!buffer_uptodate(bh
) && !buffer_delay(bh
) &&
1986 (block_start
< from
|| block_end
> to
)) {
1987 ll_rw_block(READ
, 1, &bh
);
1992 * If we issued read requests - let them complete.
1994 while(wait_bh
> wait
) {
1995 wait_on_buffer(*--wait_bh
);
1996 if (!buffer_uptodate(*wait_bh
))
2002 * Zero out any newly allocated blocks to avoid exposing stale
2003 * data. If BH_New is set, we know that the block was newly
2004 * allocated in the above loop.
2009 block_end
= block_start
+blocksize
;
2010 if (block_end
<= from
)
2012 if (block_start
>= to
)
2014 if (buffer_new(bh
)) {
2017 clear_buffer_new(bh
);
2018 kaddr
= kmap_atomic(page
, KM_USER0
);
2019 memset(kaddr
+block_start
, 0, bh
->b_size
);
2020 kunmap_atomic(kaddr
, KM_USER0
);
2021 set_buffer_uptodate(bh
);
2022 mark_buffer_dirty(bh
);
2025 block_start
= block_end
;
2026 bh
= bh
->b_this_page
;
2027 } while (bh
!= head
);
2031 static int __block_commit_write(struct inode
*inode
, struct page
*page
,
2032 unsigned from
, unsigned to
)
2034 unsigned block_start
, block_end
;
2037 struct buffer_head
*bh
, *head
;
2039 blocksize
= 1 << inode
->i_blkbits
;
2041 for(bh
= head
= page_buffers(page
), block_start
= 0;
2042 bh
!= head
|| !block_start
;
2043 block_start
=block_end
, bh
= bh
->b_this_page
) {
2044 block_end
= block_start
+ blocksize
;
2045 if (block_end
<= from
|| block_start
>= to
) {
2046 if (!buffer_uptodate(bh
))
2049 set_buffer_uptodate(bh
);
2050 mark_buffer_dirty(bh
);
2055 * If this is a partial write which happened to make all buffers
2056 * uptodate then we can optimize away a bogus readpage() for
2057 * the next read(). Here we 'discover' whether the page went
2058 * uptodate as a result of this (potentially partial) write.
2061 SetPageUptodate(page
);
2066 * Generic "read page" function for block devices that have the normal
2067 * get_block functionality. This is most of the block device filesystems.
2068 * Reads the page asynchronously --- the unlock_buffer() and
2069 * set/clear_buffer_uptodate() functions propagate buffer state into the
2070 * page struct once IO has completed.
2072 int block_read_full_page(struct page
*page
, get_block_t
*get_block
)
2074 struct inode
*inode
= page
->mapping
->host
;
2075 sector_t iblock
, lblock
;
2076 struct buffer_head
*bh
, *head
, *arr
[MAX_BUF_PER_PAGE
];
2077 unsigned int blocksize
;
2079 int fully_mapped
= 1;
2081 BUG_ON(!PageLocked(page
));
2082 blocksize
= 1 << inode
->i_blkbits
;
2083 if (!page_has_buffers(page
))
2084 create_empty_buffers(page
, blocksize
, 0);
2085 head
= page_buffers(page
);
2087 iblock
= (sector_t
)page
->index
<< (PAGE_CACHE_SHIFT
- inode
->i_blkbits
);
2088 lblock
= (i_size_read(inode
)+blocksize
-1) >> inode
->i_blkbits
;
2094 if (buffer_uptodate(bh
))
2097 if (!buffer_mapped(bh
)) {
2099 if (iblock
< lblock
) {
2100 if (get_block(inode
, iblock
, bh
, 0))
2103 if (!buffer_mapped(bh
)) {
2104 void *kaddr
= kmap_atomic(page
, KM_USER0
);
2105 memset(kaddr
+ i
* blocksize
, 0, blocksize
);
2106 flush_dcache_page(page
);
2107 kunmap_atomic(kaddr
, KM_USER0
);
2108 set_buffer_uptodate(bh
);
2112 * get_block() might have updated the buffer
2115 if (buffer_uptodate(bh
))
2119 } while (i
++, iblock
++, (bh
= bh
->b_this_page
) != head
);
2122 SetPageMappedToDisk(page
);
2126 * All buffers are uptodate - we can set the page uptodate
2127 * as well. But not if get_block() returned an error.
2129 if (!PageError(page
))
2130 SetPageUptodate(page
);
2135 /* Stage two: lock the buffers */
2136 for (i
= 0; i
< nr
; i
++) {
2139 mark_buffer_async_read(bh
);
2143 * Stage 3: start the IO. Check for uptodateness
2144 * inside the buffer lock in case another process reading
2145 * the underlying blockdev brought it uptodate (the sct fix).
2147 for (i
= 0; i
< nr
; i
++) {
2149 if (buffer_uptodate(bh
))
2150 end_buffer_async_read(bh
, 1);
2152 submit_bh(READ
, bh
);
2157 /* utility function for filesystems that need to do work on expanding
2158 * truncates. Uses prepare/commit_write to allow the filesystem to
2159 * deal with the hole.
2161 int generic_cont_expand(struct inode
*inode
, loff_t size
)
2163 struct address_space
*mapping
= inode
->i_mapping
;
2165 unsigned long index
, offset
, limit
;
2169 limit
= current
->signal
->rlim
[RLIMIT_FSIZE
].rlim_cur
;
2170 if (limit
!= RLIM_INFINITY
&& size
> (loff_t
)limit
) {
2171 send_sig(SIGXFSZ
, current
, 0);
2174 if (size
> inode
->i_sb
->s_maxbytes
)
2177 offset
= (size
& (PAGE_CACHE_SIZE
-1)); /* Within page */
2179 /* ugh. in prepare/commit_write, if from==to==start of block, we
2180 ** skip the prepare. make sure we never send an offset for the start
2183 if ((offset
& (inode
->i_sb
->s_blocksize
- 1)) == 0) {
2186 index
= size
>> PAGE_CACHE_SHIFT
;
2188 page
= grab_cache_page(mapping
, index
);
2191 err
= mapping
->a_ops
->prepare_write(NULL
, page
, offset
, offset
);
2193 err
= mapping
->a_ops
->commit_write(NULL
, page
, offset
, offset
);
2196 page_cache_release(page
);
2204 * For moronic filesystems that do not allow holes in file.
2205 * We may have to extend the file.
2208 int cont_prepare_write(struct page
*page
, unsigned offset
,
2209 unsigned to
, get_block_t
*get_block
, loff_t
*bytes
)
2211 struct address_space
*mapping
= page
->mapping
;
2212 struct inode
*inode
= mapping
->host
;
2213 struct page
*new_page
;
2217 unsigned blocksize
= 1 << inode
->i_blkbits
;
2220 while(page
->index
> (pgpos
= *bytes
>>PAGE_CACHE_SHIFT
)) {
2222 new_page
= grab_cache_page(mapping
, pgpos
);
2225 /* we might sleep */
2226 if (*bytes
>>PAGE_CACHE_SHIFT
!= pgpos
) {
2227 unlock_page(new_page
);
2228 page_cache_release(new_page
);
2231 zerofrom
= *bytes
& ~PAGE_CACHE_MASK
;
2232 if (zerofrom
& (blocksize
-1)) {
2233 *bytes
|= (blocksize
-1);
2236 status
= __block_prepare_write(inode
, new_page
, zerofrom
,
2237 PAGE_CACHE_SIZE
, get_block
);
2240 kaddr
= kmap_atomic(new_page
, KM_USER0
);
2241 memset(kaddr
+zerofrom
, 0, PAGE_CACHE_SIZE
-zerofrom
);
2242 flush_dcache_page(new_page
);
2243 kunmap_atomic(kaddr
, KM_USER0
);
2244 generic_commit_write(NULL
, new_page
, zerofrom
, PAGE_CACHE_SIZE
);
2245 unlock_page(new_page
);
2246 page_cache_release(new_page
);
2249 if (page
->index
< pgpos
) {
2250 /* completely inside the area */
2253 /* page covers the boundary, find the boundary offset */
2254 zerofrom
= *bytes
& ~PAGE_CACHE_MASK
;
2256 /* if we will expand the thing last block will be filled */
2257 if (to
> zerofrom
&& (zerofrom
& (blocksize
-1))) {
2258 *bytes
|= (blocksize
-1);
2262 /* starting below the boundary? Nothing to zero out */
2263 if (offset
<= zerofrom
)
2266 status
= __block_prepare_write(inode
, page
, zerofrom
, to
, get_block
);
2269 if (zerofrom
< offset
) {
2270 kaddr
= kmap_atomic(page
, KM_USER0
);
2271 memset(kaddr
+zerofrom
, 0, offset
-zerofrom
);
2272 flush_dcache_page(page
);
2273 kunmap_atomic(kaddr
, KM_USER0
);
2274 __block_commit_write(inode
, page
, zerofrom
, offset
);
2278 ClearPageUptodate(page
);
2282 ClearPageUptodate(new_page
);
2283 unlock_page(new_page
);
2284 page_cache_release(new_page
);
2289 int block_prepare_write(struct page
*page
, unsigned from
, unsigned to
,
2290 get_block_t
*get_block
)
2292 struct inode
*inode
= page
->mapping
->host
;
2293 int err
= __block_prepare_write(inode
, page
, from
, to
, get_block
);
2295 ClearPageUptodate(page
);
2299 int block_commit_write(struct page
*page
, unsigned from
, unsigned to
)
2301 struct inode
*inode
= page
->mapping
->host
;
2302 __block_commit_write(inode
,page
,from
,to
);
2306 int generic_commit_write(struct file
*file
, struct page
*page
,
2307 unsigned from
, unsigned to
)
2309 struct inode
*inode
= page
->mapping
->host
;
2310 loff_t pos
= ((loff_t
)page
->index
<< PAGE_CACHE_SHIFT
) + to
;
2311 __block_commit_write(inode
,page
,from
,to
);
2313 * No need to use i_size_read() here, the i_size
2314 * cannot change under us because we hold i_sem.
2316 if (pos
> inode
->i_size
) {
2317 i_size_write(inode
, pos
);
2318 mark_inode_dirty(inode
);
2325 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2326 * immediately, while under the page lock. So it needs a special end_io
2327 * handler which does not touch the bh after unlocking it.
2329 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2330 * a race there is benign: unlock_buffer() only use the bh's address for
2331 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2334 static void end_buffer_read_nobh(struct buffer_head
*bh
, int uptodate
)
2337 set_buffer_uptodate(bh
);
2339 /* This happens, due to failed READA attempts. */
2340 clear_buffer_uptodate(bh
);
2346 * On entry, the page is fully not uptodate.
2347 * On exit the page is fully uptodate in the areas outside (from,to)
2349 int nobh_prepare_write(struct page
*page
, unsigned from
, unsigned to
,
2350 get_block_t
*get_block
)
2352 struct inode
*inode
= page
->mapping
->host
;
2353 const unsigned blkbits
= inode
->i_blkbits
;
2354 const unsigned blocksize
= 1 << blkbits
;
2355 struct buffer_head map_bh
;
2356 struct buffer_head
*read_bh
[MAX_BUF_PER_PAGE
];
2357 unsigned block_in_page
;
2358 unsigned block_start
;
2359 sector_t block_in_file
;
2364 int is_mapped_to_disk
= 1;
2367 if (PageMappedToDisk(page
))
2370 block_in_file
= (sector_t
)page
->index
<< (PAGE_CACHE_SHIFT
- blkbits
);
2371 map_bh
.b_page
= page
;
2374 * We loop across all blocks in the page, whether or not they are
2375 * part of the affected region. This is so we can discover if the
2376 * page is fully mapped-to-disk.
2378 for (block_start
= 0, block_in_page
= 0;
2379 block_start
< PAGE_CACHE_SIZE
;
2380 block_in_page
++, block_start
+= blocksize
) {
2381 unsigned block_end
= block_start
+ blocksize
;
2386 if (block_start
>= to
)
2388 ret
= get_block(inode
, block_in_file
+ block_in_page
,
2392 if (!buffer_mapped(&map_bh
))
2393 is_mapped_to_disk
= 0;
2394 if (buffer_new(&map_bh
))
2395 unmap_underlying_metadata(map_bh
.b_bdev
,
2397 if (PageUptodate(page
))
2399 if (buffer_new(&map_bh
) || !buffer_mapped(&map_bh
)) {
2400 kaddr
= kmap_atomic(page
, KM_USER0
);
2401 if (block_start
< from
) {
2402 memset(kaddr
+block_start
, 0, from
-block_start
);
2405 if (block_end
> to
) {
2406 memset(kaddr
+ to
, 0, block_end
- to
);
2409 flush_dcache_page(page
);
2410 kunmap_atomic(kaddr
, KM_USER0
);
2413 if (buffer_uptodate(&map_bh
))
2414 continue; /* reiserfs does this */
2415 if (block_start
< from
|| block_end
> to
) {
2416 struct buffer_head
*bh
= alloc_buffer_head(GFP_NOFS
);
2422 bh
->b_state
= map_bh
.b_state
;
2423 atomic_set(&bh
->b_count
, 0);
2424 bh
->b_this_page
= NULL
;
2426 bh
->b_blocknr
= map_bh
.b_blocknr
;
2427 bh
->b_size
= blocksize
;
2428 bh
->b_data
= (char *)(long)block_start
;
2429 bh
->b_bdev
= map_bh
.b_bdev
;
2430 bh
->b_private
= NULL
;
2431 read_bh
[nr_reads
++] = bh
;
2436 struct buffer_head
*bh
;
2439 * The page is locked, so these buffers are protected from
2440 * any VM or truncate activity. Hence we don't need to care
2441 * for the buffer_head refcounts.
2443 for (i
= 0; i
< nr_reads
; i
++) {
2446 bh
->b_end_io
= end_buffer_read_nobh
;
2447 submit_bh(READ
, bh
);
2449 for (i
= 0; i
< nr_reads
; i
++) {
2452 if (!buffer_uptodate(bh
))
2454 free_buffer_head(bh
);
2461 if (is_mapped_to_disk
)
2462 SetPageMappedToDisk(page
);
2463 SetPageUptodate(page
);
2466 * Setting the page dirty here isn't necessary for the prepare_write
2467 * function - commit_write will do that. But if/when this function is
2468 * used within the pagefault handler to ensure that all mmapped pages
2469 * have backing space in the filesystem, we will need to dirty the page
2470 * if its contents were altered.
2473 set_page_dirty(page
);
2478 for (i
= 0; i
< nr_reads
; i
++) {
2480 free_buffer_head(read_bh
[i
]);
2484 * Error recovery is pretty slack. Clear the page and mark it dirty
2485 * so we'll later zero out any blocks which _were_ allocated.
2487 kaddr
= kmap_atomic(page
, KM_USER0
);
2488 memset(kaddr
, 0, PAGE_CACHE_SIZE
);
2489 kunmap_atomic(kaddr
, KM_USER0
);
2490 SetPageUptodate(page
);
2491 set_page_dirty(page
);
2494 EXPORT_SYMBOL(nobh_prepare_write
);
2496 int nobh_commit_write(struct file
*file
, struct page
*page
,
2497 unsigned from
, unsigned to
)
2499 struct inode
*inode
= page
->mapping
->host
;
2500 loff_t pos
= ((loff_t
)page
->index
<< PAGE_CACHE_SHIFT
) + to
;
2502 set_page_dirty(page
);
2503 if (pos
> inode
->i_size
) {
2504 i_size_write(inode
, pos
);
2505 mark_inode_dirty(inode
);
2509 EXPORT_SYMBOL(nobh_commit_write
);
2512 * nobh_writepage() - based on block_full_write_page() except
2513 * that it tries to operate without attaching bufferheads to
2516 int nobh_writepage(struct page
*page
, get_block_t
*get_block
,
2517 struct writeback_control
*wbc
)
2519 struct inode
* const inode
= page
->mapping
->host
;
2520 loff_t i_size
= i_size_read(inode
);
2521 const pgoff_t end_index
= i_size
>> PAGE_CACHE_SHIFT
;
2526 /* Is the page fully inside i_size? */
2527 if (page
->index
< end_index
)
2530 /* Is the page fully outside i_size? (truncate in progress) */
2531 offset
= i_size
& (PAGE_CACHE_SIZE
-1);
2532 if (page
->index
>= end_index
+1 || !offset
) {
2534 * The page may have dirty, unmapped buffers. For example,
2535 * they may have been added in ext3_writepage(). Make them
2536 * freeable here, so the page does not leak.
2539 /* Not really sure about this - do we need this ? */
2540 if (page
->mapping
->a_ops
->invalidatepage
)
2541 page
->mapping
->a_ops
->invalidatepage(page
, offset
);
2544 return 0; /* don't care */
2548 * The page straddles i_size. It must be zeroed out on each and every
2549 * writepage invocation because it may be mmapped. "A file is mapped
2550 * in multiples of the page size. For a file that is not a multiple of
2551 * the page size, the remaining memory is zeroed when mapped, and
2552 * writes to that region are not written out to the file."
2554 kaddr
= kmap_atomic(page
, KM_USER0
);
2555 memset(kaddr
+ offset
, 0, PAGE_CACHE_SIZE
- offset
);
2556 flush_dcache_page(page
);
2557 kunmap_atomic(kaddr
, KM_USER0
);
2559 ret
= mpage_writepage(page
, get_block
, wbc
);
2561 ret
= __block_write_full_page(inode
, page
, get_block
, wbc
);
2564 EXPORT_SYMBOL(nobh_writepage
);
2567 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2569 int nobh_truncate_page(struct address_space
*mapping
, loff_t from
)
2571 struct inode
*inode
= mapping
->host
;
2572 unsigned blocksize
= 1 << inode
->i_blkbits
;
2573 pgoff_t index
= from
>> PAGE_CACHE_SHIFT
;
2574 unsigned offset
= from
& (PAGE_CACHE_SIZE
-1);
2577 struct address_space_operations
*a_ops
= mapping
->a_ops
;
2581 if ((offset
& (blocksize
- 1)) == 0)
2585 page
= grab_cache_page(mapping
, index
);
2589 to
= (offset
+ blocksize
) & ~(blocksize
- 1);
2590 ret
= a_ops
->prepare_write(NULL
, page
, offset
, to
);
2592 kaddr
= kmap_atomic(page
, KM_USER0
);
2593 memset(kaddr
+ offset
, 0, PAGE_CACHE_SIZE
- offset
);
2594 flush_dcache_page(page
);
2595 kunmap_atomic(kaddr
, KM_USER0
);
2596 set_page_dirty(page
);
2599 page_cache_release(page
);
2603 EXPORT_SYMBOL(nobh_truncate_page
);
2605 int block_truncate_page(struct address_space
*mapping
,
2606 loff_t from
, get_block_t
*get_block
)
2608 pgoff_t index
= from
>> PAGE_CACHE_SHIFT
;
2609 unsigned offset
= from
& (PAGE_CACHE_SIZE
-1);
2612 unsigned length
, pos
;
2613 struct inode
*inode
= mapping
->host
;
2615 struct buffer_head
*bh
;
2619 blocksize
= 1 << inode
->i_blkbits
;
2620 length
= offset
& (blocksize
- 1);
2622 /* Block boundary? Nothing to do */
2626 length
= blocksize
- length
;
2627 iblock
= index
<< (PAGE_CACHE_SHIFT
- inode
->i_blkbits
);
2629 page
= grab_cache_page(mapping
, index
);
2634 if (!page_has_buffers(page
))
2635 create_empty_buffers(page
, blocksize
, 0);
2637 /* Find the buffer that contains "offset" */
2638 bh
= page_buffers(page
);
2640 while (offset
>= pos
) {
2641 bh
= bh
->b_this_page
;
2647 if (!buffer_mapped(bh
)) {
2648 err
= get_block(inode
, iblock
, bh
, 0);
2651 /* unmapped? It's a hole - nothing to do */
2652 if (!buffer_mapped(bh
))
2656 /* Ok, it's mapped. Make sure it's up-to-date */
2657 if (PageUptodate(page
))
2658 set_buffer_uptodate(bh
);
2660 if (!buffer_uptodate(bh
) && !buffer_delay(bh
)) {
2662 ll_rw_block(READ
, 1, &bh
);
2664 /* Uhhuh. Read error. Complain and punt. */
2665 if (!buffer_uptodate(bh
))
2669 kaddr
= kmap_atomic(page
, KM_USER0
);
2670 memset(kaddr
+ offset
, 0, length
);
2671 flush_dcache_page(page
);
2672 kunmap_atomic(kaddr
, KM_USER0
);
2674 mark_buffer_dirty(bh
);
2679 page_cache_release(page
);
2685 * The generic ->writepage function for buffer-backed address_spaces
2687 int block_write_full_page(struct page
*page
, get_block_t
*get_block
,
2688 struct writeback_control
*wbc
)
2690 struct inode
* const inode
= page
->mapping
->host
;
2691 loff_t i_size
= i_size_read(inode
);
2692 const pgoff_t end_index
= i_size
>> PAGE_CACHE_SHIFT
;
2696 /* Is the page fully inside i_size? */
2697 if (page
->index
< end_index
)
2698 return __block_write_full_page(inode
, page
, get_block
, wbc
);
2700 /* Is the page fully outside i_size? (truncate in progress) */
2701 offset
= i_size
& (PAGE_CACHE_SIZE
-1);
2702 if (page
->index
>= end_index
+1 || !offset
) {
2704 * The page may have dirty, unmapped buffers. For example,
2705 * they may have been added in ext3_writepage(). Make them
2706 * freeable here, so the page does not leak.
2708 block_invalidatepage(page
, 0);
2710 return 0; /* don't care */
2714 * The page straddles i_size. It must be zeroed out on each and every
2715 * writepage invokation because it may be mmapped. "A file is mapped
2716 * in multiples of the page size. For a file that is not a multiple of
2717 * the page size, the remaining memory is zeroed when mapped, and
2718 * writes to that region are not written out to the file."
2720 kaddr
= kmap_atomic(page
, KM_USER0
);
2721 memset(kaddr
+ offset
, 0, PAGE_CACHE_SIZE
- offset
);
2722 flush_dcache_page(page
);
2723 kunmap_atomic(kaddr
, KM_USER0
);
2724 return __block_write_full_page(inode
, page
, get_block
, wbc
);
2727 sector_t
generic_block_bmap(struct address_space
*mapping
, sector_t block
,
2728 get_block_t
*get_block
)
2730 struct buffer_head tmp
;
2731 struct inode
*inode
= mapping
->host
;
2734 get_block(inode
, block
, &tmp
, 0);
2735 return tmp
.b_blocknr
;
2738 static int end_bio_bh_io_sync(struct bio
*bio
, unsigned int bytes_done
, int err
)
2740 struct buffer_head
*bh
= bio
->bi_private
;
2745 if (err
== -EOPNOTSUPP
) {
2746 set_bit(BIO_EOPNOTSUPP
, &bio
->bi_flags
);
2747 set_bit(BH_Eopnotsupp
, &bh
->b_state
);
2750 bh
->b_end_io(bh
, test_bit(BIO_UPTODATE
, &bio
->bi_flags
));
2755 int submit_bh(int rw
, struct buffer_head
* bh
)
2760 BUG_ON(!buffer_locked(bh
));
2761 BUG_ON(!buffer_mapped(bh
));
2762 BUG_ON(!bh
->b_end_io
);
2764 if (buffer_ordered(bh
) && (rw
== WRITE
))
2768 * Only clear out a write error when rewriting, should this
2769 * include WRITE_SYNC as well?
2771 if (test_set_buffer_req(bh
) && (rw
== WRITE
|| rw
== WRITE_BARRIER
))
2772 clear_buffer_write_io_error(bh
);
2775 * from here on down, it's all bio -- do the initial mapping,
2776 * submit_bio -> generic_make_request may further map this bio around
2778 bio
= bio_alloc(GFP_NOIO
, 1);
2780 bio
->bi_sector
= bh
->b_blocknr
* (bh
->b_size
>> 9);
2781 bio
->bi_bdev
= bh
->b_bdev
;
2782 bio
->bi_io_vec
[0].bv_page
= bh
->b_page
;
2783 bio
->bi_io_vec
[0].bv_len
= bh
->b_size
;
2784 bio
->bi_io_vec
[0].bv_offset
= bh_offset(bh
);
2788 bio
->bi_size
= bh
->b_size
;
2790 bio
->bi_end_io
= end_bio_bh_io_sync
;
2791 bio
->bi_private
= bh
;
2794 submit_bio(rw
, bio
);
2796 if (bio_flagged(bio
, BIO_EOPNOTSUPP
))
2804 * ll_rw_block: low-level access to block devices (DEPRECATED)
2805 * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
2806 * @nr: number of &struct buffer_heads in the array
2807 * @bhs: array of pointers to &struct buffer_head
2809 * ll_rw_block() takes an array of pointers to &struct buffer_heads,
2810 * and requests an I/O operation on them, either a %READ or a %WRITE.
2811 * The third %READA option is described in the documentation for
2812 * generic_make_request() which ll_rw_block() calls.
2814 * This function drops any buffer that it cannot get a lock on (with the
2815 * BH_Lock state bit), any buffer that appears to be clean when doing a
2816 * write request, and any buffer that appears to be up-to-date when doing
2817 * read request. Further it marks as clean buffers that are processed for
2818 * writing (the buffer cache won't assume that they are actually clean until
2819 * the buffer gets unlocked).
2821 * ll_rw_block sets b_end_io to simple completion handler that marks
2822 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2825 * All of the buffers must be for the same device, and must also be a
2826 * multiple of the current approved size for the device.
2828 void ll_rw_block(int rw
, int nr
, struct buffer_head
*bhs
[])
2832 for (i
= 0; i
< nr
; i
++) {
2833 struct buffer_head
*bh
= bhs
[i
];
2835 if (test_set_buffer_locked(bh
))
2840 if (test_clear_buffer_dirty(bh
)) {
2841 bh
->b_end_io
= end_buffer_write_sync
;
2842 submit_bh(WRITE
, bh
);
2846 if (!buffer_uptodate(bh
)) {
2847 bh
->b_end_io
= end_buffer_read_sync
;
2858 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2859 * and then start new I/O and then wait upon it. The caller must have a ref on
2862 int sync_dirty_buffer(struct buffer_head
*bh
)
2866 WARN_ON(atomic_read(&bh
->b_count
) < 1);
2868 if (test_clear_buffer_dirty(bh
)) {
2870 bh
->b_end_io
= end_buffer_write_sync
;
2871 ret
= submit_bh(WRITE
, bh
);
2873 if (buffer_eopnotsupp(bh
)) {
2874 clear_buffer_eopnotsupp(bh
);
2877 if (!ret
&& !buffer_uptodate(bh
))
2886 * try_to_free_buffers() checks if all the buffers on this particular page
2887 * are unused, and releases them if so.
2889 * Exclusion against try_to_free_buffers may be obtained by either
2890 * locking the page or by holding its mapping's private_lock.
2892 * If the page is dirty but all the buffers are clean then we need to
2893 * be sure to mark the page clean as well. This is because the page
2894 * may be against a block device, and a later reattachment of buffers
2895 * to a dirty page will set *all* buffers dirty. Which would corrupt
2896 * filesystem data on the same device.
2898 * The same applies to regular filesystem pages: if all the buffers are
2899 * clean then we set the page clean and proceed. To do that, we require
2900 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2903 * try_to_free_buffers() is non-blocking.
2905 static inline int buffer_busy(struct buffer_head
*bh
)
2907 return atomic_read(&bh
->b_count
) |
2908 (bh
->b_state
& ((1 << BH_Dirty
) | (1 << BH_Lock
)));
2912 drop_buffers(struct page
*page
, struct buffer_head
**buffers_to_free
)
2914 struct buffer_head
*head
= page_buffers(page
);
2915 struct buffer_head
*bh
;
2919 if (buffer_write_io_error(bh
) && page
->mapping
)
2920 set_bit(AS_EIO
, &page
->mapping
->flags
);
2921 if (buffer_busy(bh
))
2923 bh
= bh
->b_this_page
;
2924 } while (bh
!= head
);
2927 struct buffer_head
*next
= bh
->b_this_page
;
2929 if (!list_empty(&bh
->b_assoc_buffers
))
2930 __remove_assoc_queue(bh
);
2932 } while (bh
!= head
);
2933 *buffers_to_free
= head
;
2934 __clear_page_buffers(page
);
2940 int try_to_free_buffers(struct page
*page
)
2942 struct address_space
* const mapping
= page
->mapping
;
2943 struct buffer_head
*buffers_to_free
= NULL
;
2946 BUG_ON(!PageLocked(page
));
2947 if (PageWriteback(page
))
2950 if (mapping
== NULL
) { /* can this still happen? */
2951 ret
= drop_buffers(page
, &buffers_to_free
);
2955 spin_lock(&mapping
->private_lock
);
2956 ret
= drop_buffers(page
, &buffers_to_free
);
2959 * If the filesystem writes its buffers by hand (eg ext3)
2960 * then we can have clean buffers against a dirty page. We
2961 * clean the page here; otherwise later reattachment of buffers
2962 * could encounter a non-uptodate page, which is unresolvable.
2963 * This only applies in the rare case where try_to_free_buffers
2964 * succeeds but the page is not freed.
2966 clear_page_dirty(page
);
2968 spin_unlock(&mapping
->private_lock
);
2970 if (buffers_to_free
) {
2971 struct buffer_head
*bh
= buffers_to_free
;
2974 struct buffer_head
*next
= bh
->b_this_page
;
2975 free_buffer_head(bh
);
2977 } while (bh
!= buffers_to_free
);
2981 EXPORT_SYMBOL(try_to_free_buffers
);
2983 int block_sync_page(struct page
*page
)
2985 struct address_space
*mapping
;
2988 mapping
= page_mapping(page
);
2990 blk_run_backing_dev(mapping
->backing_dev_info
, page
);
2995 * There are no bdflush tunables left. But distributions are
2996 * still running obsolete flush daemons, so we terminate them here.
2998 * Use of bdflush() is deprecated and will be removed in a future kernel.
2999 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
3001 asmlinkage
long sys_bdflush(int func
, long data
)
3003 static int msg_count
;
3005 if (!capable(CAP_SYS_ADMIN
))
3008 if (msg_count
< 5) {
3011 "warning: process `%s' used the obsolete bdflush"
3012 " system call\n", current
->comm
);
3013 printk(KERN_INFO
"Fix your initscripts?\n");
3022 * Buffer-head allocation
3024 static kmem_cache_t
*bh_cachep
;
3027 * Once the number of bh's in the machine exceeds this level, we start
3028 * stripping them in writeback.
3030 static int max_buffer_heads
;
3032 int buffer_heads_over_limit
;
3034 struct bh_accounting
{
3035 int nr
; /* Number of live bh's */
3036 int ratelimit
; /* Limit cacheline bouncing */
3039 static DEFINE_PER_CPU(struct bh_accounting
, bh_accounting
) = {0, 0};
3041 static void recalc_bh_state(void)
3046 if (__get_cpu_var(bh_accounting
).ratelimit
++ < 4096)
3048 __get_cpu_var(bh_accounting
).ratelimit
= 0;
3050 tot
+= per_cpu(bh_accounting
, i
).nr
;
3051 buffer_heads_over_limit
= (tot
> max_buffer_heads
);
3054 struct buffer_head
*alloc_buffer_head(unsigned int __nocast gfp_flags
)
3056 struct buffer_head
*ret
= kmem_cache_alloc(bh_cachep
, gfp_flags
);
3059 __get_cpu_var(bh_accounting
).nr
++;
3065 EXPORT_SYMBOL(alloc_buffer_head
);
3067 void free_buffer_head(struct buffer_head
*bh
)
3069 BUG_ON(!list_empty(&bh
->b_assoc_buffers
));
3070 kmem_cache_free(bh_cachep
, bh
);
3072 __get_cpu_var(bh_accounting
).nr
--;
3076 EXPORT_SYMBOL(free_buffer_head
);
3079 init_buffer_head(void *data
, kmem_cache_t
*cachep
, unsigned long flags
)
3081 if ((flags
& (SLAB_CTOR_VERIFY
|SLAB_CTOR_CONSTRUCTOR
)) ==
3082 SLAB_CTOR_CONSTRUCTOR
) {
3083 struct buffer_head
* bh
= (struct buffer_head
*)data
;
3085 memset(bh
, 0, sizeof(*bh
));
3086 INIT_LIST_HEAD(&bh
->b_assoc_buffers
);
3090 #ifdef CONFIG_HOTPLUG_CPU
3091 static void buffer_exit_cpu(int cpu
)
3094 struct bh_lru
*b
= &per_cpu(bh_lrus
, cpu
);
3096 for (i
= 0; i
< BH_LRU_SIZE
; i
++) {
3102 static int buffer_cpu_notify(struct notifier_block
*self
,
3103 unsigned long action
, void *hcpu
)
3105 if (action
== CPU_DEAD
)
3106 buffer_exit_cpu((unsigned long)hcpu
);
3109 #endif /* CONFIG_HOTPLUG_CPU */
3111 void __init
buffer_init(void)
3115 bh_cachep
= kmem_cache_create("buffer_head",
3116 sizeof(struct buffer_head
), 0,
3117 SLAB_PANIC
, init_buffer_head
, NULL
);
3120 * Limit the bh occupancy to 10% of ZONE_NORMAL
3122 nrpages
= (nr_free_buffer_pages() * 10) / 100;
3123 max_buffer_heads
= nrpages
* (PAGE_SIZE
/ sizeof(struct buffer_head
));
3124 hotcpu_notifier(buffer_cpu_notify
, 0);
3127 EXPORT_SYMBOL(__bforget
);
3128 EXPORT_SYMBOL(__brelse
);
3129 EXPORT_SYMBOL(__wait_on_buffer
);
3130 EXPORT_SYMBOL(block_commit_write
);
3131 EXPORT_SYMBOL(block_prepare_write
);
3132 EXPORT_SYMBOL(block_read_full_page
);
3133 EXPORT_SYMBOL(block_sync_page
);
3134 EXPORT_SYMBOL(block_truncate_page
);
3135 EXPORT_SYMBOL(block_write_full_page
);
3136 EXPORT_SYMBOL(cont_prepare_write
);
3137 EXPORT_SYMBOL(end_buffer_async_write
);
3138 EXPORT_SYMBOL(end_buffer_read_sync
);
3139 EXPORT_SYMBOL(end_buffer_write_sync
);
3140 EXPORT_SYMBOL(file_fsync
);
3141 EXPORT_SYMBOL(fsync_bdev
);
3142 EXPORT_SYMBOL(generic_block_bmap
);
3143 EXPORT_SYMBOL(generic_commit_write
);
3144 EXPORT_SYMBOL(generic_cont_expand
);
3145 EXPORT_SYMBOL(init_buffer
);
3146 EXPORT_SYMBOL(invalidate_bdev
);
3147 EXPORT_SYMBOL(ll_rw_block
);
3148 EXPORT_SYMBOL(mark_buffer_dirty
);
3149 EXPORT_SYMBOL(submit_bh
);
3150 EXPORT_SYMBOL(sync_dirty_buffer
);
3151 EXPORT_SYMBOL(unlock_buffer
);