| blob | 1a30db77af3257505eae9706db2c366146666751 |
1 /*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
7 /*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/capability.h>
28 #include <linux/blkdev.h>
29 #include <linux/file.h>
30 #include <linux/quotaops.h>
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/writeback.h>
34 #include <linux/hash.h>
35 #include <linux/suspend.h>
36 #include <linux/buffer_head.h>
37 #include <linux/task_io_accounting_ops.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
47 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
51 {
54 }
58 {
61 }
64 {
66 TASK_UNINTERRUPTIBLE);
67 }
71 {
75 }
78 /*
79 * Block until a buffer comes unlocked. This doesn't stop it
80 * from becoming locked again - you have to lock it yourself
81 * if you want to preserve its state.
82 */
84 {
86 }
89 static void
91 {
95 }
99 {
103 }
107 {
112 }
114 /*
115 * End-of-IO handler helper function which does not touch the bh after
116 * unlocking it.
117 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
118 * a race there is benign: unlock_buffer() only use the bh's address for
119 * hashing after unlocking the buffer, so it doesn't actually touch the bh
120 * itself.
121 */
123 {
127 /* This happens, due to failed READA attempts. */
129 }
131 }
133 /*
134 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
135 * unlock the buffer. This is what ll_rw_block uses too.
136 */
138 {
141 }
145 {
156 }
159 }
162 }
165 /*
166 * Various filesystems appear to want __find_get_block to be non-blocking.
167 * But it's the page lock which protects the buffers. To get around this,
168 * we get exclusion from try_to_free_buffers with the blockdev mapping's
169 * private_lock.
170 *
171 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
172 * may be quite high. This code could TryLock the page, and if that
173 * succeeds, there is no need to take private_lock. (But if
174 * private_lock is contended then so is mapping->tree_lock).
175 */
178 {
182 pgoff_t index;
205 }
209 /* we might be here because some of the buffers on this page are
210 * not mapped. This is due to various races between
211 * file io on the block device and getblk. It gets dealt with
212 * elsewhere, don't buffer_error if we had some unmapped buffers
213 */
225 }
226 out_unlock:
229 out:
231 }
233 /*
234 * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
235 */
237 {
251 }
252 }
254 /*
255 * I/O completion handler for block_read_full_page() - pages
256 * which come unlocked at the end of I/O.
257 */
259 {
276 }
278 /*
279 * Be _very_ careful from here on. Bad things can happen if
280 * two buffer heads end IO at almost the same time and both
281 * decide that the page is now completely done.
282 */
295 }
301 /*
302 * If none of the buffers had errors and they are all
303 * uptodate then we can set the page uptodate.
304 */
310 still_busy:
314 }
316 /*
317 * Completion handler for block_write_full_page() - pages which are unlocked
318 * during I/O, and which have PageWriteback cleared upon I/O completion.
319 */
321 {
339 }
344 }
357 }
359 }
365 still_busy:
369 }
372 /*
373 * If a page's buffers are under async readin (end_buffer_async_read
374 * completion) then there is a possibility that another thread of
375 * control could lock one of the buffers after it has completed
376 * but while some of the other buffers have not completed. This
377 * locked buffer would confuse end_buffer_async_read() into not unlocking
378 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
379 * that this buffer is not under async I/O.
380 *
381 * The page comes unlocked when it has no locked buffer_async buffers
382 * left.
383 *
384 * PageLocked prevents anyone starting new async I/O reads any of
385 * the buffers.
386 *
387 * PageWriteback is used to prevent simultaneous writeout of the same
388 * page.
389 *
390 * PageLocked prevents anyone from starting writeback of a page which is
391 * under read I/O (PageWriteback is only ever set against a locked page).
392 */
394 {
397 }
401 {
404 }
407 {
409 }
413 /*
414 * fs/buffer.c contains helper functions for buffer-backed address space's
415 * fsync functions. A common requirement for buffer-based filesystems is
416 * that certain data from the backing blockdev needs to be written out for
417 * a successful fsync(). For example, ext2 indirect blocks need to be
418 * written back and waited upon before fsync() returns.
419 *
420 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
421 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
422 * management of a list of dependent buffers at ->i_mapping->private_list.
423 *
424 * Locking is a little subtle: try_to_free_buffers() will remove buffers
425 * from their controlling inode's queue when they are being freed. But
426 * try_to_free_buffers() will be operating against the *blockdev* mapping
427 * at the time, not against the S_ISREG file which depends on those buffers.
428 * So the locking for private_list is via the private_lock in the address_space
429 * which backs the buffers. Which is different from the address_space
430 * against which the buffers are listed. So for a particular address_space,
431 * mapping->private_lock does *not* protect mapping->private_list! In fact,
432 * mapping->private_list will always be protected by the backing blockdev's
433 * ->private_lock.
434 *
435 * Which introduces a requirement: all buffers on an address_space's
436 * ->private_list must be from the same address_space: the blockdev's.
437 *
438 * address_spaces which do not place buffers at ->private_list via these
439 * utility functions are free to use private_lock and private_list for
440 * whatever they want. The only requirement is that list_empty(private_list)
441 * be true at clear_inode() time.
442 *
443 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
444 * filesystems should do that. invalidate_inode_buffers() should just go
445 * BUG_ON(!list_empty).
446 *
447 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
448 * take an address_space, not an inode. And it should be called
449 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
450 * queued up.
451 *
452 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
453 * list if it is already on a list. Because if the buffer is on a list,
454 * it *must* already be on the right one. If not, the filesystem is being
455 * silly. This will save a ton of locking. But first we have to ensure
456 * that buffers are taken *off* the old inode's list when they are freed
457 * (presumably in truncate). That requires careful auditing of all
458 * filesystems (do it inside bforget()). It could also be done by bringing
459 * b_inode back.
460 */
462 /*
463 * The buffer's backing address_space's private_lock must be held
464 */
466 {
472 }
475 {
477 }
479 /*
480 * osync is designed to support O_SYNC io. It waits synchronously for
481 * all already-submitted IO to complete, but does not queue any new
482 * writes to the disk.
483 *
484 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
485 * you dirty the buffers, and then use osync_inode_buffers to wait for
486 * completion. Any other dirty buffers which are not yet queued for
487 * write will not be flushed to disk by the osync.
488 */
490 {
496 repeat:
508 }
509 }
512 }
515 {
520 }
523 {
527 }
529 /**
530 * emergency_thaw_all -- forcibly thaw every frozen filesystem
531 *
532 * Used for emergency unfreeze of all filesystems via SysRq
533 */
535 {
542 }
543 }
545 /**
546 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
547 * @mapping: the mapping which wants those buffers written
548 *
549 * Starts I/O against the buffers at mapping->private_list, and waits upon
550 * that I/O.
551 *
552 * Basically, this is a convenience function for fsync().
553 * @mapping is a file or directory which needs those buffers to be written for
554 * a successful fsync().
555 */
557 {
565 }
568 /*
569 * Called when we've recently written block `bblock', and it is known that
570 * `bblock' was for a buffer_boundary() buffer. This means that the block at
571 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
572 * dirty, schedule it for IO. So that indirects merge nicely with their data.
573 */
576 {
582 }
583 }
586 {
595 }
602 }
603 }
606 /*
607 * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
608 * dirty.
609 *
610 * If warn is true, then emit a warning if the page is not uptodate and has
611 * not been truncated.
612 */
615 {
622 }
625 }
627 /*
628 * Add a page to the dirty page list.
629 *
630 * It is a sad fact of life that this function is called from several places
631 * deeply under spinlocking. It may not sleep.
632 *
633 * If the page has buffers, the uptodate buffers are set dirty, to preserve
634 * dirty-state coherency between the page and the buffers. It the page does
635 * not have buffers then when they are later attached they will all be set
636 * dirty.
637 *
638 * The buffers are dirtied before the page is dirtied. There's a small race
639 * window in which a writepage caller may see the page cleanness but not the
640 * buffer dirtiness. That's fine. If this code were to set the page dirty
641 * before the buffers, a concurrent writepage caller could clear the page dirty
642 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
643 * page on the dirty page list.
644 *
645 * We use private_lock to lock against try_to_free_buffers while using the
646 * page's buffer list. Also use this to protect against clean buffers being
647 * added to the page after it was set dirty.
648 *
649 * FIXME: may need to call ->reservepage here as well. That's rather up to the
650 * address_space though.
651 */
653 {
669 }
676 }
679 /*
680 * Write out and wait upon a list of buffers.
681 *
682 * We have conflicting pressures: we want to make sure that all
683 * initially dirty buffers get waited on, but that any subsequently
684 * dirtied buffers don't. After all, we don't want fsync to last
685 * forever if somebody is actively writing to the file.
686 *
687 * Do this in two main stages: first we copy dirty buffers to a
688 * temporary inode list, queueing the writes as we go. Then we clean
689 * up, waiting for those writes to complete.
690 *
691 * During this second stage, any subsequent updates to the file may end
692 * up refiling the buffer on the original inode's dirty list again, so
693 * there is a chance we will end up with a buffer queued for write but
694 * not yet completed on that list. So, as a final cleanup we go through
695 * the osync code to catch these locked, dirty buffers without requeuing
696 * any newly dirty buffers for write.
697 */
699 {
714 /* Avoid race with mark_buffer_dirty_inode() which does
715 * a lockless check and we rely on seeing the dirty bit */
723 /*
724 * Ensure any pending I/O completes so that
725 * write_dirty_buffer() actually writes the
726 * current contents - it is a noop if I/O is
727 * still in flight on potentially older
728 * contents.
729 */
732 /*
733 * Kick off IO for the previous mapping. Note
734 * that we will not run the very last mapping,
735 * wait_on_buffer() will do that for us
736 * through sync_buffer().
737 */
740 }
741 }
742 }
753 /* Avoid race with mark_buffer_dirty_inode() which does
754 * a lockless check and we rely on seeing the dirty bit */
760 }
767 }
773 else
775 }
777 /*
778 * Invalidate any and all dirty buffers on a given inode. We are
779 * probably unmounting the fs, but that doesn't mean we have already
780 * done a sync(). Just drop the buffers from the inode list.
781 *
782 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
783 * assumes that all the buffers are against the blockdev. Not true
784 * for reiserfs.
785 */
787 {
797 }
798 }
801 /*
802 * Remove any clean buffers from the inode's buffer list. This is called
803 * when we're trying to free the inode itself. Those buffers can pin it.
804 *
805 * Returns true if all buffers were removed.
806 */
808 {
822 }
824 }
826 }
828 }
830 /*
831 * Create the appropriate buffers when given a page for data area and
832 * the size of each buffer.. Use the bh->b_this_page linked list to
833 * follow the buffers created. Return NULL if unable to create more
834 * buffers.
835 *
836 * The retry flag is used to differentiate async IO (paging, swapping)
837 * which may not fail from ordinary buffer allocations.
838 */
841 {
845 try_again:
862 /* Link the buffer to its page */
866 }
868 /*
869 * In case anything failed, we just free everything we got.
870 */
871 no_grow:
878 }
880 /*
881 * Return failure for non-async IO requests. Async IO requests
882 * are not allowed to fail, so we have to wait until buffer heads
883 * become available. But we don't want tasks sleeping with
884 * partially complete buffers, so all were released above.
885 */
889 /* We're _really_ low on memory. Now we just
890 * wait for old buffer heads to become free due to
891 * finishing IO. Since this is an async request and
892 * the reserve list is empty, we're sure there are
893 * async buffer heads in use.
894 */
897 }
902 {
912 }
914 /*
915 * Initialise the state of a blockdev page's buffers.
916 */
917 static void
920 {
933 }
934 block++;
937 }
939 /*
940 * Create the page-cache page that contains the requested block.
941 *
942 * This is user purely for blockdev mappings.
943 */
947 {
964 }
967 }
969 /*
970 * Allocate some buffers for this page
971 */
976 /*
977 * Link the page to the buffers and initialise them. Take the
978 * lock to be atomic wrt __find_get_block(), which does not
979 * run under the page lock.
980 */
987 failed:
992 }
994 /*
995 * Create buffers for the specified block device block's page. If
996 * that page was dirty, the buffers are set dirty also.
997 */
998 static int
1000 {
1002 pgoff_t index;
1007 sizebits++;
1012 /*
1013 * Check for a block which wants to lie outside our maximum possible
1014 * pagecache index. (this comparison is done using sector_t types).
1015 */
1024 }
1026 /* Create a page with the proper size buffers.. */
1033 }
1037 {
1038 /* Size must be multiple of hard sectorsize */
1042 size);
1048 }
1063 }
1064 }
1066 /*
1067 * The relationship between dirty buffers and dirty pages:
1068 *
1069 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1070 * the page is tagged dirty in its radix tree.
1071 *
1072 * At all times, the dirtiness of the buffers represents the dirtiness of
1073 * subsections of the page. If the page has buffers, the page dirty bit is
1074 * merely a hint about the true dirty state.
1075 *
1076 * When a page is set dirty in its entirety, all its buffers are marked dirty
1077 * (if the page has buffers).
1078 *
1079 * When a buffer is marked dirty, its page is dirtied, but the page's other
1080 * buffers are not.
1081 *
1082 * Also. When blockdev buffers are explicitly read with bread(), they
1083 * individually become uptodate. But their backing page remains not
1084 * uptodate - even if all of its buffers are uptodate. A subsequent
1085 * block_read_full_page() against that page will discover all the uptodate
1086 * buffers, will set the page uptodate and will perform no I/O.
1087 */
1089 /**
1090 * mark_buffer_dirty - mark a buffer_head as needing writeout
1091 * @bh: the buffer_head to mark dirty
1092 *
1093 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1094 * backing page dirty, then tag the page as dirty in its address_space's radix
1095 * tree and then attach the address_space's inode to its superblock's dirty
1096 * inode list.
1097 *
1098 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1099 * mapping->tree_lock and mapping->host->i_lock.
1100 */
1102 {
1105 /*
1106 * Very *carefully* optimize the it-is-already-dirty case.
1107 *
1108 * Don't let the final "is it dirty" escape to before we
1109 * perhaps modified the buffer.
1110 */
1115 }
1123 }
1124 }
1125 }
1128 /*
1129 * Decrement a buffer_head's reference count. If all buffers against a page
1130 * have zero reference count, are clean and unlocked, and if the page is clean
1131 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1132 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1133 * a page but it ends up not being freed, and buffers may later be reattached).
1134 */
1136 {
1140 }
1142 }
1145 /*
1146 * bforget() is like brelse(), except it discards any
1147 * potentially dirty data.
1148 */
1150 {
1159 }
1161 }
1165 {
1177 }
1180 }
1182 /*
1183 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1184 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1185 * refcount elevated by one when they're in an LRU. A buffer can only appear
1186 * once in a particular CPU's LRU. A single buffer can be present in multiple
1187 * CPU's LRUs at the same time.
1188 *
1189 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1190 * sb_find_get_block().
1191 *
1192 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1193 * a local interrupt disable for that.
1194 */
1196 #define BH_LRU_SIZE 8
1200 };
1204 #ifdef CONFIG_SMP
1205 #define bh_lru_lock() local_irq_disable()
1206 #define bh_lru_unlock() local_irq_enable()
1207 #else
1208 #define bh_lru_lock() preempt_disable()
1209 #define bh_lru_unlock() preempt_enable()
1210 #endif
1213 {
1214 #ifdef irqs_disabled
1216 #endif
1217 }
1219 /*
1220 * The LRU management algorithm is dopey-but-simple. Sorry.
1221 */
1223 {
1247 }
1248 }
1249 }
1253 }
1258 }
1260 /*
1261 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1262 */
1265 {
1280 i--;
1281 }
1283 }
1287 }
1288 }
1291 }
1293 /*
1294 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1295 * it in the LRU and mark it as accessed. If it is not present then return
1296 * NULL
1297 */
1300 {
1307 }
1311 }
1314 /*
1315 * __getblk will locate (and, if necessary, create) the buffer_head
1316 * which corresponds to the passed block_device, block and size. The
1317 * returned buffer has its reference count incremented.
1318 *
1319 * __getblk() cannot fail - it just keeps trying. If you pass it an
1320 * illegal block number, __getblk() will happily return a buffer_head
1321 * which represents the non-existent block. Very weird.
1322 *
1323 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1324 * attempt is failing. FIXME, perhaps?
1325 */
1328 {
1335 }
1338 /*
1339 * Do async read-ahead on a buffer..
1340 */
1342 {
1347 }
1348 }
1351 /**
1352 * __bread() - reads a specified block and returns the bh
1353 * @bdev: the block_device to read from
1354 * @block: number of block
1355 * @size: size (in bytes) to read
1356 *
1357 * Reads a specified block, and returns buffer head that contains it.
1358 * It returns NULL if the block was unreadable.
1359 */
1362 {
1368 }
1371 /*
1372 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1373 * This doesn't race because it runs in each cpu either in irq
1374 * or with preempt disabled.
1375 */
1377 {
1384 }
1386 }
1389 {
1391 }
1396 {
1400 /*
1401 * This catches illegal uses and preserves the offset:
1402 */
1404 else
1406 }
1409 /*
1410 * Called when truncating a buffer on a page completely.
1411 */
1413 {
1423 }
1425 /**
1426 * block_invalidatepage - invalidate part or all of a buffer-backed page
1427 *
1428 * @page: the page which is affected
1429 * @offset: the index of the truncation point
1430 *
1431 * block_invalidatepage() is called when all or part of the page has become
1432 * invalidated by a truncate operation.
1433 *
1434 * block_invalidatepage() does not have to release all buffers, but it must
1435 * ensure that no dirty buffer is left outside @offset and that no I/O
1436 * is underway against any of the blocks which are outside the truncation
1437 * point. Because the caller is about to free (and possibly reuse) those
1438 * blocks on-disk.
1439 */
1441 {
1455 /*
1456 * is this block fully invalidated?
1457 */
1464 /*
1465 * We release buffers only if the entire page is being invalidated.
1466 * The get_block cached value has been unconditionally invalidated,
1467 * so real IO is not possible anymore.
1468 */
1471 out:
1473 }
1476 /*
1477 * We attach and possibly dirty the buffers atomically wrt
1478 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1479 * is already excluded via the page lock.
1480 */
1483 {
1505 }
1508 }
1511 /*
1512 * We are taking a block for data and we don't want any output from any
1513 * buffer-cache aliases starting from return from that function and
1514 * until the moment when something will explicitly mark the buffer
1515 * dirty (hopefully that will not happen until we will free that block ;-)
1516 * We don't even need to mark it not-uptodate - nobody can expect
1517 * anything from a newly allocated buffer anyway. We used to used
1518 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1519 * don't want to mark the alias unmapped, for example - it would confuse
1520 * anyone who might pick it with bread() afterwards...
1521 *
1522 * Also.. Note that bforget() doesn't lock the buffer. So there can
1523 * be writeout I/O going on against recently-freed buffers. We don't
1524 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1525 * only if we really need to. That happens here.
1526 */
1528 {
1539 }
1540 }
1543 /*
1544 * NOTE! All mapped/uptodate combinations are valid:
1545 *
1546 * Mapped Uptodate Meaning
1547 *
1548 * No No "unknown" - must do get_block()
1549 * No Yes "hole" - zero-filled
1550 * Yes No "allocated" - allocated on disk, not read in
1551 * Yes Yes "valid" - allocated and up-to-date in memory.
1552 *
1553 * "Dirty" is valid only with the last case (mapped+uptodate).
1554 */
1556 /*
1557 * While block_write_full_page is writing back the dirty buffers under
1558 * the page lock, whoever dirtied the buffers may decide to clean them
1559 * again at any time. We handle that by only looking at the buffer
1560 * state inside lock_buffer().
1561 *
1562 * If block_write_full_page() is called for regular writeback
1563 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1564 * locked buffer. This only can happen if someone has written the buffer
1565 * directly, with submit_bh(). At the address_space level PageWriteback
1566 * prevents this contention from occurring.
1567 *
1568 * If block_write_full_page() is called with wbc->sync_mode ==
1569 * WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this
1570 * causes the writes to be flagged as synchronous writes.
1571 */
1575 {
1577 sector_t block;
1578 sector_t last_block;
1592 }
1594 /*
1595 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1596 * here, and the (potentially unmapped) buffers may become dirty at
1597 * any time. If a buffer becomes dirty here after we've inspected it
1598 * then we just miss that fact, and the page stays dirty.
1599 *
1600 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1601 * handle that here by just cleaning them.
1602 */
1608 /*
1609 * Get all the dirty buffers mapped to disk addresses and
1610 * handle any aliases from the underlying blockdev's mapping.
1611 */
1614 /*
1615 * mapped buffers outside i_size will occur, because
1616 * this page can be outside i_size when there is a
1617 * truncate in progress.
1618 */
1619 /*
1620 * The buffer was zeroed by block_write_full_page()
1621 */
1632 /* blockdev mappings never come here */
1636 }
1637 }
1639 block++;
1645 /*
1646 * If it's a fully non-blocking write attempt and we cannot
1647 * lock the buffer then redirty the page. Note that this can
1648 * potentially cause a busy-wait loop from writeback threads
1649 * and kswapd activity, but those code paths have their own
1650 * higher-level throttling.
1651 */
1657 }
1662 }
1665 /*
1666 * The page and its buffers are protected by PageWriteback(), so we can
1667 * drop the bh refcounts early.
1668 */
1676 nr_underway++;
1677 }
1683 done:
1685 /*
1686 * The page was marked dirty, but the buffers were
1687 * clean. Someone wrote them back by hand with
1688 * ll_rw_block/submit_bh. A rare case.
1689 */
1692 /*
1693 * The page and buffer_heads can be released at any time from
1694 * here on.
1695 */
1696 }
1699 recover:
1700 /*
1701 * ENOSPC, or some other error. We may already have added some
1702 * blocks to the file, so we need to write these out to avoid
1703 * exposing stale data.
1704 * The page is currently locked and not marked for writeback
1705 */
1707 /* Recovery: lock and submit the mapped buffers */
1714 /*
1715 * The buffer may have been set dirty during
1716 * attachment to a dirty page.
1717 */
1719 }
1730 nr_underway++;
1731 }
1736 }
1738 /*
1739 * If a page has any new buffers, zero them out here, and mark them uptodate
1740 * and dirty so they'll be written out (in order to prevent uninitialised
1741 * block data from leaking). And clear the new bit.
1742 */
1744 {
1767 }
1771 }
1772 }
1777 }
1782 {
1787 sector_t block;
1812 }
1814 }
1830 }
1836 }
1837 }
1842 }
1848 }
1849 }
1850 /*
1851 * If we issued read requests - let them complete.
1852 */
1857 }
1861 }
1866 {
1884 }
1886 }
1888 /*
1889 * If this is a partial write which happened to make all buffers
1890 * uptodate then we can optimize away a bogus readpage() for
1891 * the next read(). Here we 'discover' whether the page went
1892 * uptodate as a result of this (potentially partial) write.
1893 */
1897 }
1899 /*
1900 * block_write_begin takes care of the basic task of block allocation and
1901 * bringing partial write blocks uptodate first.
1902 *
1903 * The filesystem needs to handle block truncation upon failure.
1904 */
1907 {
1921 }
1925 }
1931 {
1938 /*
1939 * The buffers that were written will now be uptodate, so we
1940 * don't have to worry about a readpage reading them and
1941 * overwriting a partial write. However if we have encountered
1942 * a short write and only partially written into a buffer, it
1943 * will not be marked uptodate, so a readpage might come in and
1944 * destroy our partial write.
1945 *
1946 * Do the simplest thing, and just treat any short write to a
1947 * non uptodate page as a zero-length write, and force the
1948 * caller to redo the whole thing.
1949 */
1954 }
1957 /* This could be a short (even 0-length) commit */
1961 }
1967 {
1973 /*
1974 * No need to use i_size_read() here, the i_size
1975 * cannot change under us because we hold i_mutex.
1976 *
1977 * But it's important to update i_size while still holding page lock:
1978 * page writeout could otherwise come in and zero beyond i_size.
1979 */
1983 }
1988 /*
1989 * Don't mark the inode dirty under page lock. First, it unnecessarily
1990 * makes the holding time of page lock longer. Second, it forces lock
1991 * ordering of page lock and transaction start for journaling
1992 * filesystems.
1993 */
1998 }
2001 /*
2002 * block_is_partially_uptodate checks whether buffers within a page are
2003 * uptodate or not.
2004 *
2005 * Returns true if all buffers which correspond to a file portion
2006 * we want to read are uptodate.
2007 */
2010 {
2035 }
2038 }
2044 }
2047 /*
2048 * Generic "read page" function for block devices that have the normal
2049 * get_block functionality. This is most of the block device filesystems.
2050 * Reads the page asynchronously --- the unlock_buffer() and
2051 * set/clear_buffer_uptodate() functions propagate buffer state into the
2052 * page struct once IO has completed.
2053 */
2055 {
2088 }
2094 }
2095 /*
2096 * get_block() might have updated the buffer
2097 * synchronously
2098 */
2101 }
2109 /*
2110 * All buffers are uptodate - we can set the page uptodate
2111 * as well. But not if get_block() returned an error.
2112 */
2117 }
2119 /* Stage two: lock the buffers */
2124 }
2126 /*
2127 * Stage 3: start the IO. Check for uptodateness
2128 * inside the buffer lock in case another process reading
2129 * the underlying blockdev brought it uptodate (the sct fix).
2130 */
2135 else
2137 }
2139 }
2142 /* utility function for filesystems that need to do work on expanding
2143 * truncates. Uses filesystem pagecache writes to allow the filesystem to
2144 * deal with the hole.
2145 */
2147 {
2166 out:
2168 }
2173 {
2179 loff_t curpos;
2191 }
2195 AOP_FLAG_UNINTERRUPTIBLE,
2208 }
2210 /* page covers the boundary, find the boundary offset */
2213 /* if we will expand the thing last block will be filled */
2216 }
2220 }
2224 AOP_FLAG_UNINTERRUPTIBLE,
2235 }
2236 out:
2238 }
2240 /*
2241 * For moronic filesystems that do not allow holes in file.
2242 * We may have to extend the file.
2243 */
2248 {
2262 }
2265 }
2269 {
2273 }
2276 /*
2277 * block_page_mkwrite() is not allowed to change the file size as it gets
2278 * called from a page fault handler when a page is first dirtied. Hence we must
2279 * be careful to check for EOF conditions here. We set the page up correctly
2280 * for a written page which means we get ENOSPC checking when writing into
2281 * holes and correct delalloc and unwritten extent mapping on filesystems that
2282 * support these features.
2283 *
2284 * We are not allowed to take the i_mutex here so we have to play games to
2285 * protect against truncate races as the page could now be beyond EOF. Because
2286 * truncate writes the inode size before removing pages, once we have the
2287 * page lock we can determine safely if the page is beyond EOF. If it is not
2288 * beyond EOF, then the page is guaranteed safe against truncation until we
2289 * unlock the page.
2290 *
2291 * Direct callers of this function should call vfs_check_frozen() so that page
2292 * fault does not busyloop until the fs is thawed.
2293 */
2295 get_block_t get_block)
2296 {
2300 loff_t size;
2307 /* We overload EFAULT to mean page got truncated */
2310 }
2312 /* page is wholly or partially inside EOF */
2315 else
2324 /*
2325 * Freezing in progress? We check after the page is marked dirty and
2326 * with page lock held so if the test here fails, we are sure freezing
2327 * code will wait during syncing until the page fault is done - at that
2328 * point page will be dirty and unlocked so freezing code will write it
2329 * and writeprotect it again.
2330 */
2335 }
2338 out_unlock:
2341 }
2345 get_block_t get_block)
2346 {
2350 /*
2351 * This check is racy but catches the common case. The check in
2352 * __block_page_mkwrite() is reliable.
2353 */
2357 }
2360 /*
2361 * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2362 * immediately, while under the page lock. So it needs a special end_io
2363 * handler which does not touch the bh after unlocking it.
2364 */
2366 {
2368 }
2370 /*
2371 * Attach the singly-linked list of buffers created by nobh_write_begin, to
2372 * the page (converting it to circular linked list and taking care of page
2373 * dirty races).
2374 */
2376 {
2392 }
2394 /*
2395 * On entry, the page is fully not uptodate.
2396 * On exit the page is fully uptodate in the areas outside (from,to)
2397 * The filesystem needs to handle block truncation upon failure.
2398 */
2403 {
2409 pgoff_t index;
2413 sector_t block_in_file;
2433 }
2438 /*
2439 * Allocate buffers so that we can keep track of state, and potentially
2440 * attach them to the page if an error occurs. In the common case of
2441 * no error, they will just be freed again without ever being attached
2442 * to the page (which is all OK, because we're under the page lock).
2443 *
2444 * Be careful: the buffer linked list is a NULL terminated one, rather
2445 * than the circular one we're used to.
2446 */
2451 }
2455 /*
2456 * We loop across all blocks in the page, whether or not they are
2457 * part of the affected region. This is so we can discover if the
2458 * page is fully mapped-to-disk.
2459 */
2481 }
2486 }
2493 nr_reads++;
2494 }
2495 }
2498 /*
2499 * The page is locked, so these buffers are protected from
2500 * any VM or truncate activity. Hence we don't need to care
2501 * for the buffer_head refcounts.
2502 */
2507 }
2510 }
2519 failed:
2521 /*
2522 * Error recovery is a bit difficult. We need to zero out blocks that
2523 * were newly allocated, and dirty them to ensure they get written out.
2524 * Buffers need to be attached to the page at this point, otherwise
2525 * the handling of potential IO errors during writeout would be hard
2526 * (could try doing synchronous writeout, but what if that fails too?)
2527 */
2531 out_release:
2537 }
2543 {
2560 }
2569 }
2572 }
2575 /*
2576 * nobh_writepage() - based on block_full_write_page() except
2577 * that it tries to operate without attaching bufferheads to
2578 * the page.
2579 */
2582 {
2589 /* Is the page fully inside i_size? */
2593 /* Is the page fully outside i_size? (truncate in progress) */
2596 /*
2597 * The page may have dirty, unmapped buffers. For example,
2598 * they may have been added in ext3_writepage(). Make them
2599 * freeable here, so the page does not leak.
2600 */
2601 #if 0
2602 /* Not really sure about this - do we need this ? */
2605 #endif
2608 }
2610 /*
2611 * The page straddles i_size. It must be zeroed out on each and every
2612 * writepage invocation because it may be mmapped. "A file is mapped
2613 * in multiples of the page size. For a file that is not a multiple of
2614 * the page size, the remaining memory is zeroed when mapped, and
2615 * writes to that region are not written out to the file."
2616 */
2618 out:
2622 end_buffer_async_write);
2624 }
2629 {
2633 sector_t iblock;
2643 /* Block boundary? Nothing to do */
2656 has_buffers:
2660 }
2662 /* Find the buffer that contains "offset" */
2665 iblock++;
2667 }
2674 /* unmapped? It's a hole - nothing to do */
2678 /* Ok, it's mapped. Make sure it's up-to-date */
2684 }
2689 }
2692 }
2697 unlock:
2700 out:
2702 }
2707 {
2711 sector_t iblock;
2721 /* Block boundary? Nothing to do */
2736 /* Find the buffer that contains "offset" */
2741 iblock++;
2743 }
2751 /* unmapped? It's a hole - nothing to do */
2754 }
2756 /* Ok, it's mapped. Make sure it's up-to-date */
2764 /* Uhhuh. Read error. Complain and punt. */
2767 }
2773 unlock:
2776 out:
2778 }
2781 /*
2782 * The generic ->writepage function for buffer-backed address_spaces
2783 * this form passes in the end_io handler used to finish the IO.
2784 */
2787 {
2793 /* Is the page fully inside i_size? */
2796 handler);
2798 /* Is the page fully outside i_size? (truncate in progress) */
2801 /*
2802 * The page may have dirty, unmapped buffers. For example,
2803 * they may have been added in ext3_writepage(). Make them
2804 * freeable here, so the page does not leak.
2805 */
2809 }
2811 /*
2812 * The page straddles i_size. It must be zeroed out on each and every
2813 * writepage invocation because it may be mmapped. "A file is mapped
2814 * in multiples of the page size. For a file that is not a multiple of
2815 * the page size, the remaining memory is zeroed when mapped, and
2816 * writes to that region are not written out to the file."
2817 */
2820 }
2823 /*
2824 * The generic ->writepage function for buffer-backed address_spaces
2825 */
2828 {
2830 end_buffer_async_write);
2831 }
2836 {
2844 }
2848 {
2853 }
2860 }
2863 {
2873 /*
2874 * Only clear out a write error when rewriting
2875 */
2879 /*
2880 * from here on down, it's all bio -- do the initial mapping,
2881 * submit_bio -> generic_make_request may further map this bio around
2882 */
2906 }
2909 /**
2910 * ll_rw_block: low-level access to block devices (DEPRECATED)
2911 * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
2912 * @nr: number of &struct buffer_heads in the array
2913 * @bhs: array of pointers to &struct buffer_head
2914 *
2915 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2916 * requests an I/O operation on them, either a %READ or a %WRITE. The third
2917 * %READA option is described in the documentation for generic_make_request()
2918 * which ll_rw_block() calls.
2919 *
2920 * This function drops any buffer that it cannot get a lock on (with the
2921 * BH_Lock state bit), any buffer that appears to be clean when doing a write
2922 * request, and any buffer that appears to be up-to-date when doing read
2923 * request. Further it marks as clean buffers that are processed for
2924 * writing (the buffer cache won't assume that they are actually clean
2925 * until the buffer gets unlocked).
2926 *
2927 * ll_rw_block sets b_end_io to simple completion handler that marks
2928 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2929 * any waiters.
2930 *
2931 * All of the buffers must be for the same device, and must also be a
2932 * multiple of the current approved size for the device.
2933 */
2935 {
2949 }
2956 }
2957 }
2959 }
2960 }
2964 {
2969 }
2973 }
2976 /*
2977 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2978 * and then start new I/O and then wait upon it. The caller must have a ref on
2979 * the buffer_head.
2980 */
2982 {
2996 }
2998 }
3002 {
3004 }
3007 /*
3008 * try_to_free_buffers() checks if all the buffers on this particular page
3009 * are unused, and releases them if so.
3010 *
3011 * Exclusion against try_to_free_buffers may be obtained by either
3012 * locking the page or by holding its mapping's private_lock.
3013 *
3014 * If the page is dirty but all the buffers are clean then we need to
3015 * be sure to mark the page clean as well. This is because the page
3016 * may be against a block device, and a later reattachment of buffers
3017 * to a dirty page will set *all* buffers dirty. Which would corrupt
3018 * filesystem data on the same device.
3019 *
3020 * The same applies to regular filesystem pages: if all the buffers are
3021 * clean then we set the page clean and proceed. To do that, we require
3022 * total exclusion from __set_page_dirty_buffers(). That is obtained with
3023 * private_lock.
3024 *
3025 * try_to_free_buffers() is non-blocking.
3026 */
3028 {
3031 }
3033 static int
3035 {
3058 failed:
3060 }
3063 {
3075 }
3080 /*
3081 * If the filesystem writes its buffers by hand (eg ext3)
3082 * then we can have clean buffers against a dirty page. We
3083 * clean the page here; otherwise the VM will never notice
3084 * that the filesystem did any IO at all.
3085 *
3086 * Also, during truncate, discard_buffer will have marked all
3087 * the page's buffers clean. We discover that here and clean
3088 * the page also.
3089 *
3090 * private_lock must be held over this entire operation in order
3091 * to synchronise against __set_page_dirty_buffers and prevent the
3092 * dirty bit from being lost.
3093 */
3097 out:
3106 }
3108 }
3111 /*
3112 * There are no bdflush tunables left. But distributions are
3113 * still running obsolete flush daemons, so we terminate them here.
3114 *
3115 * Use of bdflush() is deprecated and will be removed in a future kernel.
3116 * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3117 */
3119 {
3126 msg_count++;
3128 "warning: process `%s' used the obsolete bdflush"
3131 }
3136 }
3138 /*
3139 * Buffer-head allocation
3140 */
3143 /*
3144 * Once the number of bh's in the machine exceeds this level, we start
3145 * stripping them in writeback.
3146 */
3154 };
3159 {
3169 }
3172 {
3180 }
3182 }
3186 {
3193 }
3197 {
3204 }
3207 }
3211 {
3215 }
3217 /**
3218 * bh_uptodate_or_lock - Test whether the buffer is uptodate
3219 * @bh: struct buffer_head
3220 *
3221 * Return true if the buffer is up-to-date and false,
3222 * with the buffer locked, if not.
3223 */
3225 {
3231 }
3233 }
3236 /**
3237 * bh_submit_read - Submit a locked buffer for reading
3238 * @bh: struct buffer_head
3239 *
3240 * Returns zero on success and -EIO on error.
3241 */
3243 {
3249 }
3258 }
3262 {
3268 SLAB_MEM_SPREAD),
3269 NULL);
3271 /*
3272 * Limit the bh occupancy to 10% of ZONE_NORMAL
3273 */
3277 }