4 * Copyright (C) 1994-1999 Linus Torvalds
8 * This file handles the generic file mmap semantics used by
9 * most "normal" filesystems (but you don't /have/ to use this:
10 * the NFS filesystem used to do this differently, for example)
12 #include <linux/export.h>
13 #include <linux/compiler.h>
15 #include <linux/uaccess.h>
16 #include <linux/capability.h>
17 #include <linux/kernel_stat.h>
18 #include <linux/gfp.h>
20 #include <linux/swap.h>
21 #include <linux/mman.h>
22 #include <linux/pagemap.h>
23 #include <linux/file.h>
24 #include <linux/uio.h>
25 #include <linux/hash.h>
26 #include <linux/writeback.h>
27 #include <linux/backing-dev.h>
28 #include <linux/pagevec.h>
29 #include <linux/blkdev.h>
30 #include <linux/security.h>
31 #include <linux/cpuset.h>
32 #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
33 #include <linux/hugetlb.h>
34 #include <linux/memcontrol.h>
35 #include <linux/cleancache.h>
36 #include <linux/rmap.h>
39 #define CREATE_TRACE_POINTS
40 #include <trace/events/filemap.h>
43 * FIXME: remove all knowledge of the buffer layer from the core VM
45 #include <linux/buffer_head.h> /* for try_to_free_buffers */
50 * Shared mappings implemented 30.11.1994. It's not fully working yet,
53 * Shared mappings now work. 15.8.1995 Bruno.
55 * finished 'unifying' the page and buffer cache and SMP-threaded the
56 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
58 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
64 * ->i_mmap_rwsem (truncate_pagecache)
65 * ->private_lock (__free_pte->__set_page_dirty_buffers)
66 * ->swap_lock (exclusive_swap_page, others)
67 * ->mapping->tree_lock
70 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
74 * ->page_table_lock or pte_lock (various, mainly in memory.c)
75 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
78 * ->lock_page (access_process_vm)
80 * ->i_mutex (generic_perform_write)
81 * ->mmap_sem (fault_in_pages_readable->do_page_fault)
84 * sb_lock (fs/fs-writeback.c)
85 * ->mapping->tree_lock (__sync_single_inode)
88 * ->anon_vma.lock (vma_adjust)
91 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
93 * ->page_table_lock or pte_lock
94 * ->swap_lock (try_to_unmap_one)
95 * ->private_lock (try_to_unmap_one)
96 * ->tree_lock (try_to_unmap_one)
97 * ->zone.lru_lock (follow_page->mark_page_accessed)
98 * ->zone.lru_lock (check_pte_range->isolate_lru_page)
99 * ->private_lock (page_remove_rmap->set_page_dirty)
100 * ->tree_lock (page_remove_rmap->set_page_dirty)
101 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
102 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
103 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
104 * ->inode->i_lock (zap_pte_range->set_page_dirty)
105 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
108 * ->tasklist_lock (memory_failure, collect_procs_ao)
111 static void page_cache_tree_delete(struct address_space
*mapping
,
112 struct page
*page
, void *shadow
)
114 struct radix_tree_node
*node
;
120 VM_BUG_ON(!PageLocked(page
));
122 __radix_tree_lookup(&mapping
->page_tree
, page
->index
, &node
, &slot
);
125 mapping
->nrshadows
++;
127 * Make sure the nrshadows update is committed before
128 * the nrpages update so that final truncate racing
129 * with reclaim does not see both counters 0 at the
130 * same time and miss a shadow entry.
137 /* Clear direct pointer tags in root node */
138 mapping
->page_tree
.gfp_mask
&= __GFP_BITS_MASK
;
139 radix_tree_replace_slot(slot
, shadow
);
143 /* Clear tree tags for the removed page */
145 offset
= index
& RADIX_TREE_MAP_MASK
;
146 for (tag
= 0; tag
< RADIX_TREE_MAX_TAGS
; tag
++) {
147 if (test_bit(offset
, node
->tags
[tag
]))
148 radix_tree_tag_clear(&mapping
->page_tree
, index
, tag
);
151 /* Delete page, swap shadow entry */
152 radix_tree_replace_slot(slot
, shadow
);
153 workingset_node_pages_dec(node
);
155 workingset_node_shadows_inc(node
);
157 if (__radix_tree_delete_node(&mapping
->page_tree
, node
))
161 * Track node that only contains shadow entries.
163 * Avoid acquiring the list_lru lock if already tracked. The
164 * list_empty() test is safe as node->private_list is
165 * protected by mapping->tree_lock.
167 if (!workingset_node_pages(node
) &&
168 list_empty(&node
->private_list
)) {
169 node
->private_data
= mapping
;
170 list_lru_add(&workingset_shadow_nodes
, &node
->private_list
);
175 * Delete a page from the page cache and free it. Caller has to make
176 * sure the page is locked and that nobody else uses it - or that usage
177 * is safe. The caller must hold the mapping's tree_lock.
179 void __delete_from_page_cache(struct page
*page
, void *shadow
)
181 struct address_space
*mapping
= page
->mapping
;
183 trace_mm_filemap_delete_from_page_cache(page
);
185 * if we're uptodate, flush out into the cleancache, otherwise
186 * invalidate any existing cleancache entries. We can't leave
187 * stale data around in the cleancache once our page is gone
189 if (PageUptodate(page
) && PageMappedToDisk(page
))
190 cleancache_put_page(page
);
192 cleancache_invalidate_page(mapping
, page
);
194 page_cache_tree_delete(mapping
, page
, shadow
);
196 page
->mapping
= NULL
;
197 /* Leave page->index set: truncation lookup relies upon it */
199 /* hugetlb pages do not participate in page cache accounting. */
201 __dec_zone_page_state(page
, NR_FILE_PAGES
);
202 if (PageSwapBacked(page
))
203 __dec_zone_page_state(page
, NR_SHMEM
);
204 BUG_ON(page_mapped(page
));
207 * At this point page must be either written or cleaned by truncate.
208 * Dirty page here signals a bug and loss of unwritten data.
210 * This fixes dirty accounting after removing the page entirely but
211 * leaves PageDirty set: it has no effect for truncated page and
212 * anyway will be cleared before returning page into buddy allocator.
214 if (WARN_ON_ONCE(PageDirty(page
)))
215 account_page_cleaned(page
, mapping
);
219 * delete_from_page_cache - delete page from page cache
220 * @page: the page which the kernel is trying to remove from page cache
222 * This must be called only on pages that have been verified to be in the page
223 * cache and locked. It will never put the page into the free list, the caller
224 * has a reference on the page.
226 void delete_from_page_cache(struct page
*page
)
228 struct address_space
*mapping
= page
->mapping
;
229 void (*freepage
)(struct page
*);
231 BUG_ON(!PageLocked(page
));
233 freepage
= mapping
->a_ops
->freepage
;
234 spin_lock_irq(&mapping
->tree_lock
);
235 __delete_from_page_cache(page
, NULL
);
236 spin_unlock_irq(&mapping
->tree_lock
);
240 page_cache_release(page
);
242 EXPORT_SYMBOL(delete_from_page_cache
);
244 static int filemap_check_errors(struct address_space
*mapping
)
247 /* Check for outstanding write errors */
248 if (test_bit(AS_ENOSPC
, &mapping
->flags
) &&
249 test_and_clear_bit(AS_ENOSPC
, &mapping
->flags
))
251 if (test_bit(AS_EIO
, &mapping
->flags
) &&
252 test_and_clear_bit(AS_EIO
, &mapping
->flags
))
258 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
259 * @mapping: address space structure to write
260 * @start: offset in bytes where the range starts
261 * @end: offset in bytes where the range ends (inclusive)
262 * @sync_mode: enable synchronous operation
264 * Start writeback against all of a mapping's dirty pages that lie
265 * within the byte offsets <start, end> inclusive.
267 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
268 * opposed to a regular memory cleansing writeback. The difference between
269 * these two operations is that if a dirty page/buffer is encountered, it must
270 * be waited upon, and not just skipped over.
272 int __filemap_fdatawrite_range(struct address_space
*mapping
, loff_t start
,
273 loff_t end
, int sync_mode
)
276 struct writeback_control wbc
= {
277 .sync_mode
= sync_mode
,
278 .nr_to_write
= LONG_MAX
,
279 .range_start
= start
,
283 if (!mapping_cap_writeback_dirty(mapping
))
286 ret
= do_writepages(mapping
, &wbc
);
290 static inline int __filemap_fdatawrite(struct address_space
*mapping
,
293 return __filemap_fdatawrite_range(mapping
, 0, LLONG_MAX
, sync_mode
);
296 int filemap_fdatawrite(struct address_space
*mapping
)
298 return __filemap_fdatawrite(mapping
, WB_SYNC_ALL
);
300 EXPORT_SYMBOL(filemap_fdatawrite
);
302 int filemap_fdatawrite_range(struct address_space
*mapping
, loff_t start
,
305 return __filemap_fdatawrite_range(mapping
, start
, end
, WB_SYNC_ALL
);
307 EXPORT_SYMBOL(filemap_fdatawrite_range
);
310 * filemap_flush - mostly a non-blocking flush
311 * @mapping: target address_space
313 * This is a mostly non-blocking flush. Not suitable for data-integrity
314 * purposes - I/O may not be started against all dirty pages.
316 int filemap_flush(struct address_space
*mapping
)
318 return __filemap_fdatawrite(mapping
, WB_SYNC_NONE
);
320 EXPORT_SYMBOL(filemap_flush
);
323 * filemap_fdatawait_range - wait for writeback to complete
324 * @mapping: address space structure to wait for
325 * @start_byte: offset in bytes where the range starts
326 * @end_byte: offset in bytes where the range ends (inclusive)
328 * Walk the list of under-writeback pages of the given address space
329 * in the given range and wait for all of them.
331 int filemap_fdatawait_range(struct address_space
*mapping
, loff_t start_byte
,
334 pgoff_t index
= start_byte
>> PAGE_CACHE_SHIFT
;
335 pgoff_t end
= end_byte
>> PAGE_CACHE_SHIFT
;
340 if (end_byte
< start_byte
)
343 pagevec_init(&pvec
, 0);
344 while ((index
<= end
) &&
345 (nr_pages
= pagevec_lookup_tag(&pvec
, mapping
, &index
,
346 PAGECACHE_TAG_WRITEBACK
,
347 min(end
- index
, (pgoff_t
)PAGEVEC_SIZE
-1) + 1)) != 0) {
350 for (i
= 0; i
< nr_pages
; i
++) {
351 struct page
*page
= pvec
.pages
[i
];
353 /* until radix tree lookup accepts end_index */
354 if (page
->index
> end
)
357 wait_on_page_writeback(page
);
358 if (TestClearPageError(page
))
361 pagevec_release(&pvec
);
365 ret2
= filemap_check_errors(mapping
);
371 EXPORT_SYMBOL(filemap_fdatawait_range
);
374 * filemap_fdatawait - wait for all under-writeback pages to complete
375 * @mapping: address space structure to wait for
377 * Walk the list of under-writeback pages of the given address space
378 * and wait for all of them.
380 int filemap_fdatawait(struct address_space
*mapping
)
382 loff_t i_size
= i_size_read(mapping
->host
);
387 return filemap_fdatawait_range(mapping
, 0, i_size
- 1);
389 EXPORT_SYMBOL(filemap_fdatawait
);
391 int filemap_write_and_wait(struct address_space
*mapping
)
395 if (mapping
->nrpages
) {
396 err
= filemap_fdatawrite(mapping
);
398 * Even if the above returned error, the pages may be
399 * written partially (e.g. -ENOSPC), so we wait for it.
400 * But the -EIO is special case, it may indicate the worst
401 * thing (e.g. bug) happened, so we avoid waiting for it.
404 int err2
= filemap_fdatawait(mapping
);
409 err
= filemap_check_errors(mapping
);
413 EXPORT_SYMBOL(filemap_write_and_wait
);
416 * filemap_write_and_wait_range - write out & wait on a file range
417 * @mapping: the address_space for the pages
418 * @lstart: offset in bytes where the range starts
419 * @lend: offset in bytes where the range ends (inclusive)
421 * Write out and wait upon file offsets lstart->lend, inclusive.
423 * Note that `lend' is inclusive (describes the last byte to be written) so
424 * that this function can be used to write to the very end-of-file (end = -1).
426 int filemap_write_and_wait_range(struct address_space
*mapping
,
427 loff_t lstart
, loff_t lend
)
431 if (mapping
->nrpages
) {
432 err
= __filemap_fdatawrite_range(mapping
, lstart
, lend
,
434 /* See comment of filemap_write_and_wait() */
436 int err2
= filemap_fdatawait_range(mapping
,
442 err
= filemap_check_errors(mapping
);
446 EXPORT_SYMBOL(filemap_write_and_wait_range
);
449 * replace_page_cache_page - replace a pagecache page with a new one
450 * @old: page to be replaced
451 * @new: page to replace with
452 * @gfp_mask: allocation mode
454 * This function replaces a page in the pagecache with a new one. On
455 * success it acquires the pagecache reference for the new page and
456 * drops it for the old page. Both the old and new pages must be
457 * locked. This function does not add the new page to the LRU, the
458 * caller must do that.
460 * The remove + add is atomic. The only way this function can fail is
461 * memory allocation failure.
463 int replace_page_cache_page(struct page
*old
, struct page
*new, gfp_t gfp_mask
)
467 VM_BUG_ON_PAGE(!PageLocked(old
), old
);
468 VM_BUG_ON_PAGE(!PageLocked(new), new);
469 VM_BUG_ON_PAGE(new->mapping
, new);
471 error
= radix_tree_preload(gfp_mask
& ~__GFP_HIGHMEM
);
473 struct address_space
*mapping
= old
->mapping
;
474 void (*freepage
)(struct page
*);
476 pgoff_t offset
= old
->index
;
477 freepage
= mapping
->a_ops
->freepage
;
480 new->mapping
= mapping
;
483 spin_lock_irq(&mapping
->tree_lock
);
484 __delete_from_page_cache(old
, NULL
);
485 error
= radix_tree_insert(&mapping
->page_tree
, offset
, new);
490 * hugetlb pages do not participate in page cache accounting.
493 __inc_zone_page_state(new, NR_FILE_PAGES
);
494 if (PageSwapBacked(new))
495 __inc_zone_page_state(new, NR_SHMEM
);
496 spin_unlock_irq(&mapping
->tree_lock
);
497 mem_cgroup_migrate(old
, new, true);
498 radix_tree_preload_end();
501 page_cache_release(old
);
506 EXPORT_SYMBOL_GPL(replace_page_cache_page
);
508 static int page_cache_tree_insert(struct address_space
*mapping
,
509 struct page
*page
, void **shadowp
)
511 struct radix_tree_node
*node
;
515 error
= __radix_tree_create(&mapping
->page_tree
, page
->index
,
522 p
= radix_tree_deref_slot_protected(slot
, &mapping
->tree_lock
);
523 if (!radix_tree_exceptional_entry(p
))
527 mapping
->nrshadows
--;
529 workingset_node_shadows_dec(node
);
531 radix_tree_replace_slot(slot
, page
);
534 workingset_node_pages_inc(node
);
536 * Don't track node that contains actual pages.
538 * Avoid acquiring the list_lru lock if already
539 * untracked. The list_empty() test is safe as
540 * node->private_list is protected by
541 * mapping->tree_lock.
543 if (!list_empty(&node
->private_list
))
544 list_lru_del(&workingset_shadow_nodes
,
545 &node
->private_list
);
550 static int __add_to_page_cache_locked(struct page
*page
,
551 struct address_space
*mapping
,
552 pgoff_t offset
, gfp_t gfp_mask
,
555 int huge
= PageHuge(page
);
556 struct mem_cgroup
*memcg
;
559 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
560 VM_BUG_ON_PAGE(PageSwapBacked(page
), page
);
563 error
= mem_cgroup_try_charge(page
, current
->mm
,
569 error
= radix_tree_maybe_preload(gfp_mask
& ~__GFP_HIGHMEM
);
572 mem_cgroup_cancel_charge(page
, memcg
);
576 page_cache_get(page
);
577 page
->mapping
= mapping
;
578 page
->index
= offset
;
580 spin_lock_irq(&mapping
->tree_lock
);
581 error
= page_cache_tree_insert(mapping
, page
, shadowp
);
582 radix_tree_preload_end();
586 /* hugetlb pages do not participate in page cache accounting. */
588 __inc_zone_page_state(page
, NR_FILE_PAGES
);
589 spin_unlock_irq(&mapping
->tree_lock
);
591 mem_cgroup_commit_charge(page
, memcg
, false);
592 trace_mm_filemap_add_to_page_cache(page
);
595 page
->mapping
= NULL
;
596 /* Leave page->index set: truncation relies upon it */
597 spin_unlock_irq(&mapping
->tree_lock
);
599 mem_cgroup_cancel_charge(page
, memcg
);
600 page_cache_release(page
);
605 * add_to_page_cache_locked - add a locked page to the pagecache
607 * @mapping: the page's address_space
608 * @offset: page index
609 * @gfp_mask: page allocation mode
611 * This function is used to add a page to the pagecache. It must be locked.
612 * This function does not add the page to the LRU. The caller must do that.
614 int add_to_page_cache_locked(struct page
*page
, struct address_space
*mapping
,
615 pgoff_t offset
, gfp_t gfp_mask
)
617 return __add_to_page_cache_locked(page
, mapping
, offset
,
620 EXPORT_SYMBOL(add_to_page_cache_locked
);
622 int add_to_page_cache_lru(struct page
*page
, struct address_space
*mapping
,
623 pgoff_t offset
, gfp_t gfp_mask
)
628 __set_page_locked(page
);
629 ret
= __add_to_page_cache_locked(page
, mapping
, offset
,
632 __clear_page_locked(page
);
635 * The page might have been evicted from cache only
636 * recently, in which case it should be activated like
637 * any other repeatedly accessed page.
639 if (shadow
&& workingset_refault(shadow
)) {
641 workingset_activation(page
);
643 ClearPageActive(page
);
648 EXPORT_SYMBOL_GPL(add_to_page_cache_lru
);
651 struct page
*__page_cache_alloc(gfp_t gfp
)
656 if (cpuset_do_page_mem_spread()) {
657 unsigned int cpuset_mems_cookie
;
659 cpuset_mems_cookie
= read_mems_allowed_begin();
660 n
= cpuset_mem_spread_node();
661 page
= alloc_pages_exact_node(n
, gfp
, 0);
662 } while (!page
&& read_mems_allowed_retry(cpuset_mems_cookie
));
666 return alloc_pages(gfp
, 0);
668 EXPORT_SYMBOL(__page_cache_alloc
);
672 * In order to wait for pages to become available there must be
673 * waitqueues associated with pages. By using a hash table of
674 * waitqueues where the bucket discipline is to maintain all
675 * waiters on the same queue and wake all when any of the pages
676 * become available, and for the woken contexts to check to be
677 * sure the appropriate page became available, this saves space
678 * at a cost of "thundering herd" phenomena during rare hash
681 wait_queue_head_t
*page_waitqueue(struct page
*page
)
683 const struct zone
*zone
= page_zone(page
);
685 return &zone
->wait_table
[hash_ptr(page
, zone
->wait_table_bits
)];
687 EXPORT_SYMBOL(page_waitqueue
);
689 void wait_on_page_bit(struct page
*page
, int bit_nr
)
691 DEFINE_WAIT_BIT(wait
, &page
->flags
, bit_nr
);
693 if (test_bit(bit_nr
, &page
->flags
))
694 __wait_on_bit(page_waitqueue(page
), &wait
, bit_wait_io
,
695 TASK_UNINTERRUPTIBLE
);
697 EXPORT_SYMBOL(wait_on_page_bit
);
699 int wait_on_page_bit_killable(struct page
*page
, int bit_nr
)
701 DEFINE_WAIT_BIT(wait
, &page
->flags
, bit_nr
);
703 if (!test_bit(bit_nr
, &page
->flags
))
706 return __wait_on_bit(page_waitqueue(page
), &wait
,
707 bit_wait_io
, TASK_KILLABLE
);
710 int wait_on_page_bit_killable_timeout(struct page
*page
,
711 int bit_nr
, unsigned long timeout
)
713 DEFINE_WAIT_BIT(wait
, &page
->flags
, bit_nr
);
715 wait
.key
.timeout
= jiffies
+ timeout
;
716 if (!test_bit(bit_nr
, &page
->flags
))
718 return __wait_on_bit(page_waitqueue(page
), &wait
,
719 bit_wait_io_timeout
, TASK_KILLABLE
);
721 EXPORT_SYMBOL_GPL(wait_on_page_bit_killable_timeout
);
724 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
725 * @page: Page defining the wait queue of interest
726 * @waiter: Waiter to add to the queue
728 * Add an arbitrary @waiter to the wait queue for the nominated @page.
730 void add_page_wait_queue(struct page
*page
, wait_queue_t
*waiter
)
732 wait_queue_head_t
*q
= page_waitqueue(page
);
735 spin_lock_irqsave(&q
->lock
, flags
);
736 __add_wait_queue(q
, waiter
);
737 spin_unlock_irqrestore(&q
->lock
, flags
);
739 EXPORT_SYMBOL_GPL(add_page_wait_queue
);
742 * unlock_page - unlock a locked page
745 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
746 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
747 * mechanism between PageLocked pages and PageWriteback pages is shared.
748 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
750 * The mb is necessary to enforce ordering between the clear_bit and the read
751 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
753 void unlock_page(struct page
*page
)
755 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
756 clear_bit_unlock(PG_locked
, &page
->flags
);
757 smp_mb__after_atomic();
758 wake_up_page(page
, PG_locked
);
760 EXPORT_SYMBOL(unlock_page
);
763 * end_page_writeback - end writeback against a page
766 void end_page_writeback(struct page
*page
)
769 * TestClearPageReclaim could be used here but it is an atomic
770 * operation and overkill in this particular case. Failing to
771 * shuffle a page marked for immediate reclaim is too mild to
772 * justify taking an atomic operation penalty at the end of
773 * ever page writeback.
775 if (PageReclaim(page
)) {
776 ClearPageReclaim(page
);
777 rotate_reclaimable_page(page
);
780 if (!test_clear_page_writeback(page
))
783 smp_mb__after_atomic();
784 wake_up_page(page
, PG_writeback
);
786 EXPORT_SYMBOL(end_page_writeback
);
789 * After completing I/O on a page, call this routine to update the page
790 * flags appropriately
792 void page_endio(struct page
*page
, int rw
, int err
)
796 SetPageUptodate(page
);
798 ClearPageUptodate(page
);
802 } else { /* rw == WRITE */
806 mapping_set_error(page
->mapping
, err
);
808 end_page_writeback(page
);
811 EXPORT_SYMBOL_GPL(page_endio
);
814 * __lock_page - get a lock on the page, assuming we need to sleep to get it
815 * @page: the page to lock
817 void __lock_page(struct page
*page
)
819 DEFINE_WAIT_BIT(wait
, &page
->flags
, PG_locked
);
821 __wait_on_bit_lock(page_waitqueue(page
), &wait
, bit_wait_io
,
822 TASK_UNINTERRUPTIBLE
);
824 EXPORT_SYMBOL(__lock_page
);
826 int __lock_page_killable(struct page
*page
)
828 DEFINE_WAIT_BIT(wait
, &page
->flags
, PG_locked
);
830 return __wait_on_bit_lock(page_waitqueue(page
), &wait
,
831 bit_wait_io
, TASK_KILLABLE
);
833 EXPORT_SYMBOL_GPL(__lock_page_killable
);
837 * 1 - page is locked; mmap_sem is still held.
838 * 0 - page is not locked.
839 * mmap_sem has been released (up_read()), unless flags had both
840 * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
841 * which case mmap_sem is still held.
843 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
844 * with the page locked and the mmap_sem unperturbed.
846 int __lock_page_or_retry(struct page
*page
, struct mm_struct
*mm
,
849 if (flags
& FAULT_FLAG_ALLOW_RETRY
) {
851 * CAUTION! In this case, mmap_sem is not released
852 * even though return 0.
854 if (flags
& FAULT_FLAG_RETRY_NOWAIT
)
857 up_read(&mm
->mmap_sem
);
858 if (flags
& FAULT_FLAG_KILLABLE
)
859 wait_on_page_locked_killable(page
);
861 wait_on_page_locked(page
);
864 if (flags
& FAULT_FLAG_KILLABLE
) {
867 ret
= __lock_page_killable(page
);
869 up_read(&mm
->mmap_sem
);
879 * page_cache_next_hole - find the next hole (not-present entry)
882 * @max_scan: maximum range to search
884 * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
885 * lowest indexed hole.
887 * Returns: the index of the hole if found, otherwise returns an index
888 * outside of the set specified (in which case 'return - index >=
889 * max_scan' will be true). In rare cases of index wrap-around, 0 will
892 * page_cache_next_hole may be called under rcu_read_lock. However,
893 * like radix_tree_gang_lookup, this will not atomically search a
894 * snapshot of the tree at a single point in time. For example, if a
895 * hole is created at index 5, then subsequently a hole is created at
896 * index 10, page_cache_next_hole covering both indexes may return 10
897 * if called under rcu_read_lock.
899 pgoff_t
page_cache_next_hole(struct address_space
*mapping
,
900 pgoff_t index
, unsigned long max_scan
)
904 for (i
= 0; i
< max_scan
; i
++) {
907 page
= radix_tree_lookup(&mapping
->page_tree
, index
);
908 if (!page
|| radix_tree_exceptional_entry(page
))
917 EXPORT_SYMBOL(page_cache_next_hole
);
920 * page_cache_prev_hole - find the prev hole (not-present entry)
923 * @max_scan: maximum range to search
925 * Search backwards in the range [max(index-max_scan+1, 0), index] for
928 * Returns: the index of the hole if found, otherwise returns an index
929 * outside of the set specified (in which case 'index - return >=
930 * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
933 * page_cache_prev_hole may be called under rcu_read_lock. However,
934 * like radix_tree_gang_lookup, this will not atomically search a
935 * snapshot of the tree at a single point in time. For example, if a
936 * hole is created at index 10, then subsequently a hole is created at
937 * index 5, page_cache_prev_hole covering both indexes may return 5 if
938 * called under rcu_read_lock.
940 pgoff_t
page_cache_prev_hole(struct address_space
*mapping
,
941 pgoff_t index
, unsigned long max_scan
)
945 for (i
= 0; i
< max_scan
; i
++) {
948 page
= radix_tree_lookup(&mapping
->page_tree
, index
);
949 if (!page
|| radix_tree_exceptional_entry(page
))
952 if (index
== ULONG_MAX
)
958 EXPORT_SYMBOL(page_cache_prev_hole
);
961 * find_get_entry - find and get a page cache entry
962 * @mapping: the address_space to search
963 * @offset: the page cache index
965 * Looks up the page cache slot at @mapping & @offset. If there is a
966 * page cache page, it is returned with an increased refcount.
968 * If the slot holds a shadow entry of a previously evicted page, or a
969 * swap entry from shmem/tmpfs, it is returned.
971 * Otherwise, %NULL is returned.
973 struct page
*find_get_entry(struct address_space
*mapping
, pgoff_t offset
)
981 pagep
= radix_tree_lookup_slot(&mapping
->page_tree
, offset
);
983 page
= radix_tree_deref_slot(pagep
);
986 if (radix_tree_exception(page
)) {
987 if (radix_tree_deref_retry(page
))
990 * A shadow entry of a recently evicted page,
991 * or a swap entry from shmem/tmpfs. Return
992 * it without attempting to raise page count.
996 if (!page_cache_get_speculative(page
))
1000 * Has the page moved?
1001 * This is part of the lockless pagecache protocol. See
1002 * include/linux/pagemap.h for details.
1004 if (unlikely(page
!= *pagep
)) {
1005 page_cache_release(page
);
1014 EXPORT_SYMBOL(find_get_entry
);
1017 * find_lock_entry - locate, pin and lock a page cache entry
1018 * @mapping: the address_space to search
1019 * @offset: the page cache index
1021 * Looks up the page cache slot at @mapping & @offset. If there is a
1022 * page cache page, it is returned locked and with an increased
1025 * If the slot holds a shadow entry of a previously evicted page, or a
1026 * swap entry from shmem/tmpfs, it is returned.
1028 * Otherwise, %NULL is returned.
1030 * find_lock_entry() may sleep.
1032 struct page
*find_lock_entry(struct address_space
*mapping
, pgoff_t offset
)
1037 page
= find_get_entry(mapping
, offset
);
1038 if (page
&& !radix_tree_exception(page
)) {
1040 /* Has the page been truncated? */
1041 if (unlikely(page
->mapping
!= mapping
)) {
1043 page_cache_release(page
);
1046 VM_BUG_ON_PAGE(page
->index
!= offset
, page
);
1050 EXPORT_SYMBOL(find_lock_entry
);
1053 * pagecache_get_page - find and get a page reference
1054 * @mapping: the address_space to search
1055 * @offset: the page index
1056 * @fgp_flags: PCG flags
1057 * @gfp_mask: gfp mask to use for the page cache data page allocation
1059 * Looks up the page cache slot at @mapping & @offset.
1061 * PCG flags modify how the page is returned.
1063 * FGP_ACCESSED: the page will be marked accessed
1064 * FGP_LOCK: Page is return locked
1065 * FGP_CREAT: If page is not present then a new page is allocated using
1066 * @gfp_mask and added to the page cache and the VM's LRU
1067 * list. The page is returned locked and with an increased
1068 * refcount. Otherwise, %NULL is returned.
1070 * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even
1071 * if the GFP flags specified for FGP_CREAT are atomic.
1073 * If there is a page cache page, it is returned with an increased refcount.
1075 struct page
*pagecache_get_page(struct address_space
*mapping
, pgoff_t offset
,
1076 int fgp_flags
, gfp_t gfp_mask
)
1081 page
= find_get_entry(mapping
, offset
);
1082 if (radix_tree_exceptional_entry(page
))
1087 if (fgp_flags
& FGP_LOCK
) {
1088 if (fgp_flags
& FGP_NOWAIT
) {
1089 if (!trylock_page(page
)) {
1090 page_cache_release(page
);
1097 /* Has the page been truncated? */
1098 if (unlikely(page
->mapping
!= mapping
)) {
1100 page_cache_release(page
);
1103 VM_BUG_ON_PAGE(page
->index
!= offset
, page
);
1106 if (page
&& (fgp_flags
& FGP_ACCESSED
))
1107 mark_page_accessed(page
);
1110 if (!page
&& (fgp_flags
& FGP_CREAT
)) {
1112 if ((fgp_flags
& FGP_WRITE
) && mapping_cap_account_dirty(mapping
))
1113 gfp_mask
|= __GFP_WRITE
;
1114 if (fgp_flags
& FGP_NOFS
)
1115 gfp_mask
&= ~__GFP_FS
;
1117 page
= __page_cache_alloc(gfp_mask
);
1121 if (WARN_ON_ONCE(!(fgp_flags
& FGP_LOCK
)))
1122 fgp_flags
|= FGP_LOCK
;
1124 /* Init accessed so avoid atomic mark_page_accessed later */
1125 if (fgp_flags
& FGP_ACCESSED
)
1126 __SetPageReferenced(page
);
1128 err
= add_to_page_cache_lru(page
, mapping
, offset
,
1129 gfp_mask
& GFP_RECLAIM_MASK
);
1130 if (unlikely(err
)) {
1131 page_cache_release(page
);
1140 EXPORT_SYMBOL(pagecache_get_page
);
1143 * find_get_entries - gang pagecache lookup
1144 * @mapping: The address_space to search
1145 * @start: The starting page cache index
1146 * @nr_entries: The maximum number of entries
1147 * @entries: Where the resulting entries are placed
1148 * @indices: The cache indices corresponding to the entries in @entries
1150 * find_get_entries() will search for and return a group of up to
1151 * @nr_entries entries in the mapping. The entries are placed at
1152 * @entries. find_get_entries() takes a reference against any actual
1155 * The search returns a group of mapping-contiguous page cache entries
1156 * with ascending indexes. There may be holes in the indices due to
1157 * not-present pages.
1159 * Any shadow entries of evicted pages, or swap entries from
1160 * shmem/tmpfs, are included in the returned array.
1162 * find_get_entries() returns the number of pages and shadow entries
1165 unsigned find_get_entries(struct address_space
*mapping
,
1166 pgoff_t start
, unsigned int nr_entries
,
1167 struct page
**entries
, pgoff_t
*indices
)
1170 unsigned int ret
= 0;
1171 struct radix_tree_iter iter
;
1178 radix_tree_for_each_slot(slot
, &mapping
->page_tree
, &iter
, start
) {
1181 page
= radix_tree_deref_slot(slot
);
1182 if (unlikely(!page
))
1184 if (radix_tree_exception(page
)) {
1185 if (radix_tree_deref_retry(page
))
1188 * A shadow entry of a recently evicted page,
1189 * or a swap entry from shmem/tmpfs. Return
1190 * it without attempting to raise page count.
1194 if (!page_cache_get_speculative(page
))
1197 /* Has the page moved? */
1198 if (unlikely(page
!= *slot
)) {
1199 page_cache_release(page
);
1203 indices
[ret
] = iter
.index
;
1204 entries
[ret
] = page
;
1205 if (++ret
== nr_entries
)
1213 * find_get_pages - gang pagecache lookup
1214 * @mapping: The address_space to search
1215 * @start: The starting page index
1216 * @nr_pages: The maximum number of pages
1217 * @pages: Where the resulting pages are placed
1219 * find_get_pages() will search for and return a group of up to
1220 * @nr_pages pages in the mapping. The pages are placed at @pages.
1221 * find_get_pages() takes a reference against the returned pages.
1223 * The search returns a group of mapping-contiguous pages with ascending
1224 * indexes. There may be holes in the indices due to not-present pages.
1226 * find_get_pages() returns the number of pages which were found.
1228 unsigned find_get_pages(struct address_space
*mapping
, pgoff_t start
,
1229 unsigned int nr_pages
, struct page
**pages
)
1231 struct radix_tree_iter iter
;
1235 if (unlikely(!nr_pages
))
1240 radix_tree_for_each_slot(slot
, &mapping
->page_tree
, &iter
, start
) {
1243 page
= radix_tree_deref_slot(slot
);
1244 if (unlikely(!page
))
1247 if (radix_tree_exception(page
)) {
1248 if (radix_tree_deref_retry(page
)) {
1250 * Transient condition which can only trigger
1251 * when entry at index 0 moves out of or back
1252 * to root: none yet gotten, safe to restart.
1254 WARN_ON(iter
.index
);
1258 * A shadow entry of a recently evicted page,
1259 * or a swap entry from shmem/tmpfs. Skip
1265 if (!page_cache_get_speculative(page
))
1268 /* Has the page moved? */
1269 if (unlikely(page
!= *slot
)) {
1270 page_cache_release(page
);
1275 if (++ret
== nr_pages
)
1284 * find_get_pages_contig - gang contiguous pagecache lookup
1285 * @mapping: The address_space to search
1286 * @index: The starting page index
1287 * @nr_pages: The maximum number of pages
1288 * @pages: Where the resulting pages are placed
1290 * find_get_pages_contig() works exactly like find_get_pages(), except
1291 * that the returned number of pages are guaranteed to be contiguous.
1293 * find_get_pages_contig() returns the number of pages which were found.
1295 unsigned find_get_pages_contig(struct address_space
*mapping
, pgoff_t index
,
1296 unsigned int nr_pages
, struct page
**pages
)
1298 struct radix_tree_iter iter
;
1300 unsigned int ret
= 0;
1302 if (unlikely(!nr_pages
))
1307 radix_tree_for_each_contig(slot
, &mapping
->page_tree
, &iter
, index
) {
1310 page
= radix_tree_deref_slot(slot
);
1311 /* The hole, there no reason to continue */
1312 if (unlikely(!page
))
1315 if (radix_tree_exception(page
)) {
1316 if (radix_tree_deref_retry(page
)) {
1318 * Transient condition which can only trigger
1319 * when entry at index 0 moves out of or back
1320 * to root: none yet gotten, safe to restart.
1325 * A shadow entry of a recently evicted page,
1326 * or a swap entry from shmem/tmpfs. Stop
1327 * looking for contiguous pages.
1332 if (!page_cache_get_speculative(page
))
1335 /* Has the page moved? */
1336 if (unlikely(page
!= *slot
)) {
1337 page_cache_release(page
);
1342 * must check mapping and index after taking the ref.
1343 * otherwise we can get both false positives and false
1344 * negatives, which is just confusing to the caller.
1346 if (page
->mapping
== NULL
|| page
->index
!= iter
.index
) {
1347 page_cache_release(page
);
1352 if (++ret
== nr_pages
)
1358 EXPORT_SYMBOL(find_get_pages_contig
);
1361 * find_get_pages_tag - find and return pages that match @tag
1362 * @mapping: the address_space to search
1363 * @index: the starting page index
1364 * @tag: the tag index
1365 * @nr_pages: the maximum number of pages
1366 * @pages: where the resulting pages are placed
1368 * Like find_get_pages, except we only return pages which are tagged with
1369 * @tag. We update @index to index the next page for the traversal.
1371 unsigned find_get_pages_tag(struct address_space
*mapping
, pgoff_t
*index
,
1372 int tag
, unsigned int nr_pages
, struct page
**pages
)
1374 struct radix_tree_iter iter
;
1378 if (unlikely(!nr_pages
))
1383 radix_tree_for_each_tagged(slot
, &mapping
->page_tree
,
1384 &iter
, *index
, tag
) {
1387 page
= radix_tree_deref_slot(slot
);
1388 if (unlikely(!page
))
1391 if (radix_tree_exception(page
)) {
1392 if (radix_tree_deref_retry(page
)) {
1394 * Transient condition which can only trigger
1395 * when entry at index 0 moves out of or back
1396 * to root: none yet gotten, safe to restart.
1401 * A shadow entry of a recently evicted page.
1403 * Those entries should never be tagged, but
1404 * this tree walk is lockless and the tags are
1405 * looked up in bulk, one radix tree node at a
1406 * time, so there is a sizable window for page
1407 * reclaim to evict a page we saw tagged.
1414 if (!page_cache_get_speculative(page
))
1417 /* Has the page moved? */
1418 if (unlikely(page
!= *slot
)) {
1419 page_cache_release(page
);
1424 if (++ret
== nr_pages
)
1431 *index
= pages
[ret
- 1]->index
+ 1;
1435 EXPORT_SYMBOL(find_get_pages_tag
);
1438 * CD/DVDs are error prone. When a medium error occurs, the driver may fail
1439 * a _large_ part of the i/o request. Imagine the worst scenario:
1441 * ---R__________________________________________B__________
1442 * ^ reading here ^ bad block(assume 4k)
1444 * read(R) => miss => readahead(R...B) => media error => frustrating retries
1445 * => failing the whole request => read(R) => read(R+1) =>
1446 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
1447 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
1448 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
1450 * It is going insane. Fix it by quickly scaling down the readahead size.
1452 static void shrink_readahead_size_eio(struct file
*filp
,
1453 struct file_ra_state
*ra
)
1459 * do_generic_file_read - generic file read routine
1460 * @filp: the file to read
1461 * @ppos: current file position
1462 * @iter: data destination
1463 * @written: already copied
1465 * This is a generic file read routine, and uses the
1466 * mapping->a_ops->readpage() function for the actual low-level stuff.
1468 * This is really ugly. But the goto's actually try to clarify some
1469 * of the logic when it comes to error handling etc.
1471 static ssize_t
do_generic_file_read(struct file
*filp
, loff_t
*ppos
,
1472 struct iov_iter
*iter
, ssize_t written
)
1474 struct address_space
*mapping
= filp
->f_mapping
;
1475 struct inode
*inode
= mapping
->host
;
1476 struct file_ra_state
*ra
= &filp
->f_ra
;
1480 unsigned long offset
; /* offset into pagecache page */
1481 unsigned int prev_offset
;
1484 index
= *ppos
>> PAGE_CACHE_SHIFT
;
1485 prev_index
= ra
->prev_pos
>> PAGE_CACHE_SHIFT
;
1486 prev_offset
= ra
->prev_pos
& (PAGE_CACHE_SIZE
-1);
1487 last_index
= (*ppos
+ iter
->count
+ PAGE_CACHE_SIZE
-1) >> PAGE_CACHE_SHIFT
;
1488 offset
= *ppos
& ~PAGE_CACHE_MASK
;
1494 unsigned long nr
, ret
;
1498 page
= find_get_page(mapping
, index
);
1500 page_cache_sync_readahead(mapping
,
1502 index
, last_index
- index
);
1503 page
= find_get_page(mapping
, index
);
1504 if (unlikely(page
== NULL
))
1505 goto no_cached_page
;
1507 if (PageReadahead(page
)) {
1508 page_cache_async_readahead(mapping
,
1510 index
, last_index
- index
);
1512 if (!PageUptodate(page
)) {
1513 if (inode
->i_blkbits
== PAGE_CACHE_SHIFT
||
1514 !mapping
->a_ops
->is_partially_uptodate
)
1515 goto page_not_up_to_date
;
1516 if (!trylock_page(page
))
1517 goto page_not_up_to_date
;
1518 /* Did it get truncated before we got the lock? */
1520 goto page_not_up_to_date_locked
;
1521 if (!mapping
->a_ops
->is_partially_uptodate(page
,
1522 offset
, iter
->count
))
1523 goto page_not_up_to_date_locked
;
1528 * i_size must be checked after we know the page is Uptodate.
1530 * Checking i_size after the check allows us to calculate
1531 * the correct value for "nr", which means the zero-filled
1532 * part of the page is not copied back to userspace (unless
1533 * another truncate extends the file - this is desired though).
1536 isize
= i_size_read(inode
);
1537 end_index
= (isize
- 1) >> PAGE_CACHE_SHIFT
;
1538 if (unlikely(!isize
|| index
> end_index
)) {
1539 page_cache_release(page
);
1543 /* nr is the maximum number of bytes to copy from this page */
1544 nr
= PAGE_CACHE_SIZE
;
1545 if (index
== end_index
) {
1546 nr
= ((isize
- 1) & ~PAGE_CACHE_MASK
) + 1;
1548 page_cache_release(page
);
1554 /* If users can be writing to this page using arbitrary
1555 * virtual addresses, take care about potential aliasing
1556 * before reading the page on the kernel side.
1558 if (mapping_writably_mapped(mapping
))
1559 flush_dcache_page(page
);
1562 * When a sequential read accesses a page several times,
1563 * only mark it as accessed the first time.
1565 if (prev_index
!= index
|| offset
!= prev_offset
)
1566 mark_page_accessed(page
);
1570 * Ok, we have the page, and it's up-to-date, so
1571 * now we can copy it to user space...
1574 ret
= copy_page_to_iter(page
, offset
, nr
, iter
);
1576 index
+= offset
>> PAGE_CACHE_SHIFT
;
1577 offset
&= ~PAGE_CACHE_MASK
;
1578 prev_offset
= offset
;
1580 page_cache_release(page
);
1582 if (!iov_iter_count(iter
))
1590 page_not_up_to_date
:
1591 /* Get exclusive access to the page ... */
1592 error
= lock_page_killable(page
);
1593 if (unlikely(error
))
1594 goto readpage_error
;
1596 page_not_up_to_date_locked
:
1597 /* Did it get truncated before we got the lock? */
1598 if (!page
->mapping
) {
1600 page_cache_release(page
);
1604 /* Did somebody else fill it already? */
1605 if (PageUptodate(page
)) {
1612 * A previous I/O error may have been due to temporary
1613 * failures, eg. multipath errors.
1614 * PG_error will be set again if readpage fails.
1616 ClearPageError(page
);
1617 /* Start the actual read. The read will unlock the page. */
1618 error
= mapping
->a_ops
->readpage(filp
, page
);
1620 if (unlikely(error
)) {
1621 if (error
== AOP_TRUNCATED_PAGE
) {
1622 page_cache_release(page
);
1626 goto readpage_error
;
1629 if (!PageUptodate(page
)) {
1630 error
= lock_page_killable(page
);
1631 if (unlikely(error
))
1632 goto readpage_error
;
1633 if (!PageUptodate(page
)) {
1634 if (page
->mapping
== NULL
) {
1636 * invalidate_mapping_pages got it
1639 page_cache_release(page
);
1643 shrink_readahead_size_eio(filp
, ra
);
1645 goto readpage_error
;
1653 /* UHHUH! A synchronous read error occurred. Report it */
1654 page_cache_release(page
);
1659 * Ok, it wasn't cached, so we need to create a new
1662 page
= page_cache_alloc_cold(mapping
);
1667 error
= add_to_page_cache_lru(page
, mapping
, index
,
1668 GFP_KERNEL
& mapping_gfp_mask(mapping
));
1670 page_cache_release(page
);
1671 if (error
== -EEXIST
) {
1681 ra
->prev_pos
= prev_index
;
1682 ra
->prev_pos
<<= PAGE_CACHE_SHIFT
;
1683 ra
->prev_pos
|= prev_offset
;
1685 *ppos
= ((loff_t
)index
<< PAGE_CACHE_SHIFT
) + offset
;
1686 file_accessed(filp
);
1687 return written
? written
: error
;
1691 * generic_file_read_iter - generic filesystem read routine
1692 * @iocb: kernel I/O control block
1693 * @iter: destination for the data read
1695 * This is the "read_iter()" routine for all filesystems
1696 * that can use the page cache directly.
1699 generic_file_read_iter(struct kiocb
*iocb
, struct iov_iter
*iter
)
1701 struct file
*file
= iocb
->ki_filp
;
1703 loff_t
*ppos
= &iocb
->ki_pos
;
1706 if (iocb
->ki_flags
& IOCB_DIRECT
) {
1707 struct address_space
*mapping
= file
->f_mapping
;
1708 struct inode
*inode
= mapping
->host
;
1709 size_t count
= iov_iter_count(iter
);
1713 goto out
; /* skip atime */
1714 size
= i_size_read(inode
);
1715 retval
= filemap_write_and_wait_range(mapping
, pos
,
1718 struct iov_iter data
= *iter
;
1719 retval
= mapping
->a_ops
->direct_IO(iocb
, &data
, pos
);
1723 *ppos
= pos
+ retval
;
1724 iov_iter_advance(iter
, retval
);
1728 * Btrfs can have a short DIO read if we encounter
1729 * compressed extents, so if there was an error, or if
1730 * we've already read everything we wanted to, or if
1731 * there was a short read because we hit EOF, go ahead
1732 * and return. Otherwise fallthrough to buffered io for
1733 * the rest of the read. Buffered reads will not work for
1734 * DAX files, so don't bother trying.
1736 if (retval
< 0 || !iov_iter_count(iter
) || *ppos
>= size
||
1738 file_accessed(file
);
1743 retval
= do_generic_file_read(file
, ppos
, iter
, retval
);
1747 EXPORT_SYMBOL(generic_file_read_iter
);
1751 * page_cache_read - adds requested page to the page cache if not already there
1752 * @file: file to read
1753 * @offset: page index
1755 * This adds the requested page to the page cache if it isn't already there,
1756 * and schedules an I/O to read in its contents from disk.
1758 static int page_cache_read(struct file
*file
, pgoff_t offset
)
1760 struct address_space
*mapping
= file
->f_mapping
;
1765 page
= page_cache_alloc_cold(mapping
);
1769 ret
= add_to_page_cache_lru(page
, mapping
, offset
,
1770 GFP_KERNEL
& mapping_gfp_mask(mapping
));
1772 ret
= mapping
->a_ops
->readpage(file
, page
);
1773 else if (ret
== -EEXIST
)
1774 ret
= 0; /* losing race to add is OK */
1776 page_cache_release(page
);
1778 } while (ret
== AOP_TRUNCATED_PAGE
);
1783 #define MMAP_LOTSAMISS (100)
1786 * Synchronous readahead happens when we don't even find
1787 * a page in the page cache at all.
1789 static void do_sync_mmap_readahead(struct vm_area_struct
*vma
,
1790 struct file_ra_state
*ra
,
1794 unsigned long ra_pages
;
1795 struct address_space
*mapping
= file
->f_mapping
;
1797 /* If we don't want any read-ahead, don't bother */
1798 if (vma
->vm_flags
& VM_RAND_READ
)
1803 if (vma
->vm_flags
& VM_SEQ_READ
) {
1804 page_cache_sync_readahead(mapping
, ra
, file
, offset
,
1809 /* Avoid banging the cache line if not needed */
1810 if (ra
->mmap_miss
< MMAP_LOTSAMISS
* 10)
1814 * Do we miss much more than hit in this file? If so,
1815 * stop bothering with read-ahead. It will only hurt.
1817 if (ra
->mmap_miss
> MMAP_LOTSAMISS
)
1823 ra_pages
= max_sane_readahead(ra
->ra_pages
);
1824 ra
->start
= max_t(long, 0, offset
- ra_pages
/ 2);
1825 ra
->size
= ra_pages
;
1826 ra
->async_size
= ra_pages
/ 4;
1827 ra_submit(ra
, mapping
, file
);
1831 * Asynchronous readahead happens when we find the page and PG_readahead,
1832 * so we want to possibly extend the readahead further..
1834 static void do_async_mmap_readahead(struct vm_area_struct
*vma
,
1835 struct file_ra_state
*ra
,
1840 struct address_space
*mapping
= file
->f_mapping
;
1842 /* If we don't want any read-ahead, don't bother */
1843 if (vma
->vm_flags
& VM_RAND_READ
)
1845 if (ra
->mmap_miss
> 0)
1847 if (PageReadahead(page
))
1848 page_cache_async_readahead(mapping
, ra
, file
,
1849 page
, offset
, ra
->ra_pages
);
1853 * filemap_fault - read in file data for page fault handling
1854 * @vma: vma in which the fault was taken
1855 * @vmf: struct vm_fault containing details of the fault
1857 * filemap_fault() is invoked via the vma operations vector for a
1858 * mapped memory region to read in file data during a page fault.
1860 * The goto's are kind of ugly, but this streamlines the normal case of having
1861 * it in the page cache, and handles the special cases reasonably without
1862 * having a lot of duplicated code.
1864 * vma->vm_mm->mmap_sem must be held on entry.
1866 * If our return value has VM_FAULT_RETRY set, it's because
1867 * lock_page_or_retry() returned 0.
1868 * The mmap_sem has usually been released in this case.
1869 * See __lock_page_or_retry() for the exception.
1871 * If our return value does not have VM_FAULT_RETRY set, the mmap_sem
1872 * has not been released.
1874 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
1876 int filemap_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
1879 struct file
*file
= vma
->vm_file
;
1880 struct address_space
*mapping
= file
->f_mapping
;
1881 struct file_ra_state
*ra
= &file
->f_ra
;
1882 struct inode
*inode
= mapping
->host
;
1883 pgoff_t offset
= vmf
->pgoff
;
1888 size
= round_up(i_size_read(inode
), PAGE_CACHE_SIZE
);
1889 if (offset
>= size
>> PAGE_CACHE_SHIFT
)
1890 return VM_FAULT_SIGBUS
;
1893 * Do we have something in the page cache already?
1895 page
= find_get_page(mapping
, offset
);
1896 if (likely(page
) && !(vmf
->flags
& FAULT_FLAG_TRIED
)) {
1898 * We found the page, so try async readahead before
1899 * waiting for the lock.
1901 do_async_mmap_readahead(vma
, ra
, file
, page
, offset
);
1903 /* No page in the page cache at all */
1904 do_sync_mmap_readahead(vma
, ra
, file
, offset
);
1905 count_vm_event(PGMAJFAULT
);
1906 mem_cgroup_count_vm_event(vma
->vm_mm
, PGMAJFAULT
);
1907 ret
= VM_FAULT_MAJOR
;
1909 page
= find_get_page(mapping
, offset
);
1911 goto no_cached_page
;
1914 if (!lock_page_or_retry(page
, vma
->vm_mm
, vmf
->flags
)) {
1915 page_cache_release(page
);
1916 return ret
| VM_FAULT_RETRY
;
1919 /* Did it get truncated? */
1920 if (unlikely(page
->mapping
!= mapping
)) {
1925 VM_BUG_ON_PAGE(page
->index
!= offset
, page
);
1928 * We have a locked page in the page cache, now we need to check
1929 * that it's up-to-date. If not, it is going to be due to an error.
1931 if (unlikely(!PageUptodate(page
)))
1932 goto page_not_uptodate
;
1935 * Found the page and have a reference on it.
1936 * We must recheck i_size under page lock.
1938 size
= round_up(i_size_read(inode
), PAGE_CACHE_SIZE
);
1939 if (unlikely(offset
>= size
>> PAGE_CACHE_SHIFT
)) {
1941 page_cache_release(page
);
1942 return VM_FAULT_SIGBUS
;
1946 return ret
| VM_FAULT_LOCKED
;
1950 * We're only likely to ever get here if MADV_RANDOM is in
1953 error
= page_cache_read(file
, offset
);
1956 * The page we want has now been added to the page cache.
1957 * In the unlikely event that someone removed it in the
1958 * meantime, we'll just come back here and read it again.
1964 * An error return from page_cache_read can result if the
1965 * system is low on memory, or a problem occurs while trying
1968 if (error
== -ENOMEM
)
1969 return VM_FAULT_OOM
;
1970 return VM_FAULT_SIGBUS
;
1974 * Umm, take care of errors if the page isn't up-to-date.
1975 * Try to re-read it _once_. We do this synchronously,
1976 * because there really aren't any performance issues here
1977 * and we need to check for errors.
1979 ClearPageError(page
);
1980 error
= mapping
->a_ops
->readpage(file
, page
);
1982 wait_on_page_locked(page
);
1983 if (!PageUptodate(page
))
1986 page_cache_release(page
);
1988 if (!error
|| error
== AOP_TRUNCATED_PAGE
)
1991 /* Things didn't work out. Return zero to tell the mm layer so. */
1992 shrink_readahead_size_eio(file
, ra
);
1993 return VM_FAULT_SIGBUS
;
1995 EXPORT_SYMBOL(filemap_fault
);
1997 void filemap_map_pages(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
1999 struct radix_tree_iter iter
;
2001 struct file
*file
= vma
->vm_file
;
2002 struct address_space
*mapping
= file
->f_mapping
;
2005 unsigned long address
= (unsigned long) vmf
->virtual_address
;
2010 radix_tree_for_each_slot(slot
, &mapping
->page_tree
, &iter
, vmf
->pgoff
) {
2011 if (iter
.index
> vmf
->max_pgoff
)
2014 page
= radix_tree_deref_slot(slot
);
2015 if (unlikely(!page
))
2017 if (radix_tree_exception(page
)) {
2018 if (radix_tree_deref_retry(page
))
2024 if (!page_cache_get_speculative(page
))
2027 /* Has the page moved? */
2028 if (unlikely(page
!= *slot
)) {
2029 page_cache_release(page
);
2033 if (!PageUptodate(page
) ||
2034 PageReadahead(page
) ||
2037 if (!trylock_page(page
))
2040 if (page
->mapping
!= mapping
|| !PageUptodate(page
))
2043 size
= round_up(i_size_read(mapping
->host
), PAGE_CACHE_SIZE
);
2044 if (page
->index
>= size
>> PAGE_CACHE_SHIFT
)
2047 pte
= vmf
->pte
+ page
->index
- vmf
->pgoff
;
2048 if (!pte_none(*pte
))
2051 if (file
->f_ra
.mmap_miss
> 0)
2052 file
->f_ra
.mmap_miss
--;
2053 addr
= address
+ (page
->index
- vmf
->pgoff
) * PAGE_SIZE
;
2054 do_set_pte(vma
, addr
, page
, pte
, false, false);
2060 page_cache_release(page
);
2062 if (iter
.index
== vmf
->max_pgoff
)
2067 EXPORT_SYMBOL(filemap_map_pages
);
2069 int filemap_page_mkwrite(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
2071 struct page
*page
= vmf
->page
;
2072 struct inode
*inode
= file_inode(vma
->vm_file
);
2073 int ret
= VM_FAULT_LOCKED
;
2075 sb_start_pagefault(inode
->i_sb
);
2076 file_update_time(vma
->vm_file
);
2078 if (page
->mapping
!= inode
->i_mapping
) {
2080 ret
= VM_FAULT_NOPAGE
;
2084 * We mark the page dirty already here so that when freeze is in
2085 * progress, we are guaranteed that writeback during freezing will
2086 * see the dirty page and writeprotect it again.
2088 set_page_dirty(page
);
2089 wait_for_stable_page(page
);
2091 sb_end_pagefault(inode
->i_sb
);
2094 EXPORT_SYMBOL(filemap_page_mkwrite
);
2096 const struct vm_operations_struct generic_file_vm_ops
= {
2097 .fault
= filemap_fault
,
2098 .map_pages
= filemap_map_pages
,
2099 .page_mkwrite
= filemap_page_mkwrite
,
2102 /* This is used for a general mmap of a disk file */
2104 int generic_file_mmap(struct file
* file
, struct vm_area_struct
* vma
)
2106 struct address_space
*mapping
= file
->f_mapping
;
2108 if (!mapping
->a_ops
->readpage
)
2110 file_accessed(file
);
2111 vma
->vm_ops
= &generic_file_vm_ops
;
2116 * This is for filesystems which do not implement ->writepage.
2118 int generic_file_readonly_mmap(struct file
*file
, struct vm_area_struct
*vma
)
2120 if ((vma
->vm_flags
& VM_SHARED
) && (vma
->vm_flags
& VM_MAYWRITE
))
2122 return generic_file_mmap(file
, vma
);
2125 int generic_file_mmap(struct file
* file
, struct vm_area_struct
* vma
)
2129 int generic_file_readonly_mmap(struct file
* file
, struct vm_area_struct
* vma
)
2133 #endif /* CONFIG_MMU */
2135 EXPORT_SYMBOL(generic_file_mmap
);
2136 EXPORT_SYMBOL(generic_file_readonly_mmap
);
2138 static struct page
*wait_on_page_read(struct page
*page
)
2140 if (!IS_ERR(page
)) {
2141 wait_on_page_locked(page
);
2142 if (!PageUptodate(page
)) {
2143 page_cache_release(page
);
2144 page
= ERR_PTR(-EIO
);
2150 static struct page
*__read_cache_page(struct address_space
*mapping
,
2152 int (*filler
)(void *, struct page
*),
2159 page
= find_get_page(mapping
, index
);
2161 page
= __page_cache_alloc(gfp
| __GFP_COLD
);
2163 return ERR_PTR(-ENOMEM
);
2164 err
= add_to_page_cache_lru(page
, mapping
, index
, gfp
);
2165 if (unlikely(err
)) {
2166 page_cache_release(page
);
2169 /* Presumably ENOMEM for radix tree node */
2170 return ERR_PTR(err
);
2172 err
= filler(data
, page
);
2174 page_cache_release(page
);
2175 page
= ERR_PTR(err
);
2177 page
= wait_on_page_read(page
);
2183 static struct page
*do_read_cache_page(struct address_space
*mapping
,
2185 int (*filler
)(void *, struct page
*),
2194 page
= __read_cache_page(mapping
, index
, filler
, data
, gfp
);
2197 if (PageUptodate(page
))
2201 if (!page
->mapping
) {
2203 page_cache_release(page
);
2206 if (PageUptodate(page
)) {
2210 err
= filler(data
, page
);
2212 page_cache_release(page
);
2213 return ERR_PTR(err
);
2215 page
= wait_on_page_read(page
);
2220 mark_page_accessed(page
);
2225 * read_cache_page - read into page cache, fill it if needed
2226 * @mapping: the page's address_space
2227 * @index: the page index
2228 * @filler: function to perform the read
2229 * @data: first arg to filler(data, page) function, often left as NULL
2231 * Read into the page cache. If a page already exists, and PageUptodate() is
2232 * not set, try to fill the page and wait for it to become unlocked.
2234 * If the page does not get brought uptodate, return -EIO.
2236 struct page
*read_cache_page(struct address_space
*mapping
,
2238 int (*filler
)(void *, struct page
*),
2241 return do_read_cache_page(mapping
, index
, filler
, data
, mapping_gfp_mask(mapping
));
2243 EXPORT_SYMBOL(read_cache_page
);
2246 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
2247 * @mapping: the page's address_space
2248 * @index: the page index
2249 * @gfp: the page allocator flags to use if allocating
2251 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
2252 * any new page allocations done using the specified allocation flags.
2254 * If the page does not get brought uptodate, return -EIO.
2256 struct page
*read_cache_page_gfp(struct address_space
*mapping
,
2260 filler_t
*filler
= (filler_t
*)mapping
->a_ops
->readpage
;
2262 return do_read_cache_page(mapping
, index
, filler
, NULL
, gfp
);
2264 EXPORT_SYMBOL(read_cache_page_gfp
);
2267 * Performs necessary checks before doing a write
2269 * Can adjust writing position or amount of bytes to write.
2270 * Returns appropriate error code that caller should return or
2271 * zero in case that write should be allowed.
2273 inline ssize_t
generic_write_checks(struct kiocb
*iocb
, struct iov_iter
*from
)
2275 struct file
*file
= iocb
->ki_filp
;
2276 struct inode
*inode
= file
->f_mapping
->host
;
2277 unsigned long limit
= rlimit(RLIMIT_FSIZE
);
2280 if (!iov_iter_count(from
))
2283 /* FIXME: this is for backwards compatibility with 2.4 */
2284 if (iocb
->ki_flags
& IOCB_APPEND
)
2285 iocb
->ki_pos
= i_size_read(inode
);
2289 if (limit
!= RLIM_INFINITY
) {
2290 if (iocb
->ki_pos
>= limit
) {
2291 send_sig(SIGXFSZ
, current
, 0);
2294 iov_iter_truncate(from
, limit
- (unsigned long)pos
);
2300 if (unlikely(pos
+ iov_iter_count(from
) > MAX_NON_LFS
&&
2301 !(file
->f_flags
& O_LARGEFILE
))) {
2302 if (pos
>= MAX_NON_LFS
)
2304 iov_iter_truncate(from
, MAX_NON_LFS
- (unsigned long)pos
);
2308 * Are we about to exceed the fs block limit ?
2310 * If we have written data it becomes a short write. If we have
2311 * exceeded without writing data we send a signal and return EFBIG.
2312 * Linus frestrict idea will clean these up nicely..
2314 if (unlikely(pos
>= inode
->i_sb
->s_maxbytes
))
2317 iov_iter_truncate(from
, inode
->i_sb
->s_maxbytes
- pos
);
2318 return iov_iter_count(from
);
2320 EXPORT_SYMBOL(generic_write_checks
);
2322 int pagecache_write_begin(struct file
*file
, struct address_space
*mapping
,
2323 loff_t pos
, unsigned len
, unsigned flags
,
2324 struct page
**pagep
, void **fsdata
)
2326 const struct address_space_operations
*aops
= mapping
->a_ops
;
2328 return aops
->write_begin(file
, mapping
, pos
, len
, flags
,
2331 EXPORT_SYMBOL(pagecache_write_begin
);
2333 int pagecache_write_end(struct file
*file
, struct address_space
*mapping
,
2334 loff_t pos
, unsigned len
, unsigned copied
,
2335 struct page
*page
, void *fsdata
)
2337 const struct address_space_operations
*aops
= mapping
->a_ops
;
2339 return aops
->write_end(file
, mapping
, pos
, len
, copied
, page
, fsdata
);
2341 EXPORT_SYMBOL(pagecache_write_end
);
2344 generic_file_direct_write(struct kiocb
*iocb
, struct iov_iter
*from
, loff_t pos
)
2346 struct file
*file
= iocb
->ki_filp
;
2347 struct address_space
*mapping
= file
->f_mapping
;
2348 struct inode
*inode
= mapping
->host
;
2352 struct iov_iter data
;
2354 write_len
= iov_iter_count(from
);
2355 end
= (pos
+ write_len
- 1) >> PAGE_CACHE_SHIFT
;
2357 written
= filemap_write_and_wait_range(mapping
, pos
, pos
+ write_len
- 1);
2362 * After a write we want buffered reads to be sure to go to disk to get
2363 * the new data. We invalidate clean cached page from the region we're
2364 * about to write. We do this *before* the write so that we can return
2365 * without clobbering -EIOCBQUEUED from ->direct_IO().
2367 if (mapping
->nrpages
) {
2368 written
= invalidate_inode_pages2_range(mapping
,
2369 pos
>> PAGE_CACHE_SHIFT
, end
);
2371 * If a page can not be invalidated, return 0 to fall back
2372 * to buffered write.
2375 if (written
== -EBUSY
)
2382 written
= mapping
->a_ops
->direct_IO(iocb
, &data
, pos
);
2385 * Finally, try again to invalidate clean pages which might have been
2386 * cached by non-direct readahead, or faulted in by get_user_pages()
2387 * if the source of the write was an mmap'ed region of the file
2388 * we're writing. Either one is a pretty crazy thing to do,
2389 * so we don't support it 100%. If this invalidation
2390 * fails, tough, the write still worked...
2392 if (mapping
->nrpages
) {
2393 invalidate_inode_pages2_range(mapping
,
2394 pos
>> PAGE_CACHE_SHIFT
, end
);
2399 iov_iter_advance(from
, written
);
2400 if (pos
> i_size_read(inode
) && !S_ISBLK(inode
->i_mode
)) {
2401 i_size_write(inode
, pos
);
2402 mark_inode_dirty(inode
);
2409 EXPORT_SYMBOL(generic_file_direct_write
);
2412 * Find or create a page at the given pagecache position. Return the locked
2413 * page. This function is specifically for buffered writes.
2415 struct page
*grab_cache_page_write_begin(struct address_space
*mapping
,
2416 pgoff_t index
, unsigned flags
)
2419 int fgp_flags
= FGP_LOCK
|FGP_ACCESSED
|FGP_WRITE
|FGP_CREAT
;
2421 if (flags
& AOP_FLAG_NOFS
)
2422 fgp_flags
|= FGP_NOFS
;
2424 page
= pagecache_get_page(mapping
, index
, fgp_flags
,
2425 mapping_gfp_mask(mapping
));
2427 wait_for_stable_page(page
);
2431 EXPORT_SYMBOL(grab_cache_page_write_begin
);
2433 ssize_t
generic_perform_write(struct file
*file
,
2434 struct iov_iter
*i
, loff_t pos
)
2436 struct address_space
*mapping
= file
->f_mapping
;
2437 const struct address_space_operations
*a_ops
= mapping
->a_ops
;
2439 ssize_t written
= 0;
2440 unsigned int flags
= 0;
2443 * Copies from kernel address space cannot fail (NFSD is a big user).
2445 if (!iter_is_iovec(i
))
2446 flags
|= AOP_FLAG_UNINTERRUPTIBLE
;
2450 unsigned long offset
; /* Offset into pagecache page */
2451 unsigned long bytes
; /* Bytes to write to page */
2452 size_t copied
; /* Bytes copied from user */
2455 offset
= (pos
& (PAGE_CACHE_SIZE
- 1));
2456 bytes
= min_t(unsigned long, PAGE_CACHE_SIZE
- offset
,
2461 * Bring in the user page that we will copy from _first_.
2462 * Otherwise there's a nasty deadlock on copying from the
2463 * same page as we're writing to, without it being marked
2466 * Not only is this an optimisation, but it is also required
2467 * to check that the address is actually valid, when atomic
2468 * usercopies are used, below.
2470 if (unlikely(iov_iter_fault_in_readable(i
, bytes
))) {
2475 status
= a_ops
->write_begin(file
, mapping
, pos
, bytes
, flags
,
2477 if (unlikely(status
< 0))
2480 if (mapping_writably_mapped(mapping
))
2481 flush_dcache_page(page
);
2483 copied
= iov_iter_copy_from_user_atomic(page
, i
, offset
, bytes
);
2484 flush_dcache_page(page
);
2486 status
= a_ops
->write_end(file
, mapping
, pos
, bytes
, copied
,
2488 if (unlikely(status
< 0))
2494 iov_iter_advance(i
, copied
);
2495 if (unlikely(copied
== 0)) {
2497 * If we were unable to copy any data at all, we must
2498 * fall back to a single segment length write.
2500 * If we didn't fallback here, we could livelock
2501 * because not all segments in the iov can be copied at
2502 * once without a pagefault.
2504 bytes
= min_t(unsigned long, PAGE_CACHE_SIZE
- offset
,
2505 iov_iter_single_seg_count(i
));
2511 balance_dirty_pages_ratelimited(mapping
);
2512 if (fatal_signal_pending(current
)) {
2516 } while (iov_iter_count(i
));
2518 return written
? written
: status
;
2520 EXPORT_SYMBOL(generic_perform_write
);
2523 * __generic_file_write_iter - write data to a file
2524 * @iocb: IO state structure (file, offset, etc.)
2525 * @from: iov_iter with data to write
2527 * This function does all the work needed for actually writing data to a
2528 * file. It does all basic checks, removes SUID from the file, updates
2529 * modification times and calls proper subroutines depending on whether we
2530 * do direct IO or a standard buffered write.
2532 * It expects i_mutex to be grabbed unless we work on a block device or similar
2533 * object which does not need locking at all.
2535 * This function does *not* take care of syncing data in case of O_SYNC write.
2536 * A caller has to handle it. This is mainly due to the fact that we want to
2537 * avoid syncing under i_mutex.
2539 ssize_t
__generic_file_write_iter(struct kiocb
*iocb
, struct iov_iter
*from
)
2541 struct file
*file
= iocb
->ki_filp
;
2542 struct address_space
* mapping
= file
->f_mapping
;
2543 struct inode
*inode
= mapping
->host
;
2544 ssize_t written
= 0;
2548 /* We can write back this queue in page reclaim */
2549 current
->backing_dev_info
= inode_to_bdi(inode
);
2550 err
= file_remove_suid(file
);
2554 err
= file_update_time(file
);
2558 if (iocb
->ki_flags
& IOCB_DIRECT
) {
2559 loff_t pos
, endbyte
;
2561 written
= generic_file_direct_write(iocb
, from
, iocb
->ki_pos
);
2563 * If the write stopped short of completing, fall back to
2564 * buffered writes. Some filesystems do this for writes to
2565 * holes, for example. For DAX files, a buffered write will
2566 * not succeed (even if it did, DAX does not handle dirty
2567 * page-cache pages correctly).
2569 if (written
< 0 || !iov_iter_count(from
) || IS_DAX(inode
))
2572 status
= generic_perform_write(file
, from
, pos
= iocb
->ki_pos
);
2574 * If generic_perform_write() returned a synchronous error
2575 * then we want to return the number of bytes which were
2576 * direct-written, or the error code if that was zero. Note
2577 * that this differs from normal direct-io semantics, which
2578 * will return -EFOO even if some bytes were written.
2580 if (unlikely(status
< 0)) {
2585 * We need to ensure that the page cache pages are written to
2586 * disk and invalidated to preserve the expected O_DIRECT
2589 endbyte
= pos
+ status
- 1;
2590 err
= filemap_write_and_wait_range(mapping
, pos
, endbyte
);
2592 iocb
->ki_pos
= endbyte
+ 1;
2594 invalidate_mapping_pages(mapping
,
2595 pos
>> PAGE_CACHE_SHIFT
,
2596 endbyte
>> PAGE_CACHE_SHIFT
);
2599 * We don't know how much we wrote, so just return
2600 * the number of bytes which were direct-written
2604 written
= generic_perform_write(file
, from
, iocb
->ki_pos
);
2605 if (likely(written
> 0))
2606 iocb
->ki_pos
+= written
;
2609 current
->backing_dev_info
= NULL
;
2610 return written
? written
: err
;
2612 EXPORT_SYMBOL(__generic_file_write_iter
);
2615 * generic_file_write_iter - write data to a file
2616 * @iocb: IO state structure
2617 * @from: iov_iter with data to write
2619 * This is a wrapper around __generic_file_write_iter() to be used by most
2620 * filesystems. It takes care of syncing the file in case of O_SYNC file
2621 * and acquires i_mutex as needed.
2623 ssize_t
generic_file_write_iter(struct kiocb
*iocb
, struct iov_iter
*from
)
2625 struct file
*file
= iocb
->ki_filp
;
2626 struct inode
*inode
= file
->f_mapping
->host
;
2629 mutex_lock(&inode
->i_mutex
);
2630 ret
= generic_write_checks(iocb
, from
);
2632 ret
= __generic_file_write_iter(iocb
, from
);
2633 mutex_unlock(&inode
->i_mutex
);
2638 err
= generic_write_sync(file
, iocb
->ki_pos
- ret
, ret
);
2644 EXPORT_SYMBOL(generic_file_write_iter
);
2647 * try_to_release_page() - release old fs-specific metadata on a page
2649 * @page: the page which the kernel is trying to free
2650 * @gfp_mask: memory allocation flags (and I/O mode)
2652 * The address_space is to try to release any data against the page
2653 * (presumably at page->private). If the release was successful, return `1'.
2654 * Otherwise return zero.
2656 * This may also be called if PG_fscache is set on a page, indicating that the
2657 * page is known to the local caching routines.
2659 * The @gfp_mask argument specifies whether I/O may be performed to release
2660 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT & __GFP_FS).
2663 int try_to_release_page(struct page
*page
, gfp_t gfp_mask
)
2665 struct address_space
* const mapping
= page
->mapping
;
2667 BUG_ON(!PageLocked(page
));
2668 if (PageWriteback(page
))
2671 if (mapping
&& mapping
->a_ops
->releasepage
)
2672 return mapping
->a_ops
->releasepage(page
, gfp_mask
);
2673 return try_to_free_buffers(page
);
2676 EXPORT_SYMBOL(try_to_release_page
);