percpu: fix chunk range calculation
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / mm / filemap.c
blob9e0826ea7bbe759dfb2356d13bdfb038cc3e0869
1 /*
2 * linux/mm/filemap.c
4 * Copyright (C) 1994-1999 Linus Torvalds
5 */
7 /*
8 * This file handles the generic file mmap semantics used by
9 * most "normal" filesystems (but you don't /have/ to use this:
10 * the NFS filesystem used to do this differently, for example)
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/compiler.h>
15 #include <linux/fs.h>
16 #include <linux/uaccess.h>
17 #include <linux/aio.h>
18 #include <linux/capability.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/mm.h>
21 #include <linux/swap.h>
22 #include <linux/mman.h>
23 #include <linux/pagemap.h>
24 #include <linux/file.h>
25 #include <linux/uio.h>
26 #include <linux/hash.h>
27 #include <linux/writeback.h>
28 #include <linux/backing-dev.h>
29 #include <linux/pagevec.h>
30 #include <linux/blkdev.h>
31 #include <linux/security.h>
32 #include <linux/syscalls.h>
33 #include <linux/cpuset.h>
34 #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
35 #include <linux/memcontrol.h>
36 #include <linux/mm_inline.h> /* for page_is_file_cache() */
37 #include "internal.h"
40 * FIXME: remove all knowledge of the buffer layer from the core VM
42 #include <linux/buffer_head.h> /* for try_to_free_buffers */
44 #include <asm/mman.h>
47 * Shared mappings implemented 30.11.1994. It's not fully working yet,
48 * though.
50 * Shared mappings now work. 15.8.1995 Bruno.
52 * finished 'unifying' the page and buffer cache and SMP-threaded the
53 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
55 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
59 * Lock ordering:
61 * ->i_mmap_lock (truncate_pagecache)
62 * ->private_lock (__free_pte->__set_page_dirty_buffers)
63 * ->swap_lock (exclusive_swap_page, others)
64 * ->mapping->tree_lock
66 * ->i_mutex
67 * ->i_mmap_lock (truncate->unmap_mapping_range)
69 * ->mmap_sem
70 * ->i_mmap_lock
71 * ->page_table_lock or pte_lock (various, mainly in memory.c)
72 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
74 * ->mmap_sem
75 * ->lock_page (access_process_vm)
77 * ->i_mutex (generic_file_buffered_write)
78 * ->mmap_sem (fault_in_pages_readable->do_page_fault)
80 * ->i_mutex
81 * ->i_alloc_sem (various)
83 * ->inode_lock
84 * ->sb_lock (fs/fs-writeback.c)
85 * ->mapping->tree_lock (__sync_single_inode)
87 * ->i_mmap_lock
88 * ->anon_vma.lock (vma_adjust)
90 * ->anon_vma.lock
91 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
93 * ->page_table_lock or pte_lock
94 * ->swap_lock (try_to_unmap_one)
95 * ->private_lock (try_to_unmap_one)
96 * ->tree_lock (try_to_unmap_one)
97 * ->zone.lru_lock (follow_page->mark_page_accessed)
98 * ->zone.lru_lock (check_pte_range->isolate_lru_page)
99 * ->private_lock (page_remove_rmap->set_page_dirty)
100 * ->tree_lock (page_remove_rmap->set_page_dirty)
101 * ->inode_lock (page_remove_rmap->set_page_dirty)
102 * ->inode_lock (zap_pte_range->set_page_dirty)
103 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
105 * ->task->proc_lock
106 * ->dcache_lock (proc_pid_lookup)
108 * (code doesn't rely on that order, so you could switch it around)
109 * ->tasklist_lock (memory_failure, collect_procs_ao)
110 * ->i_mmap_lock
114 * Remove a page from the page cache and free it. Caller has to make
115 * sure the page is locked and that nobody else uses it - or that usage
116 * is safe. The caller must hold the mapping's tree_lock.
118 void __remove_from_page_cache(struct page *page)
120 struct address_space *mapping = page->mapping;
122 radix_tree_delete(&mapping->page_tree, page->index);
123 page->mapping = NULL;
124 mapping->nrpages--;
125 __dec_zone_page_state(page, NR_FILE_PAGES);
126 if (PageSwapBacked(page))
127 __dec_zone_page_state(page, NR_SHMEM);
128 BUG_ON(page_mapped(page));
131 * Some filesystems seem to re-dirty the page even after
132 * the VM has canceled the dirty bit (eg ext3 journaling).
134 * Fix it up by doing a final dirty accounting check after
135 * having removed the page entirely.
137 if (PageDirty(page) && mapping_cap_account_dirty(mapping)) {
138 dec_zone_page_state(page, NR_FILE_DIRTY);
139 dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
143 void remove_from_page_cache(struct page *page)
145 struct address_space *mapping = page->mapping;
147 BUG_ON(!PageLocked(page));
149 spin_lock_irq(&mapping->tree_lock);
150 __remove_from_page_cache(page);
151 spin_unlock_irq(&mapping->tree_lock);
152 mem_cgroup_uncharge_cache_page(page);
155 static int sync_page(void *word)
157 struct address_space *mapping;
158 struct page *page;
160 page = container_of((unsigned long *)word, struct page, flags);
163 * page_mapping() is being called without PG_locked held.
164 * Some knowledge of the state and use of the page is used to
165 * reduce the requirements down to a memory barrier.
166 * The danger here is of a stale page_mapping() return value
167 * indicating a struct address_space different from the one it's
168 * associated with when it is associated with one.
169 * After smp_mb(), it's either the correct page_mapping() for
170 * the page, or an old page_mapping() and the page's own
171 * page_mapping() has gone NULL.
172 * The ->sync_page() address_space operation must tolerate
173 * page_mapping() going NULL. By an amazing coincidence,
174 * this comes about because none of the users of the page
175 * in the ->sync_page() methods make essential use of the
176 * page_mapping(), merely passing the page down to the backing
177 * device's unplug functions when it's non-NULL, which in turn
178 * ignore it for all cases but swap, where only page_private(page) is
179 * of interest. When page_mapping() does go NULL, the entire
180 * call stack gracefully ignores the page and returns.
181 * -- wli
183 smp_mb();
184 mapping = page_mapping(page);
185 if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
186 mapping->a_ops->sync_page(page);
187 io_schedule();
188 return 0;
191 static int sync_page_killable(void *word)
193 sync_page(word);
194 return fatal_signal_pending(current) ? -EINTR : 0;
198 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
199 * @mapping: address space structure to write
200 * @start: offset in bytes where the range starts
201 * @end: offset in bytes where the range ends (inclusive)
202 * @sync_mode: enable synchronous operation
204 * Start writeback against all of a mapping's dirty pages that lie
205 * within the byte offsets <start, end> inclusive.
207 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
208 * opposed to a regular memory cleansing writeback. The difference between
209 * these two operations is that if a dirty page/buffer is encountered, it must
210 * be waited upon, and not just skipped over.
212 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
213 loff_t end, int sync_mode)
215 int ret;
216 struct writeback_control wbc = {
217 .sync_mode = sync_mode,
218 .nr_to_write = LONG_MAX,
219 .range_start = start,
220 .range_end = end,
223 if (!mapping_cap_writeback_dirty(mapping))
224 return 0;
226 ret = do_writepages(mapping, &wbc);
227 return ret;
230 static inline int __filemap_fdatawrite(struct address_space *mapping,
231 int sync_mode)
233 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
236 int filemap_fdatawrite(struct address_space *mapping)
238 return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
240 EXPORT_SYMBOL(filemap_fdatawrite);
242 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
243 loff_t end)
245 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
247 EXPORT_SYMBOL(filemap_fdatawrite_range);
250 * filemap_flush - mostly a non-blocking flush
251 * @mapping: target address_space
253 * This is a mostly non-blocking flush. Not suitable for data-integrity
254 * purposes - I/O may not be started against all dirty pages.
256 int filemap_flush(struct address_space *mapping)
258 return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
260 EXPORT_SYMBOL(filemap_flush);
263 * wait_on_page_writeback_range - wait for writeback to complete
264 * @mapping: target address_space
265 * @start: beginning page index
266 * @end: ending page index
268 * Wait for writeback to complete against pages indexed by start->end
269 * inclusive
271 int wait_on_page_writeback_range(struct address_space *mapping,
272 pgoff_t start, pgoff_t end)
274 struct pagevec pvec;
275 int nr_pages;
276 int ret = 0;
277 pgoff_t index;
279 if (end < start)
280 return 0;
282 pagevec_init(&pvec, 0);
283 index = start;
284 while ((index <= end) &&
285 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
286 PAGECACHE_TAG_WRITEBACK,
287 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
288 unsigned i;
290 for (i = 0; i < nr_pages; i++) {
291 struct page *page = pvec.pages[i];
293 /* until radix tree lookup accepts end_index */
294 if (page->index > end)
295 continue;
297 wait_on_page_writeback(page);
298 if (PageError(page))
299 ret = -EIO;
301 pagevec_release(&pvec);
302 cond_resched();
305 /* Check for outstanding write errors */
306 if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
307 ret = -ENOSPC;
308 if (test_and_clear_bit(AS_EIO, &mapping->flags))
309 ret = -EIO;
311 return ret;
315 * filemap_fdatawait_range - wait for all under-writeback pages to complete in a given range
316 * @mapping: address space structure to wait for
317 * @start: offset in bytes where the range starts
318 * @end: offset in bytes where the range ends (inclusive)
320 * Walk the list of under-writeback pages of the given address space
321 * in the given range and wait for all of them.
323 * This is just a simple wrapper so that callers don't have to convert offsets
324 * to page indexes themselves
326 int filemap_fdatawait_range(struct address_space *mapping, loff_t start,
327 loff_t end)
329 return wait_on_page_writeback_range(mapping, start >> PAGE_CACHE_SHIFT,
330 end >> PAGE_CACHE_SHIFT);
332 EXPORT_SYMBOL(filemap_fdatawait_range);
335 * filemap_fdatawait - wait for all under-writeback pages to complete
336 * @mapping: address space structure to wait for
338 * Walk the list of under-writeback pages of the given address space
339 * and wait for all of them.
341 int filemap_fdatawait(struct address_space *mapping)
343 loff_t i_size = i_size_read(mapping->host);
345 if (i_size == 0)
346 return 0;
348 return wait_on_page_writeback_range(mapping, 0,
349 (i_size - 1) >> PAGE_CACHE_SHIFT);
351 EXPORT_SYMBOL(filemap_fdatawait);
353 int filemap_write_and_wait(struct address_space *mapping)
355 int err = 0;
357 if (mapping->nrpages) {
358 err = filemap_fdatawrite(mapping);
360 * Even if the above returned error, the pages may be
361 * written partially (e.g. -ENOSPC), so we wait for it.
362 * But the -EIO is special case, it may indicate the worst
363 * thing (e.g. bug) happened, so we avoid waiting for it.
365 if (err != -EIO) {
366 int err2 = filemap_fdatawait(mapping);
367 if (!err)
368 err = err2;
371 return err;
373 EXPORT_SYMBOL(filemap_write_and_wait);
376 * filemap_write_and_wait_range - write out & wait on a file range
377 * @mapping: the address_space for the pages
378 * @lstart: offset in bytes where the range starts
379 * @lend: offset in bytes where the range ends (inclusive)
381 * Write out and wait upon file offsets lstart->lend, inclusive.
383 * Note that `lend' is inclusive (describes the last byte to be written) so
384 * that this function can be used to write to the very end-of-file (end = -1).
386 int filemap_write_and_wait_range(struct address_space *mapping,
387 loff_t lstart, loff_t lend)
389 int err = 0;
391 if (mapping->nrpages) {
392 err = __filemap_fdatawrite_range(mapping, lstart, lend,
393 WB_SYNC_ALL);
394 /* See comment of filemap_write_and_wait() */
395 if (err != -EIO) {
396 int err2 = wait_on_page_writeback_range(mapping,
397 lstart >> PAGE_CACHE_SHIFT,
398 lend >> PAGE_CACHE_SHIFT);
399 if (!err)
400 err = err2;
403 return err;
405 EXPORT_SYMBOL(filemap_write_and_wait_range);
408 * add_to_page_cache_locked - add a locked page to the pagecache
409 * @page: page to add
410 * @mapping: the page's address_space
411 * @offset: page index
412 * @gfp_mask: page allocation mode
414 * This function is used to add a page to the pagecache. It must be locked.
415 * This function does not add the page to the LRU. The caller must do that.
417 int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
418 pgoff_t offset, gfp_t gfp_mask)
420 int error;
422 VM_BUG_ON(!PageLocked(page));
424 error = mem_cgroup_cache_charge(page, current->mm,
425 gfp_mask & GFP_RECLAIM_MASK);
426 if (error)
427 goto out;
429 error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
430 if (error == 0) {
431 page_cache_get(page);
432 page->mapping = mapping;
433 page->index = offset;
435 spin_lock_irq(&mapping->tree_lock);
436 error = radix_tree_insert(&mapping->page_tree, offset, page);
437 if (likely(!error)) {
438 mapping->nrpages++;
439 __inc_zone_page_state(page, NR_FILE_PAGES);
440 if (PageSwapBacked(page))
441 __inc_zone_page_state(page, NR_SHMEM);
442 spin_unlock_irq(&mapping->tree_lock);
443 } else {
444 page->mapping = NULL;
445 spin_unlock_irq(&mapping->tree_lock);
446 mem_cgroup_uncharge_cache_page(page);
447 page_cache_release(page);
449 radix_tree_preload_end();
450 } else
451 mem_cgroup_uncharge_cache_page(page);
452 out:
453 return error;
455 EXPORT_SYMBOL(add_to_page_cache_locked);
457 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
458 pgoff_t offset, gfp_t gfp_mask)
460 int ret;
463 * Splice_read and readahead add shmem/tmpfs pages into the page cache
464 * before shmem_readpage has a chance to mark them as SwapBacked: they
465 * need to go on the anon lru below, and mem_cgroup_cache_charge
466 * (called in add_to_page_cache) needs to know where they're going too.
468 if (mapping_cap_swap_backed(mapping))
469 SetPageSwapBacked(page);
471 ret = add_to_page_cache(page, mapping, offset, gfp_mask);
472 if (ret == 0) {
473 if (page_is_file_cache(page))
474 lru_cache_add_file(page);
475 else
476 lru_cache_add_anon(page);
478 return ret;
480 EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
482 #ifdef CONFIG_NUMA
483 struct page *__page_cache_alloc(gfp_t gfp)
485 if (cpuset_do_page_mem_spread()) {
486 int n = cpuset_mem_spread_node();
487 return alloc_pages_exact_node(n, gfp, 0);
489 return alloc_pages(gfp, 0);
491 EXPORT_SYMBOL(__page_cache_alloc);
492 #endif
494 static int __sleep_on_page_lock(void *word)
496 io_schedule();
497 return 0;
501 * In order to wait for pages to become available there must be
502 * waitqueues associated with pages. By using a hash table of
503 * waitqueues where the bucket discipline is to maintain all
504 * waiters on the same queue and wake all when any of the pages
505 * become available, and for the woken contexts to check to be
506 * sure the appropriate page became available, this saves space
507 * at a cost of "thundering herd" phenomena during rare hash
508 * collisions.
510 static wait_queue_head_t *page_waitqueue(struct page *page)
512 const struct zone *zone = page_zone(page);
514 return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
517 static inline void wake_up_page(struct page *page, int bit)
519 __wake_up_bit(page_waitqueue(page), &page->flags, bit);
522 void wait_on_page_bit(struct page *page, int bit_nr)
524 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
526 if (test_bit(bit_nr, &page->flags))
527 __wait_on_bit(page_waitqueue(page), &wait, sync_page,
528 TASK_UNINTERRUPTIBLE);
530 EXPORT_SYMBOL(wait_on_page_bit);
533 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
534 * @page: Page defining the wait queue of interest
535 * @waiter: Waiter to add to the queue
537 * Add an arbitrary @waiter to the wait queue for the nominated @page.
539 void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
541 wait_queue_head_t *q = page_waitqueue(page);
542 unsigned long flags;
544 spin_lock_irqsave(&q->lock, flags);
545 __add_wait_queue(q, waiter);
546 spin_unlock_irqrestore(&q->lock, flags);
548 EXPORT_SYMBOL_GPL(add_page_wait_queue);
551 * unlock_page - unlock a locked page
552 * @page: the page
554 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
555 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
556 * mechananism between PageLocked pages and PageWriteback pages is shared.
557 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
559 * The mb is necessary to enforce ordering between the clear_bit and the read
560 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
562 void unlock_page(struct page *page)
564 VM_BUG_ON(!PageLocked(page));
565 clear_bit_unlock(PG_locked, &page->flags);
566 smp_mb__after_clear_bit();
567 wake_up_page(page, PG_locked);
569 EXPORT_SYMBOL(unlock_page);
572 * end_page_writeback - end writeback against a page
573 * @page: the page
575 void end_page_writeback(struct page *page)
577 if (TestClearPageReclaim(page))
578 rotate_reclaimable_page(page);
580 if (!test_clear_page_writeback(page))
581 BUG();
583 smp_mb__after_clear_bit();
584 wake_up_page(page, PG_writeback);
586 EXPORT_SYMBOL(end_page_writeback);
589 * __lock_page - get a lock on the page, assuming we need to sleep to get it
590 * @page: the page to lock
592 * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some
593 * random driver's requestfn sets TASK_RUNNING, we could busywait. However
594 * chances are that on the second loop, the block layer's plug list is empty,
595 * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
597 void __lock_page(struct page *page)
599 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
601 __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page,
602 TASK_UNINTERRUPTIBLE);
604 EXPORT_SYMBOL(__lock_page);
606 int __lock_page_killable(struct page *page)
608 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
610 return __wait_on_bit_lock(page_waitqueue(page), &wait,
611 sync_page_killable, TASK_KILLABLE);
613 EXPORT_SYMBOL_GPL(__lock_page_killable);
616 * __lock_page_nosync - get a lock on the page, without calling sync_page()
617 * @page: the page to lock
619 * Variant of lock_page that does not require the caller to hold a reference
620 * on the page's mapping.
622 void __lock_page_nosync(struct page *page)
624 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
625 __wait_on_bit_lock(page_waitqueue(page), &wait, __sleep_on_page_lock,
626 TASK_UNINTERRUPTIBLE);
630 * find_get_page - find and get a page reference
631 * @mapping: the address_space to search
632 * @offset: the page index
634 * Is there a pagecache struct page at the given (mapping, offset) tuple?
635 * If yes, increment its refcount and return it; if no, return NULL.
637 struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
639 void **pagep;
640 struct page *page;
642 rcu_read_lock();
643 repeat:
644 page = NULL;
645 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
646 if (pagep) {
647 page = radix_tree_deref_slot(pagep);
648 if (unlikely(!page || page == RADIX_TREE_RETRY))
649 goto repeat;
651 if (!page_cache_get_speculative(page))
652 goto repeat;
655 * Has the page moved?
656 * This is part of the lockless pagecache protocol. See
657 * include/linux/pagemap.h for details.
659 if (unlikely(page != *pagep)) {
660 page_cache_release(page);
661 goto repeat;
664 rcu_read_unlock();
666 return page;
668 EXPORT_SYMBOL(find_get_page);
671 * find_lock_page - locate, pin and lock a pagecache page
672 * @mapping: the address_space to search
673 * @offset: the page index
675 * Locates the desired pagecache page, locks it, increments its reference
676 * count and returns its address.
678 * Returns zero if the page was not present. find_lock_page() may sleep.
680 struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
682 struct page *page;
684 repeat:
685 page = find_get_page(mapping, offset);
686 if (page) {
687 lock_page(page);
688 /* Has the page been truncated? */
689 if (unlikely(page->mapping != mapping)) {
690 unlock_page(page);
691 page_cache_release(page);
692 goto repeat;
694 VM_BUG_ON(page->index != offset);
696 return page;
698 EXPORT_SYMBOL(find_lock_page);
701 * find_or_create_page - locate or add a pagecache page
702 * @mapping: the page's address_space
703 * @index: the page's index into the mapping
704 * @gfp_mask: page allocation mode
706 * Locates a page in the pagecache. If the page is not present, a new page
707 * is allocated using @gfp_mask and is added to the pagecache and to the VM's
708 * LRU list. The returned page is locked and has its reference count
709 * incremented.
711 * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
712 * allocation!
714 * find_or_create_page() returns the desired page's address, or zero on
715 * memory exhaustion.
717 struct page *find_or_create_page(struct address_space *mapping,
718 pgoff_t index, gfp_t gfp_mask)
720 struct page *page;
721 int err;
722 repeat:
723 page = find_lock_page(mapping, index);
724 if (!page) {
725 page = __page_cache_alloc(gfp_mask);
726 if (!page)
727 return NULL;
729 * We want a regular kernel memory (not highmem or DMA etc)
730 * allocation for the radix tree nodes, but we need to honour
731 * the context-specific requirements the caller has asked for.
732 * GFP_RECLAIM_MASK collects those requirements.
734 err = add_to_page_cache_lru(page, mapping, index,
735 (gfp_mask & GFP_RECLAIM_MASK));
736 if (unlikely(err)) {
737 page_cache_release(page);
738 page = NULL;
739 if (err == -EEXIST)
740 goto repeat;
743 return page;
745 EXPORT_SYMBOL(find_or_create_page);
748 * find_get_pages - gang pagecache lookup
749 * @mapping: The address_space to search
750 * @start: The starting page index
751 * @nr_pages: The maximum number of pages
752 * @pages: Where the resulting pages are placed
754 * find_get_pages() will search for and return a group of up to
755 * @nr_pages pages in the mapping. The pages are placed at @pages.
756 * find_get_pages() takes a reference against the returned pages.
758 * The search returns a group of mapping-contiguous pages with ascending
759 * indexes. There may be holes in the indices due to not-present pages.
761 * find_get_pages() returns the number of pages which were found.
763 unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
764 unsigned int nr_pages, struct page **pages)
766 unsigned int i;
767 unsigned int ret;
768 unsigned int nr_found;
770 rcu_read_lock();
771 restart:
772 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
773 (void ***)pages, start, nr_pages);
774 ret = 0;
775 for (i = 0; i < nr_found; i++) {
776 struct page *page;
777 repeat:
778 page = radix_tree_deref_slot((void **)pages[i]);
779 if (unlikely(!page))
780 continue;
782 * this can only trigger if nr_found == 1, making livelock
783 * a non issue.
785 if (unlikely(page == RADIX_TREE_RETRY))
786 goto restart;
788 if (!page_cache_get_speculative(page))
789 goto repeat;
791 /* Has the page moved? */
792 if (unlikely(page != *((void **)pages[i]))) {
793 page_cache_release(page);
794 goto repeat;
797 pages[ret] = page;
798 ret++;
800 rcu_read_unlock();
801 return ret;
805 * find_get_pages_contig - gang contiguous pagecache lookup
806 * @mapping: The address_space to search
807 * @index: The starting page index
808 * @nr_pages: The maximum number of pages
809 * @pages: Where the resulting pages are placed
811 * find_get_pages_contig() works exactly like find_get_pages(), except
812 * that the returned number of pages are guaranteed to be contiguous.
814 * find_get_pages_contig() returns the number of pages which were found.
816 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
817 unsigned int nr_pages, struct page **pages)
819 unsigned int i;
820 unsigned int ret;
821 unsigned int nr_found;
823 rcu_read_lock();
824 restart:
825 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
826 (void ***)pages, index, nr_pages);
827 ret = 0;
828 for (i = 0; i < nr_found; i++) {
829 struct page *page;
830 repeat:
831 page = radix_tree_deref_slot((void **)pages[i]);
832 if (unlikely(!page))
833 continue;
835 * this can only trigger if nr_found == 1, making livelock
836 * a non issue.
838 if (unlikely(page == RADIX_TREE_RETRY))
839 goto restart;
841 if (page->mapping == NULL || page->index != index)
842 break;
844 if (!page_cache_get_speculative(page))
845 goto repeat;
847 /* Has the page moved? */
848 if (unlikely(page != *((void **)pages[i]))) {
849 page_cache_release(page);
850 goto repeat;
853 pages[ret] = page;
854 ret++;
855 index++;
857 rcu_read_unlock();
858 return ret;
860 EXPORT_SYMBOL(find_get_pages_contig);
863 * find_get_pages_tag - find and return pages that match @tag
864 * @mapping: the address_space to search
865 * @index: the starting page index
866 * @tag: the tag index
867 * @nr_pages: the maximum number of pages
868 * @pages: where the resulting pages are placed
870 * Like find_get_pages, except we only return pages which are tagged with
871 * @tag. We update @index to index the next page for the traversal.
873 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
874 int tag, unsigned int nr_pages, struct page **pages)
876 unsigned int i;
877 unsigned int ret;
878 unsigned int nr_found;
880 rcu_read_lock();
881 restart:
882 nr_found = radix_tree_gang_lookup_tag_slot(&mapping->page_tree,
883 (void ***)pages, *index, nr_pages, tag);
884 ret = 0;
885 for (i = 0; i < nr_found; i++) {
886 struct page *page;
887 repeat:
888 page = radix_tree_deref_slot((void **)pages[i]);
889 if (unlikely(!page))
890 continue;
892 * this can only trigger if nr_found == 1, making livelock
893 * a non issue.
895 if (unlikely(page == RADIX_TREE_RETRY))
896 goto restart;
898 if (!page_cache_get_speculative(page))
899 goto repeat;
901 /* Has the page moved? */
902 if (unlikely(page != *((void **)pages[i]))) {
903 page_cache_release(page);
904 goto repeat;
907 pages[ret] = page;
908 ret++;
910 rcu_read_unlock();
912 if (ret)
913 *index = pages[ret - 1]->index + 1;
915 return ret;
917 EXPORT_SYMBOL(find_get_pages_tag);
920 * grab_cache_page_nowait - returns locked page at given index in given cache
921 * @mapping: target address_space
922 * @index: the page index
924 * Same as grab_cache_page(), but do not wait if the page is unavailable.
925 * This is intended for speculative data generators, where the data can
926 * be regenerated if the page couldn't be grabbed. This routine should
927 * be safe to call while holding the lock for another page.
929 * Clear __GFP_FS when allocating the page to avoid recursion into the fs
930 * and deadlock against the caller's locked page.
932 struct page *
933 grab_cache_page_nowait(struct address_space *mapping, pgoff_t index)
935 struct page *page = find_get_page(mapping, index);
937 if (page) {
938 if (trylock_page(page))
939 return page;
940 page_cache_release(page);
941 return NULL;
943 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS);
944 if (page && add_to_page_cache_lru(page, mapping, index, GFP_NOFS)) {
945 page_cache_release(page);
946 page = NULL;
948 return page;
950 EXPORT_SYMBOL(grab_cache_page_nowait);
953 * CD/DVDs are error prone. When a medium error occurs, the driver may fail
954 * a _large_ part of the i/o request. Imagine the worst scenario:
956 * ---R__________________________________________B__________
957 * ^ reading here ^ bad block(assume 4k)
959 * read(R) => miss => readahead(R...B) => media error => frustrating retries
960 * => failing the whole request => read(R) => read(R+1) =>
961 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
962 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
963 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
965 * It is going insane. Fix it by quickly scaling down the readahead size.
967 static void shrink_readahead_size_eio(struct file *filp,
968 struct file_ra_state *ra)
970 ra->ra_pages /= 4;
974 * do_generic_file_read - generic file read routine
975 * @filp: the file to read
976 * @ppos: current file position
977 * @desc: read_descriptor
978 * @actor: read method
980 * This is a generic file read routine, and uses the
981 * mapping->a_ops->readpage() function for the actual low-level stuff.
983 * This is really ugly. But the goto's actually try to clarify some
984 * of the logic when it comes to error handling etc.
986 static void do_generic_file_read(struct file *filp, loff_t *ppos,
987 read_descriptor_t *desc, read_actor_t actor)
989 struct address_space *mapping = filp->f_mapping;
990 struct inode *inode = mapping->host;
991 struct file_ra_state *ra = &filp->f_ra;
992 pgoff_t index;
993 pgoff_t last_index;
994 pgoff_t prev_index;
995 unsigned long offset; /* offset into pagecache page */
996 unsigned int prev_offset;
997 int error;
999 index = *ppos >> PAGE_CACHE_SHIFT;
1000 prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT;
1001 prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1);
1002 last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
1003 offset = *ppos & ~PAGE_CACHE_MASK;
1005 for (;;) {
1006 struct page *page;
1007 pgoff_t end_index;
1008 loff_t isize;
1009 unsigned long nr, ret;
1011 cond_resched();
1012 find_page:
1013 page = find_get_page(mapping, index);
1014 if (!page) {
1015 page_cache_sync_readahead(mapping,
1016 ra, filp,
1017 index, last_index - index);
1018 page = find_get_page(mapping, index);
1019 if (unlikely(page == NULL))
1020 goto no_cached_page;
1022 if (PageReadahead(page)) {
1023 page_cache_async_readahead(mapping,
1024 ra, filp, page,
1025 index, last_index - index);
1027 if (!PageUptodate(page)) {
1028 if (inode->i_blkbits == PAGE_CACHE_SHIFT ||
1029 !mapping->a_ops->is_partially_uptodate)
1030 goto page_not_up_to_date;
1031 if (!trylock_page(page))
1032 goto page_not_up_to_date;
1033 /* Did it get truncated before we got the lock? */
1034 if (!page->mapping)
1035 goto page_not_up_to_date_locked;
1036 if (!mapping->a_ops->is_partially_uptodate(page,
1037 desc, offset))
1038 goto page_not_up_to_date_locked;
1039 unlock_page(page);
1041 page_ok:
1043 * i_size must be checked after we know the page is Uptodate.
1045 * Checking i_size after the check allows us to calculate
1046 * the correct value for "nr", which means the zero-filled
1047 * part of the page is not copied back to userspace (unless
1048 * another truncate extends the file - this is desired though).
1051 isize = i_size_read(inode);
1052 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1053 if (unlikely(!isize || index > end_index)) {
1054 page_cache_release(page);
1055 goto out;
1058 /* nr is the maximum number of bytes to copy from this page */
1059 nr = PAGE_CACHE_SIZE;
1060 if (index == end_index) {
1061 nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1062 if (nr <= offset) {
1063 page_cache_release(page);
1064 goto out;
1067 nr = nr - offset;
1069 /* If users can be writing to this page using arbitrary
1070 * virtual addresses, take care about potential aliasing
1071 * before reading the page on the kernel side.
1073 if (mapping_writably_mapped(mapping))
1074 flush_dcache_page(page);
1077 * When a sequential read accesses a page several times,
1078 * only mark it as accessed the first time.
1080 if (prev_index != index || offset != prev_offset)
1081 mark_page_accessed(page);
1082 prev_index = index;
1085 * Ok, we have the page, and it's up-to-date, so
1086 * now we can copy it to user space...
1088 * The actor routine returns how many bytes were actually used..
1089 * NOTE! This may not be the same as how much of a user buffer
1090 * we filled up (we may be padding etc), so we can only update
1091 * "pos" here (the actor routine has to update the user buffer
1092 * pointers and the remaining count).
1094 ret = actor(desc, page, offset, nr);
1095 offset += ret;
1096 index += offset >> PAGE_CACHE_SHIFT;
1097 offset &= ~PAGE_CACHE_MASK;
1098 prev_offset = offset;
1100 page_cache_release(page);
1101 if (ret == nr && desc->count)
1102 continue;
1103 goto out;
1105 page_not_up_to_date:
1106 /* Get exclusive access to the page ... */
1107 error = lock_page_killable(page);
1108 if (unlikely(error))
1109 goto readpage_error;
1111 page_not_up_to_date_locked:
1112 /* Did it get truncated before we got the lock? */
1113 if (!page->mapping) {
1114 unlock_page(page);
1115 page_cache_release(page);
1116 continue;
1119 /* Did somebody else fill it already? */
1120 if (PageUptodate(page)) {
1121 unlock_page(page);
1122 goto page_ok;
1125 readpage:
1127 * A previous I/O error may have been due to temporary
1128 * failures, eg. multipath errors.
1129 * PG_error will be set again if readpage fails.
1131 ClearPageError(page);
1132 /* Start the actual read. The read will unlock the page. */
1133 error = mapping->a_ops->readpage(filp, page);
1135 if (unlikely(error)) {
1136 if (error == AOP_TRUNCATED_PAGE) {
1137 page_cache_release(page);
1138 goto find_page;
1140 goto readpage_error;
1143 if (!PageUptodate(page)) {
1144 error = lock_page_killable(page);
1145 if (unlikely(error))
1146 goto readpage_error;
1147 if (!PageUptodate(page)) {
1148 if (page->mapping == NULL) {
1150 * invalidate_inode_pages got it
1152 unlock_page(page);
1153 page_cache_release(page);
1154 goto find_page;
1156 unlock_page(page);
1157 shrink_readahead_size_eio(filp, ra);
1158 error = -EIO;
1159 goto readpage_error;
1161 unlock_page(page);
1164 goto page_ok;
1166 readpage_error:
1167 /* UHHUH! A synchronous read error occurred. Report it */
1168 desc->error = error;
1169 page_cache_release(page);
1170 goto out;
1172 no_cached_page:
1174 * Ok, it wasn't cached, so we need to create a new
1175 * page..
1177 page = page_cache_alloc_cold(mapping);
1178 if (!page) {
1179 desc->error = -ENOMEM;
1180 goto out;
1182 error = add_to_page_cache_lru(page, mapping,
1183 index, GFP_KERNEL);
1184 if (error) {
1185 page_cache_release(page);
1186 if (error == -EEXIST)
1187 goto find_page;
1188 desc->error = error;
1189 goto out;
1191 goto readpage;
1194 out:
1195 ra->prev_pos = prev_index;
1196 ra->prev_pos <<= PAGE_CACHE_SHIFT;
1197 ra->prev_pos |= prev_offset;
1199 *ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset;
1200 file_accessed(filp);
1203 int file_read_actor(read_descriptor_t *desc, struct page *page,
1204 unsigned long offset, unsigned long size)
1206 char *kaddr;
1207 unsigned long left, count = desc->count;
1209 if (size > count)
1210 size = count;
1213 * Faults on the destination of a read are common, so do it before
1214 * taking the kmap.
1216 if (!fault_in_pages_writeable(desc->arg.buf, size)) {
1217 kaddr = kmap_atomic(page, KM_USER0);
1218 left = __copy_to_user_inatomic(desc->arg.buf,
1219 kaddr + offset, size);
1220 kunmap_atomic(kaddr, KM_USER0);
1221 if (left == 0)
1222 goto success;
1225 /* Do it the slow way */
1226 kaddr = kmap(page);
1227 left = __copy_to_user(desc->arg.buf, kaddr + offset, size);
1228 kunmap(page);
1230 if (left) {
1231 size -= left;
1232 desc->error = -EFAULT;
1234 success:
1235 desc->count = count - size;
1236 desc->written += size;
1237 desc->arg.buf += size;
1238 return size;
1242 * Performs necessary checks before doing a write
1243 * @iov: io vector request
1244 * @nr_segs: number of segments in the iovec
1245 * @count: number of bytes to write
1246 * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE
1248 * Adjust number of segments and amount of bytes to write (nr_segs should be
1249 * properly initialized first). Returns appropriate error code that caller
1250 * should return or zero in case that write should be allowed.
1252 int generic_segment_checks(const struct iovec *iov,
1253 unsigned long *nr_segs, size_t *count, int access_flags)
1255 unsigned long seg;
1256 size_t cnt = 0;
1257 for (seg = 0; seg < *nr_segs; seg++) {
1258 const struct iovec *iv = &iov[seg];
1261 * If any segment has a negative length, or the cumulative
1262 * length ever wraps negative then return -EINVAL.
1264 cnt += iv->iov_len;
1265 if (unlikely((ssize_t)(cnt|iv->iov_len) < 0))
1266 return -EINVAL;
1267 if (access_ok(access_flags, iv->iov_base, iv->iov_len))
1268 continue;
1269 if (seg == 0)
1270 return -EFAULT;
1271 *nr_segs = seg;
1272 cnt -= iv->iov_len; /* This segment is no good */
1273 break;
1275 *count = cnt;
1276 return 0;
1278 EXPORT_SYMBOL(generic_segment_checks);
1281 * generic_file_aio_read - generic filesystem read routine
1282 * @iocb: kernel I/O control block
1283 * @iov: io vector request
1284 * @nr_segs: number of segments in the iovec
1285 * @pos: current file position
1287 * This is the "read()" routine for all filesystems
1288 * that can use the page cache directly.
1290 ssize_t
1291 generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
1292 unsigned long nr_segs, loff_t pos)
1294 struct file *filp = iocb->ki_filp;
1295 ssize_t retval;
1296 unsigned long seg;
1297 size_t count;
1298 loff_t *ppos = &iocb->ki_pos;
1300 count = 0;
1301 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1302 if (retval)
1303 return retval;
1305 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
1306 if (filp->f_flags & O_DIRECT) {
1307 loff_t size;
1308 struct address_space *mapping;
1309 struct inode *inode;
1311 mapping = filp->f_mapping;
1312 inode = mapping->host;
1313 if (!count)
1314 goto out; /* skip atime */
1315 size = i_size_read(inode);
1316 if (pos < size) {
1317 retval = filemap_write_and_wait_range(mapping, pos,
1318 pos + iov_length(iov, nr_segs) - 1);
1319 if (!retval) {
1320 retval = mapping->a_ops->direct_IO(READ, iocb,
1321 iov, pos, nr_segs);
1323 if (retval > 0)
1324 *ppos = pos + retval;
1325 if (retval) {
1326 file_accessed(filp);
1327 goto out;
1332 for (seg = 0; seg < nr_segs; seg++) {
1333 read_descriptor_t desc;
1335 desc.written = 0;
1336 desc.arg.buf = iov[seg].iov_base;
1337 desc.count = iov[seg].iov_len;
1338 if (desc.count == 0)
1339 continue;
1340 desc.error = 0;
1341 do_generic_file_read(filp, ppos, &desc, file_read_actor);
1342 retval += desc.written;
1343 if (desc.error) {
1344 retval = retval ?: desc.error;
1345 break;
1347 if (desc.count > 0)
1348 break;
1350 out:
1351 return retval;
1353 EXPORT_SYMBOL(generic_file_aio_read);
1355 static ssize_t
1356 do_readahead(struct address_space *mapping, struct file *filp,
1357 pgoff_t index, unsigned long nr)
1359 if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
1360 return -EINVAL;
1362 force_page_cache_readahead(mapping, filp, index, nr);
1363 return 0;
1366 SYSCALL_DEFINE(readahead)(int fd, loff_t offset, size_t count)
1368 ssize_t ret;
1369 struct file *file;
1371 ret = -EBADF;
1372 file = fget(fd);
1373 if (file) {
1374 if (file->f_mode & FMODE_READ) {
1375 struct address_space *mapping = file->f_mapping;
1376 pgoff_t start = offset >> PAGE_CACHE_SHIFT;
1377 pgoff_t end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
1378 unsigned long len = end - start + 1;
1379 ret = do_readahead(mapping, file, start, len);
1381 fput(file);
1383 return ret;
1385 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
1386 asmlinkage long SyS_readahead(long fd, loff_t offset, long count)
1388 return SYSC_readahead((int) fd, offset, (size_t) count);
1390 SYSCALL_ALIAS(sys_readahead, SyS_readahead);
1391 #endif
1393 #ifdef CONFIG_MMU
1395 * page_cache_read - adds requested page to the page cache if not already there
1396 * @file: file to read
1397 * @offset: page index
1399 * This adds the requested page to the page cache if it isn't already there,
1400 * and schedules an I/O to read in its contents from disk.
1402 static int page_cache_read(struct file *file, pgoff_t offset)
1404 struct address_space *mapping = file->f_mapping;
1405 struct page *page;
1406 int ret;
1408 do {
1409 page = page_cache_alloc_cold(mapping);
1410 if (!page)
1411 return -ENOMEM;
1413 ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
1414 if (ret == 0)
1415 ret = mapping->a_ops->readpage(file, page);
1416 else if (ret == -EEXIST)
1417 ret = 0; /* losing race to add is OK */
1419 page_cache_release(page);
1421 } while (ret == AOP_TRUNCATED_PAGE);
1423 return ret;
1426 #define MMAP_LOTSAMISS (100)
1429 * Synchronous readahead happens when we don't even find
1430 * a page in the page cache at all.
1432 static void do_sync_mmap_readahead(struct vm_area_struct *vma,
1433 struct file_ra_state *ra,
1434 struct file *file,
1435 pgoff_t offset)
1437 unsigned long ra_pages;
1438 struct address_space *mapping = file->f_mapping;
1440 /* If we don't want any read-ahead, don't bother */
1441 if (VM_RandomReadHint(vma))
1442 return;
1444 if (VM_SequentialReadHint(vma) ||
1445 offset - 1 == (ra->prev_pos >> PAGE_CACHE_SHIFT)) {
1446 page_cache_sync_readahead(mapping, ra, file, offset,
1447 ra->ra_pages);
1448 return;
1451 if (ra->mmap_miss < INT_MAX)
1452 ra->mmap_miss++;
1455 * Do we miss much more than hit in this file? If so,
1456 * stop bothering with read-ahead. It will only hurt.
1458 if (ra->mmap_miss > MMAP_LOTSAMISS)
1459 return;
1462 * mmap read-around
1464 ra_pages = max_sane_readahead(ra->ra_pages);
1465 if (ra_pages) {
1466 ra->start = max_t(long, 0, offset - ra_pages/2);
1467 ra->size = ra_pages;
1468 ra->async_size = 0;
1469 ra_submit(ra, mapping, file);
1474 * Asynchronous readahead happens when we find the page and PG_readahead,
1475 * so we want to possibly extend the readahead further..
1477 static void do_async_mmap_readahead(struct vm_area_struct *vma,
1478 struct file_ra_state *ra,
1479 struct file *file,
1480 struct page *page,
1481 pgoff_t offset)
1483 struct address_space *mapping = file->f_mapping;
1485 /* If we don't want any read-ahead, don't bother */
1486 if (VM_RandomReadHint(vma))
1487 return;
1488 if (ra->mmap_miss > 0)
1489 ra->mmap_miss--;
1490 if (PageReadahead(page))
1491 page_cache_async_readahead(mapping, ra, file,
1492 page, offset, ra->ra_pages);
1496 * filemap_fault - read in file data for page fault handling
1497 * @vma: vma in which the fault was taken
1498 * @vmf: struct vm_fault containing details of the fault
1500 * filemap_fault() is invoked via the vma operations vector for a
1501 * mapped memory region to read in file data during a page fault.
1503 * The goto's are kind of ugly, but this streamlines the normal case of having
1504 * it in the page cache, and handles the special cases reasonably without
1505 * having a lot of duplicated code.
1507 int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1509 int error;
1510 struct file *file = vma->vm_file;
1511 struct address_space *mapping = file->f_mapping;
1512 struct file_ra_state *ra = &file->f_ra;
1513 struct inode *inode = mapping->host;
1514 pgoff_t offset = vmf->pgoff;
1515 struct page *page;
1516 pgoff_t size;
1517 int ret = 0;
1519 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1520 if (offset >= size)
1521 return VM_FAULT_SIGBUS;
1524 * Do we have something in the page cache already?
1526 page = find_get_page(mapping, offset);
1527 if (likely(page)) {
1529 * We found the page, so try async readahead before
1530 * waiting for the lock.
1532 do_async_mmap_readahead(vma, ra, file, page, offset);
1533 lock_page(page);
1535 /* Did it get truncated? */
1536 if (unlikely(page->mapping != mapping)) {
1537 unlock_page(page);
1538 put_page(page);
1539 goto no_cached_page;
1541 } else {
1542 /* No page in the page cache at all */
1543 do_sync_mmap_readahead(vma, ra, file, offset);
1544 count_vm_event(PGMAJFAULT);
1545 ret = VM_FAULT_MAJOR;
1546 retry_find:
1547 page = find_lock_page(mapping, offset);
1548 if (!page)
1549 goto no_cached_page;
1553 * We have a locked page in the page cache, now we need to check
1554 * that it's up-to-date. If not, it is going to be due to an error.
1556 if (unlikely(!PageUptodate(page)))
1557 goto page_not_uptodate;
1560 * Found the page and have a reference on it.
1561 * We must recheck i_size under page lock.
1563 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1564 if (unlikely(offset >= size)) {
1565 unlock_page(page);
1566 page_cache_release(page);
1567 return VM_FAULT_SIGBUS;
1570 ra->prev_pos = (loff_t)offset << PAGE_CACHE_SHIFT;
1571 vmf->page = page;
1572 return ret | VM_FAULT_LOCKED;
1574 no_cached_page:
1576 * We're only likely to ever get here if MADV_RANDOM is in
1577 * effect.
1579 error = page_cache_read(file, offset);
1582 * The page we want has now been added to the page cache.
1583 * In the unlikely event that someone removed it in the
1584 * meantime, we'll just come back here and read it again.
1586 if (error >= 0)
1587 goto retry_find;
1590 * An error return from page_cache_read can result if the
1591 * system is low on memory, or a problem occurs while trying
1592 * to schedule I/O.
1594 if (error == -ENOMEM)
1595 return VM_FAULT_OOM;
1596 return VM_FAULT_SIGBUS;
1598 page_not_uptodate:
1600 * Umm, take care of errors if the page isn't up-to-date.
1601 * Try to re-read it _once_. We do this synchronously,
1602 * because there really aren't any performance issues here
1603 * and we need to check for errors.
1605 ClearPageError(page);
1606 error = mapping->a_ops->readpage(file, page);
1607 if (!error) {
1608 wait_on_page_locked(page);
1609 if (!PageUptodate(page))
1610 error = -EIO;
1612 page_cache_release(page);
1614 if (!error || error == AOP_TRUNCATED_PAGE)
1615 goto retry_find;
1617 /* Things didn't work out. Return zero to tell the mm layer so. */
1618 shrink_readahead_size_eio(file, ra);
1619 return VM_FAULT_SIGBUS;
1621 EXPORT_SYMBOL(filemap_fault);
1623 const struct vm_operations_struct generic_file_vm_ops = {
1624 .fault = filemap_fault,
1627 /* This is used for a general mmap of a disk file */
1629 int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1631 struct address_space *mapping = file->f_mapping;
1633 if (!mapping->a_ops->readpage)
1634 return -ENOEXEC;
1635 file_accessed(file);
1636 vma->vm_ops = &generic_file_vm_ops;
1637 vma->vm_flags |= VM_CAN_NONLINEAR;
1638 return 0;
1642 * This is for filesystems which do not implement ->writepage.
1644 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
1646 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
1647 return -EINVAL;
1648 return generic_file_mmap(file, vma);
1650 #else
1651 int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1653 return -ENOSYS;
1655 int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
1657 return -ENOSYS;
1659 #endif /* CONFIG_MMU */
1661 EXPORT_SYMBOL(generic_file_mmap);
1662 EXPORT_SYMBOL(generic_file_readonly_mmap);
1664 static struct page *__read_cache_page(struct address_space *mapping,
1665 pgoff_t index,
1666 int (*filler)(void *,struct page*),
1667 void *data,
1668 gfp_t gfp)
1670 struct page *page;
1671 int err;
1672 repeat:
1673 page = find_get_page(mapping, index);
1674 if (!page) {
1675 page = __page_cache_alloc(gfp | __GFP_COLD);
1676 if (!page)
1677 return ERR_PTR(-ENOMEM);
1678 err = add_to_page_cache_lru(page, mapping, index, GFP_KERNEL);
1679 if (unlikely(err)) {
1680 page_cache_release(page);
1681 if (err == -EEXIST)
1682 goto repeat;
1683 /* Presumably ENOMEM for radix tree node */
1684 return ERR_PTR(err);
1686 err = filler(data, page);
1687 if (err < 0) {
1688 page_cache_release(page);
1689 page = ERR_PTR(err);
1692 return page;
1695 static struct page *do_read_cache_page(struct address_space *mapping,
1696 pgoff_t index,
1697 int (*filler)(void *,struct page*),
1698 void *data,
1699 gfp_t gfp)
1702 struct page *page;
1703 int err;
1705 retry:
1706 page = __read_cache_page(mapping, index, filler, data, gfp);
1707 if (IS_ERR(page))
1708 return page;
1709 if (PageUptodate(page))
1710 goto out;
1712 lock_page(page);
1713 if (!page->mapping) {
1714 unlock_page(page);
1715 page_cache_release(page);
1716 goto retry;
1718 if (PageUptodate(page)) {
1719 unlock_page(page);
1720 goto out;
1722 err = filler(data, page);
1723 if (err < 0) {
1724 page_cache_release(page);
1725 return ERR_PTR(err);
1727 out:
1728 mark_page_accessed(page);
1729 return page;
1733 * read_cache_page_async - read into page cache, fill it if needed
1734 * @mapping: the page's address_space
1735 * @index: the page index
1736 * @filler: function to perform the read
1737 * @data: destination for read data
1739 * Same as read_cache_page, but don't wait for page to become unlocked
1740 * after submitting it to the filler.
1742 * Read into the page cache. If a page already exists, and PageUptodate() is
1743 * not set, try to fill the page but don't wait for it to become unlocked.
1745 * If the page does not get brought uptodate, return -EIO.
1747 struct page *read_cache_page_async(struct address_space *mapping,
1748 pgoff_t index,
1749 int (*filler)(void *,struct page*),
1750 void *data)
1752 return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
1754 EXPORT_SYMBOL(read_cache_page_async);
1756 static struct page *wait_on_page_read(struct page *page)
1758 if (!IS_ERR(page)) {
1759 wait_on_page_locked(page);
1760 if (!PageUptodate(page)) {
1761 page_cache_release(page);
1762 page = ERR_PTR(-EIO);
1765 return page;
1769 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
1770 * @mapping: the page's address_space
1771 * @index: the page index
1772 * @gfp: the page allocator flags to use if allocating
1774 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
1775 * any new page allocations done using the specified allocation flags. Note
1776 * that the Radix tree operations will still use GFP_KERNEL, so you can't
1777 * expect to do this atomically or anything like that - but you can pass in
1778 * other page requirements.
1780 * If the page does not get brought uptodate, return -EIO.
1782 struct page *read_cache_page_gfp(struct address_space *mapping,
1783 pgoff_t index,
1784 gfp_t gfp)
1786 filler_t *filler = (filler_t *)mapping->a_ops->readpage;
1788 return wait_on_page_read(do_read_cache_page(mapping, index, filler, NULL, gfp));
1790 EXPORT_SYMBOL(read_cache_page_gfp);
1793 * read_cache_page - read into page cache, fill it if needed
1794 * @mapping: the page's address_space
1795 * @index: the page index
1796 * @filler: function to perform the read
1797 * @data: destination for read data
1799 * Read into the page cache. If a page already exists, and PageUptodate() is
1800 * not set, try to fill the page then wait for it to become unlocked.
1802 * If the page does not get brought uptodate, return -EIO.
1804 struct page *read_cache_page(struct address_space *mapping,
1805 pgoff_t index,
1806 int (*filler)(void *,struct page*),
1807 void *data)
1809 return wait_on_page_read(read_cache_page_async(mapping, index, filler, data));
1811 EXPORT_SYMBOL(read_cache_page);
1814 * The logic we want is
1816 * if suid or (sgid and xgrp)
1817 * remove privs
1819 int should_remove_suid(struct dentry *dentry)
1821 mode_t mode = dentry->d_inode->i_mode;
1822 int kill = 0;
1824 /* suid always must be killed */
1825 if (unlikely(mode & S_ISUID))
1826 kill = ATTR_KILL_SUID;
1829 * sgid without any exec bits is just a mandatory locking mark; leave
1830 * it alone. If some exec bits are set, it's a real sgid; kill it.
1832 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1833 kill |= ATTR_KILL_SGID;
1835 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1836 return kill;
1838 return 0;
1840 EXPORT_SYMBOL(should_remove_suid);
1842 static int __remove_suid(struct dentry *dentry, int kill)
1844 struct iattr newattrs;
1846 newattrs.ia_valid = ATTR_FORCE | kill;
1847 return notify_change(dentry, &newattrs);
1850 int file_remove_suid(struct file *file)
1852 struct dentry *dentry = file->f_path.dentry;
1853 int killsuid = should_remove_suid(dentry);
1854 int killpriv = security_inode_need_killpriv(dentry);
1855 int error = 0;
1857 if (killpriv < 0)
1858 return killpriv;
1859 if (killpriv)
1860 error = security_inode_killpriv(dentry);
1861 if (!error && killsuid)
1862 error = __remove_suid(dentry, killsuid);
1864 return error;
1866 EXPORT_SYMBOL(file_remove_suid);
1868 static size_t __iovec_copy_from_user_inatomic(char *vaddr,
1869 const struct iovec *iov, size_t base, size_t bytes)
1871 size_t copied = 0, left = 0;
1873 while (bytes) {
1874 char __user *buf = iov->iov_base + base;
1875 int copy = min(bytes, iov->iov_len - base);
1877 base = 0;
1878 left = __copy_from_user_inatomic(vaddr, buf, copy);
1879 copied += copy;
1880 bytes -= copy;
1881 vaddr += copy;
1882 iov++;
1884 if (unlikely(left))
1885 break;
1887 return copied - left;
1891 * Copy as much as we can into the page and return the number of bytes which
1892 * were sucessfully copied. If a fault is encountered then return the number of
1893 * bytes which were copied.
1895 size_t iov_iter_copy_from_user_atomic(struct page *page,
1896 struct iov_iter *i, unsigned long offset, size_t bytes)
1898 char *kaddr;
1899 size_t copied;
1901 BUG_ON(!in_atomic());
1902 kaddr = kmap_atomic(page, KM_USER0);
1903 if (likely(i->nr_segs == 1)) {
1904 int left;
1905 char __user *buf = i->iov->iov_base + i->iov_offset;
1906 left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
1907 copied = bytes - left;
1908 } else {
1909 copied = __iovec_copy_from_user_inatomic(kaddr + offset,
1910 i->iov, i->iov_offset, bytes);
1912 kunmap_atomic(kaddr, KM_USER0);
1914 return copied;
1916 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
1919 * This has the same sideeffects and return value as
1920 * iov_iter_copy_from_user_atomic().
1921 * The difference is that it attempts to resolve faults.
1922 * Page must not be locked.
1924 size_t iov_iter_copy_from_user(struct page *page,
1925 struct iov_iter *i, unsigned long offset, size_t bytes)
1927 char *kaddr;
1928 size_t copied;
1930 kaddr = kmap(page);
1931 if (likely(i->nr_segs == 1)) {
1932 int left;
1933 char __user *buf = i->iov->iov_base + i->iov_offset;
1934 left = __copy_from_user(kaddr + offset, buf, bytes);
1935 copied = bytes - left;
1936 } else {
1937 copied = __iovec_copy_from_user_inatomic(kaddr + offset,
1938 i->iov, i->iov_offset, bytes);
1940 kunmap(page);
1941 return copied;
1943 EXPORT_SYMBOL(iov_iter_copy_from_user);
1945 void iov_iter_advance(struct iov_iter *i, size_t bytes)
1947 BUG_ON(i->count < bytes);
1949 if (likely(i->nr_segs == 1)) {
1950 i->iov_offset += bytes;
1951 i->count -= bytes;
1952 } else {
1953 const struct iovec *iov = i->iov;
1954 size_t base = i->iov_offset;
1957 * The !iov->iov_len check ensures we skip over unlikely
1958 * zero-length segments (without overruning the iovec).
1960 while (bytes || unlikely(i->count && !iov->iov_len)) {
1961 int copy;
1963 copy = min(bytes, iov->iov_len - base);
1964 BUG_ON(!i->count || i->count < copy);
1965 i->count -= copy;
1966 bytes -= copy;
1967 base += copy;
1968 if (iov->iov_len == base) {
1969 iov++;
1970 base = 0;
1973 i->iov = iov;
1974 i->iov_offset = base;
1977 EXPORT_SYMBOL(iov_iter_advance);
1980 * Fault in the first iovec of the given iov_iter, to a maximum length
1981 * of bytes. Returns 0 on success, or non-zero if the memory could not be
1982 * accessed (ie. because it is an invalid address).
1984 * writev-intensive code may want this to prefault several iovecs -- that
1985 * would be possible (callers must not rely on the fact that _only_ the
1986 * first iovec will be faulted with the current implementation).
1988 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
1990 char __user *buf = i->iov->iov_base + i->iov_offset;
1991 bytes = min(bytes, i->iov->iov_len - i->iov_offset);
1992 return fault_in_pages_readable(buf, bytes);
1994 EXPORT_SYMBOL(iov_iter_fault_in_readable);
1997 * Return the count of just the current iov_iter segment.
1999 size_t iov_iter_single_seg_count(struct iov_iter *i)
2001 const struct iovec *iov = i->iov;
2002 if (i->nr_segs == 1)
2003 return i->count;
2004 else
2005 return min(i->count, iov->iov_len - i->iov_offset);
2007 EXPORT_SYMBOL(iov_iter_single_seg_count);
2010 * Performs necessary checks before doing a write
2012 * Can adjust writing position or amount of bytes to write.
2013 * Returns appropriate error code that caller should return or
2014 * zero in case that write should be allowed.
2016 inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
2018 struct inode *inode = file->f_mapping->host;
2019 unsigned long limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2021 if (unlikely(*pos < 0))
2022 return -EINVAL;
2024 if (!isblk) {
2025 /* FIXME: this is for backwards compatibility with 2.4 */
2026 if (file->f_flags & O_APPEND)
2027 *pos = i_size_read(inode);
2029 if (limit != RLIM_INFINITY) {
2030 if (*pos >= limit) {
2031 send_sig(SIGXFSZ, current, 0);
2032 return -EFBIG;
2034 if (*count > limit - (typeof(limit))*pos) {
2035 *count = limit - (typeof(limit))*pos;
2041 * LFS rule
2043 if (unlikely(*pos + *count > MAX_NON_LFS &&
2044 !(file->f_flags & O_LARGEFILE))) {
2045 if (*pos >= MAX_NON_LFS) {
2046 return -EFBIG;
2048 if (*count > MAX_NON_LFS - (unsigned long)*pos) {
2049 *count = MAX_NON_LFS - (unsigned long)*pos;
2054 * Are we about to exceed the fs block limit ?
2056 * If we have written data it becomes a short write. If we have
2057 * exceeded without writing data we send a signal and return EFBIG.
2058 * Linus frestrict idea will clean these up nicely..
2060 if (likely(!isblk)) {
2061 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
2062 if (*count || *pos > inode->i_sb->s_maxbytes) {
2063 return -EFBIG;
2065 /* zero-length writes at ->s_maxbytes are OK */
2068 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
2069 *count = inode->i_sb->s_maxbytes - *pos;
2070 } else {
2071 #ifdef CONFIG_BLOCK
2072 loff_t isize;
2073 if (bdev_read_only(I_BDEV(inode)))
2074 return -EPERM;
2075 isize = i_size_read(inode);
2076 if (*pos >= isize) {
2077 if (*count || *pos > isize)
2078 return -ENOSPC;
2081 if (*pos + *count > isize)
2082 *count = isize - *pos;
2083 #else
2084 return -EPERM;
2085 #endif
2087 return 0;
2089 EXPORT_SYMBOL(generic_write_checks);
2091 int pagecache_write_begin(struct file *file, struct address_space *mapping,
2092 loff_t pos, unsigned len, unsigned flags,
2093 struct page **pagep, void **fsdata)
2095 const struct address_space_operations *aops = mapping->a_ops;
2097 return aops->write_begin(file, mapping, pos, len, flags,
2098 pagep, fsdata);
2100 EXPORT_SYMBOL(pagecache_write_begin);
2102 int pagecache_write_end(struct file *file, struct address_space *mapping,
2103 loff_t pos, unsigned len, unsigned copied,
2104 struct page *page, void *fsdata)
2106 const struct address_space_operations *aops = mapping->a_ops;
2108 mark_page_accessed(page);
2109 return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
2111 EXPORT_SYMBOL(pagecache_write_end);
2113 ssize_t
2114 generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
2115 unsigned long *nr_segs, loff_t pos, loff_t *ppos,
2116 size_t count, size_t ocount)
2118 struct file *file = iocb->ki_filp;
2119 struct address_space *mapping = file->f_mapping;
2120 struct inode *inode = mapping->host;
2121 ssize_t written;
2122 size_t write_len;
2123 pgoff_t end;
2125 if (count != ocount)
2126 *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count);
2128 write_len = iov_length(iov, *nr_segs);
2129 end = (pos + write_len - 1) >> PAGE_CACHE_SHIFT;
2131 written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);
2132 if (written)
2133 goto out;
2136 * After a write we want buffered reads to be sure to go to disk to get
2137 * the new data. We invalidate clean cached page from the region we're
2138 * about to write. We do this *before* the write so that we can return
2139 * without clobbering -EIOCBQUEUED from ->direct_IO().
2141 if (mapping->nrpages) {
2142 written = invalidate_inode_pages2_range(mapping,
2143 pos >> PAGE_CACHE_SHIFT, end);
2145 * If a page can not be invalidated, return 0 to fall back
2146 * to buffered write.
2148 if (written) {
2149 if (written == -EBUSY)
2150 return 0;
2151 goto out;
2155 written = mapping->a_ops->direct_IO(WRITE, iocb, iov, pos, *nr_segs);
2158 * Finally, try again to invalidate clean pages which might have been
2159 * cached by non-direct readahead, or faulted in by get_user_pages()
2160 * if the source of the write was an mmap'ed region of the file
2161 * we're writing. Either one is a pretty crazy thing to do,
2162 * so we don't support it 100%. If this invalidation
2163 * fails, tough, the write still worked...
2165 if (mapping->nrpages) {
2166 invalidate_inode_pages2_range(mapping,
2167 pos >> PAGE_CACHE_SHIFT, end);
2170 if (written > 0) {
2171 loff_t end = pos + written;
2172 if (end > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
2173 i_size_write(inode, end);
2174 mark_inode_dirty(inode);
2176 *ppos = end;
2178 out:
2179 return written;
2181 EXPORT_SYMBOL(generic_file_direct_write);
2184 * Find or create a page at the given pagecache position. Return the locked
2185 * page. This function is specifically for buffered writes.
2187 struct page *grab_cache_page_write_begin(struct address_space *mapping,
2188 pgoff_t index, unsigned flags)
2190 int status;
2191 struct page *page;
2192 gfp_t gfp_notmask = 0;
2193 if (flags & AOP_FLAG_NOFS)
2194 gfp_notmask = __GFP_FS;
2195 repeat:
2196 page = find_lock_page(mapping, index);
2197 if (likely(page))
2198 return page;
2200 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~gfp_notmask);
2201 if (!page)
2202 return NULL;
2203 status = add_to_page_cache_lru(page, mapping, index,
2204 GFP_KERNEL & ~gfp_notmask);
2205 if (unlikely(status)) {
2206 page_cache_release(page);
2207 if (status == -EEXIST)
2208 goto repeat;
2209 return NULL;
2211 return page;
2213 EXPORT_SYMBOL(grab_cache_page_write_begin);
2215 static ssize_t generic_perform_write(struct file *file,
2216 struct iov_iter *i, loff_t pos)
2218 struct address_space *mapping = file->f_mapping;
2219 const struct address_space_operations *a_ops = mapping->a_ops;
2220 long status = 0;
2221 ssize_t written = 0;
2222 unsigned int flags = 0;
2225 * Copies from kernel address space cannot fail (NFSD is a big user).
2227 if (segment_eq(get_fs(), KERNEL_DS))
2228 flags |= AOP_FLAG_UNINTERRUPTIBLE;
2230 do {
2231 struct page *page;
2232 pgoff_t index; /* Pagecache index for current page */
2233 unsigned long offset; /* Offset into pagecache page */
2234 unsigned long bytes; /* Bytes to write to page */
2235 size_t copied; /* Bytes copied from user */
2236 void *fsdata;
2238 offset = (pos & (PAGE_CACHE_SIZE - 1));
2239 index = pos >> PAGE_CACHE_SHIFT;
2240 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
2241 iov_iter_count(i));
2243 again:
2246 * Bring in the user page that we will copy from _first_.
2247 * Otherwise there's a nasty deadlock on copying from the
2248 * same page as we're writing to, without it being marked
2249 * up-to-date.
2251 * Not only is this an optimisation, but it is also required
2252 * to check that the address is actually valid, when atomic
2253 * usercopies are used, below.
2255 if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
2256 status = -EFAULT;
2257 break;
2260 status = a_ops->write_begin(file, mapping, pos, bytes, flags,
2261 &page, &fsdata);
2262 if (unlikely(status))
2263 break;
2265 if (mapping_writably_mapped(mapping))
2266 flush_dcache_page(page);
2268 pagefault_disable();
2269 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
2270 pagefault_enable();
2271 flush_dcache_page(page);
2273 mark_page_accessed(page);
2274 status = a_ops->write_end(file, mapping, pos, bytes, copied,
2275 page, fsdata);
2276 if (unlikely(status < 0))
2277 break;
2278 copied = status;
2280 cond_resched();
2282 iov_iter_advance(i, copied);
2283 if (unlikely(copied == 0)) {
2285 * If we were unable to copy any data at all, we must
2286 * fall back to a single segment length write.
2288 * If we didn't fallback here, we could livelock
2289 * because not all segments in the iov can be copied at
2290 * once without a pagefault.
2292 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
2293 iov_iter_single_seg_count(i));
2294 goto again;
2296 pos += copied;
2297 written += copied;
2299 balance_dirty_pages_ratelimited(mapping);
2301 } while (iov_iter_count(i));
2303 return written ? written : status;
2306 ssize_t
2307 generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
2308 unsigned long nr_segs, loff_t pos, loff_t *ppos,
2309 size_t count, ssize_t written)
2311 struct file *file = iocb->ki_filp;
2312 struct address_space *mapping = file->f_mapping;
2313 ssize_t status;
2314 struct iov_iter i;
2316 iov_iter_init(&i, iov, nr_segs, count, written);
2317 status = generic_perform_write(file, &i, pos);
2319 if (likely(status >= 0)) {
2320 written += status;
2321 *ppos = pos + status;
2325 * If we get here for O_DIRECT writes then we must have fallen through
2326 * to buffered writes (block instantiation inside i_size). So we sync
2327 * the file data here, to try to honour O_DIRECT expectations.
2329 if (unlikely(file->f_flags & O_DIRECT) && written)
2330 status = filemap_write_and_wait_range(mapping,
2331 pos, pos + written - 1);
2333 return written ? written : status;
2335 EXPORT_SYMBOL(generic_file_buffered_write);
2338 * __generic_file_aio_write - write data to a file
2339 * @iocb: IO state structure (file, offset, etc.)
2340 * @iov: vector with data to write
2341 * @nr_segs: number of segments in the vector
2342 * @ppos: position where to write
2344 * This function does all the work needed for actually writing data to a
2345 * file. It does all basic checks, removes SUID from the file, updates
2346 * modification times and calls proper subroutines depending on whether we
2347 * do direct IO or a standard buffered write.
2349 * It expects i_mutex to be grabbed unless we work on a block device or similar
2350 * object which does not need locking at all.
2352 * This function does *not* take care of syncing data in case of O_SYNC write.
2353 * A caller has to handle it. This is mainly due to the fact that we want to
2354 * avoid syncing under i_mutex.
2356 ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
2357 unsigned long nr_segs, loff_t *ppos)
2359 struct file *file = iocb->ki_filp;
2360 struct address_space * mapping = file->f_mapping;
2361 size_t ocount; /* original count */
2362 size_t count; /* after file limit checks */
2363 struct inode *inode = mapping->host;
2364 loff_t pos;
2365 ssize_t written;
2366 ssize_t err;
2368 ocount = 0;
2369 err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
2370 if (err)
2371 return err;
2373 count = ocount;
2374 pos = *ppos;
2376 vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
2378 /* We can write back this queue in page reclaim */
2379 current->backing_dev_info = mapping->backing_dev_info;
2380 written = 0;
2382 err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
2383 if (err)
2384 goto out;
2386 if (count == 0)
2387 goto out;
2389 err = file_remove_suid(file);
2390 if (err)
2391 goto out;
2393 file_update_time(file);
2395 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
2396 if (unlikely(file->f_flags & O_DIRECT)) {
2397 loff_t endbyte;
2398 ssize_t written_buffered;
2400 written = generic_file_direct_write(iocb, iov, &nr_segs, pos,
2401 ppos, count, ocount);
2402 if (written < 0 || written == count)
2403 goto out;
2405 * direct-io write to a hole: fall through to buffered I/O
2406 * for completing the rest of the request.
2408 pos += written;
2409 count -= written;
2410 written_buffered = generic_file_buffered_write(iocb, iov,
2411 nr_segs, pos, ppos, count,
2412 written);
2414 * If generic_file_buffered_write() retuned a synchronous error
2415 * then we want to return the number of bytes which were
2416 * direct-written, or the error code if that was zero. Note
2417 * that this differs from normal direct-io semantics, which
2418 * will return -EFOO even if some bytes were written.
2420 if (written_buffered < 0) {
2421 err = written_buffered;
2422 goto out;
2426 * We need to ensure that the page cache pages are written to
2427 * disk and invalidated to preserve the expected O_DIRECT
2428 * semantics.
2430 endbyte = pos + written_buffered - written - 1;
2431 err = do_sync_mapping_range(file->f_mapping, pos, endbyte,
2432 SYNC_FILE_RANGE_WAIT_BEFORE|
2433 SYNC_FILE_RANGE_WRITE|
2434 SYNC_FILE_RANGE_WAIT_AFTER);
2435 if (err == 0) {
2436 written = written_buffered;
2437 invalidate_mapping_pages(mapping,
2438 pos >> PAGE_CACHE_SHIFT,
2439 endbyte >> PAGE_CACHE_SHIFT);
2440 } else {
2442 * We don't know how much we wrote, so just return
2443 * the number of bytes which were direct-written
2446 } else {
2447 written = generic_file_buffered_write(iocb, iov, nr_segs,
2448 pos, ppos, count, written);
2450 out:
2451 current->backing_dev_info = NULL;
2452 return written ? written : err;
2454 EXPORT_SYMBOL(__generic_file_aio_write);
2457 * generic_file_aio_write - write data to a file
2458 * @iocb: IO state structure
2459 * @iov: vector with data to write
2460 * @nr_segs: number of segments in the vector
2461 * @pos: position in file where to write
2463 * This is a wrapper around __generic_file_aio_write() to be used by most
2464 * filesystems. It takes care of syncing the file in case of O_SYNC file
2465 * and acquires i_mutex as needed.
2467 ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
2468 unsigned long nr_segs, loff_t pos)
2470 struct file *file = iocb->ki_filp;
2471 struct inode *inode = file->f_mapping->host;
2472 ssize_t ret;
2474 BUG_ON(iocb->ki_pos != pos);
2476 mutex_lock(&inode->i_mutex);
2477 ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos);
2478 mutex_unlock(&inode->i_mutex);
2480 if (ret > 0 || ret == -EIOCBQUEUED) {
2481 ssize_t err;
2483 err = generic_write_sync(file, pos, ret);
2484 if (err < 0 && ret > 0)
2485 ret = err;
2487 return ret;
2489 EXPORT_SYMBOL(generic_file_aio_write);
2492 * try_to_release_page() - release old fs-specific metadata on a page
2494 * @page: the page which the kernel is trying to free
2495 * @gfp_mask: memory allocation flags (and I/O mode)
2497 * The address_space is to try to release any data against the page
2498 * (presumably at page->private). If the release was successful, return `1'.
2499 * Otherwise return zero.
2501 * This may also be called if PG_fscache is set on a page, indicating that the
2502 * page is known to the local caching routines.
2504 * The @gfp_mask argument specifies whether I/O may be performed to release
2505 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT & __GFP_FS).
2508 int try_to_release_page(struct page *page, gfp_t gfp_mask)
2510 struct address_space * const mapping = page->mapping;
2512 BUG_ON(!PageLocked(page));
2513 if (PageWriteback(page))
2514 return 0;
2516 if (mapping && mapping->a_ops->releasepage)
2517 return mapping->a_ops->releasepage(page, gfp_mask);
2518 return try_to_free_buffers(page);
2521 EXPORT_SYMBOL(try_to_release_page);