2 * Resizable virtual memory filesystem for Linux.
4 * Copyright (C) 2000 Linus Torvalds.
6 * 2000-2001 Christoph Rohland
9 * Copyright (C) 2002-2005 Hugh Dickins.
10 * Copyright (C) 2002-2005 VERITAS Software Corporation.
11 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 * Extended attribute support for tmpfs:
14 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
15 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17 * This file is released under the GPL.
21 * This virtual memory filesystem is heavily based on the ramfs. It
22 * extends ramfs by the ability to use swap and honor resource limits
23 * which makes it a completely usable filesystem.
26 #include <linux/config.h>
27 #include <linux/module.h>
28 #include <linux/init.h>
29 #include <linux/devfs_fs_kernel.h>
32 #include <linux/mman.h>
33 #include <linux/file.h>
34 #include <linux/swap.h>
35 #include <linux/pagemap.h>
36 #include <linux/string.h>
37 #include <linux/slab.h>
38 #include <linux/backing-dev.h>
39 #include <linux/shmem_fs.h>
40 #include <linux/mount.h>
41 #include <linux/writeback.h>
42 #include <linux/vfs.h>
43 #include <linux/blkdev.h>
44 #include <linux/security.h>
45 #include <linux/swapops.h>
46 #include <linux/mempolicy.h>
47 #include <linux/namei.h>
48 #include <linux/ctype.h>
49 #include <linux/migrate.h>
51 #include <asm/uaccess.h>
52 #include <asm/div64.h>
53 #include <asm/pgtable.h>
55 /* This magic number is used in glibc for posix shared memory */
56 #define TMPFS_MAGIC 0x01021994
58 #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
59 #define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
60 #define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
62 #define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
63 #define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
65 #define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
67 /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
68 #define SHMEM_PAGEIN VM_READ
69 #define SHMEM_TRUNCATE VM_WRITE
71 /* Definition to limit shmem_truncate's steps between cond_rescheds */
72 #define LATENCY_LIMIT 64
74 /* Pretend that each entry is of this size in directory's i_size */
75 #define BOGO_DIRENT_SIZE 20
77 /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
79 SGP_QUICK
, /* don't try more than file page cache lookup */
80 SGP_READ
, /* don't exceed i_size, don't allocate page */
81 SGP_CACHE
, /* don't exceed i_size, may allocate page */
82 SGP_WRITE
, /* may exceed i_size, may allocate page */
85 static int shmem_getpage(struct inode
*inode
, unsigned long idx
,
86 struct page
**pagep
, enum sgp_type sgp
, int *type
);
88 static inline struct page
*shmem_dir_alloc(gfp_t gfp_mask
)
91 * The above definition of ENTRIES_PER_PAGE, and the use of
92 * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
93 * might be reconsidered if it ever diverges from PAGE_SIZE.
95 return alloc_pages(gfp_mask
, PAGE_CACHE_SHIFT
-PAGE_SHIFT
);
98 static inline void shmem_dir_free(struct page
*page
)
100 __free_pages(page
, PAGE_CACHE_SHIFT
-PAGE_SHIFT
);
103 static struct page
**shmem_dir_map(struct page
*page
)
105 return (struct page
**)kmap_atomic(page
, KM_USER0
);
108 static inline void shmem_dir_unmap(struct page
**dir
)
110 kunmap_atomic(dir
, KM_USER0
);
113 static swp_entry_t
*shmem_swp_map(struct page
*page
)
115 return (swp_entry_t
*)kmap_atomic(page
, KM_USER1
);
118 static inline void shmem_swp_balance_unmap(void)
121 * When passing a pointer to an i_direct entry, to code which
122 * also handles indirect entries and so will shmem_swp_unmap,
123 * we must arrange for the preempt count to remain in balance.
124 * What kmap_atomic of a lowmem page does depends on config
125 * and architecture, so pretend to kmap_atomic some lowmem page.
127 (void) kmap_atomic(ZERO_PAGE(0), KM_USER1
);
130 static inline void shmem_swp_unmap(swp_entry_t
*entry
)
132 kunmap_atomic(entry
, KM_USER1
);
135 static inline struct shmem_sb_info
*SHMEM_SB(struct super_block
*sb
)
137 return sb
->s_fs_info
;
141 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
142 * for shared memory and for shared anonymous (/dev/zero) mappings
143 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
144 * consistent with the pre-accounting of private mappings ...
146 static inline int shmem_acct_size(unsigned long flags
, loff_t size
)
148 return (flags
& VM_ACCOUNT
)?
149 security_vm_enough_memory(VM_ACCT(size
)): 0;
152 static inline void shmem_unacct_size(unsigned long flags
, loff_t size
)
154 if (flags
& VM_ACCOUNT
)
155 vm_unacct_memory(VM_ACCT(size
));
159 * ... whereas tmpfs objects are accounted incrementally as
160 * pages are allocated, in order to allow huge sparse files.
161 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
162 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
164 static inline int shmem_acct_block(unsigned long flags
)
166 return (flags
& VM_ACCOUNT
)?
167 0: security_vm_enough_memory(VM_ACCT(PAGE_CACHE_SIZE
));
170 static inline void shmem_unacct_blocks(unsigned long flags
, long pages
)
172 if (!(flags
& VM_ACCOUNT
))
173 vm_unacct_memory(pages
* VM_ACCT(PAGE_CACHE_SIZE
));
176 static struct super_operations shmem_ops
;
177 static struct address_space_operations shmem_aops
;
178 static struct file_operations shmem_file_operations
;
179 static struct inode_operations shmem_inode_operations
;
180 static struct inode_operations shmem_dir_inode_operations
;
181 static struct vm_operations_struct shmem_vm_ops
;
183 static struct backing_dev_info shmem_backing_dev_info __read_mostly
= {
184 .ra_pages
= 0, /* No readahead */
185 .capabilities
= BDI_CAP_NO_ACCT_DIRTY
| BDI_CAP_NO_WRITEBACK
,
186 .unplug_io_fn
= default_unplug_io_fn
,
189 static LIST_HEAD(shmem_swaplist
);
190 static DEFINE_SPINLOCK(shmem_swaplist_lock
);
192 static void shmem_free_blocks(struct inode
*inode
, long pages
)
194 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
195 if (sbinfo
->max_blocks
) {
196 spin_lock(&sbinfo
->stat_lock
);
197 sbinfo
->free_blocks
+= pages
;
198 inode
->i_blocks
-= pages
*BLOCKS_PER_PAGE
;
199 spin_unlock(&sbinfo
->stat_lock
);
204 * shmem_recalc_inode - recalculate the size of an inode
206 * @inode: inode to recalc
208 * We have to calculate the free blocks since the mm can drop
209 * undirtied hole pages behind our back.
211 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
212 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
214 * It has to be called with the spinlock held.
216 static void shmem_recalc_inode(struct inode
*inode
)
218 struct shmem_inode_info
*info
= SHMEM_I(inode
);
221 freed
= info
->alloced
- info
->swapped
- inode
->i_mapping
->nrpages
;
223 info
->alloced
-= freed
;
224 shmem_unacct_blocks(info
->flags
, freed
);
225 shmem_free_blocks(inode
, freed
);
230 * shmem_swp_entry - find the swap vector position in the info structure
232 * @info: info structure for the inode
233 * @index: index of the page to find
234 * @page: optional page to add to the structure. Has to be preset to
237 * If there is no space allocated yet it will return NULL when
238 * page is NULL, else it will use the page for the needed block,
239 * setting it to NULL on return to indicate that it has been used.
241 * The swap vector is organized the following way:
243 * There are SHMEM_NR_DIRECT entries directly stored in the
244 * shmem_inode_info structure. So small files do not need an addional
247 * For pages with index > SHMEM_NR_DIRECT there is the pointer
248 * i_indirect which points to a page which holds in the first half
249 * doubly indirect blocks, in the second half triple indirect blocks:
251 * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
252 * following layout (for SHMEM_NR_DIRECT == 16):
254 * i_indirect -> dir --> 16-19
267 static swp_entry_t
*shmem_swp_entry(struct shmem_inode_info
*info
, unsigned long index
, struct page
**page
)
269 unsigned long offset
;
273 if (index
< SHMEM_NR_DIRECT
) {
274 shmem_swp_balance_unmap();
275 return info
->i_direct
+index
;
277 if (!info
->i_indirect
) {
279 info
->i_indirect
= *page
;
282 return NULL
; /* need another page */
285 index
-= SHMEM_NR_DIRECT
;
286 offset
= index
% ENTRIES_PER_PAGE
;
287 index
/= ENTRIES_PER_PAGE
;
288 dir
= shmem_dir_map(info
->i_indirect
);
290 if (index
>= ENTRIES_PER_PAGE
/2) {
291 index
-= ENTRIES_PER_PAGE
/2;
292 dir
+= ENTRIES_PER_PAGE
/2 + index
/ENTRIES_PER_PAGE
;
293 index
%= ENTRIES_PER_PAGE
;
300 shmem_dir_unmap(dir
);
301 return NULL
; /* need another page */
303 shmem_dir_unmap(dir
);
304 dir
= shmem_dir_map(subdir
);
310 if (!page
|| !(subdir
= *page
)) {
311 shmem_dir_unmap(dir
);
312 return NULL
; /* need a page */
317 shmem_dir_unmap(dir
);
318 return shmem_swp_map(subdir
) + offset
;
321 static void shmem_swp_set(struct shmem_inode_info
*info
, swp_entry_t
*entry
, unsigned long value
)
323 long incdec
= value
? 1: -1;
326 info
->swapped
+= incdec
;
327 if ((unsigned long)(entry
- info
->i_direct
) >= SHMEM_NR_DIRECT
) {
328 struct page
*page
= kmap_atomic_to_page(entry
);
329 set_page_private(page
, page_private(page
) + incdec
);
334 * shmem_swp_alloc - get the position of the swap entry for the page.
335 * If it does not exist allocate the entry.
337 * @info: info structure for the inode
338 * @index: index of the page to find
339 * @sgp: check and recheck i_size? skip allocation?
341 static swp_entry_t
*shmem_swp_alloc(struct shmem_inode_info
*info
, unsigned long index
, enum sgp_type sgp
)
343 struct inode
*inode
= &info
->vfs_inode
;
344 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
345 struct page
*page
= NULL
;
348 if (sgp
!= SGP_WRITE
&&
349 ((loff_t
) index
<< PAGE_CACHE_SHIFT
) >= i_size_read(inode
))
350 return ERR_PTR(-EINVAL
);
352 while (!(entry
= shmem_swp_entry(info
, index
, &page
))) {
354 return shmem_swp_map(ZERO_PAGE(0));
356 * Test free_blocks against 1 not 0, since we have 1 data
357 * page (and perhaps indirect index pages) yet to allocate:
358 * a waste to allocate index if we cannot allocate data.
360 if (sbinfo
->max_blocks
) {
361 spin_lock(&sbinfo
->stat_lock
);
362 if (sbinfo
->free_blocks
<= 1) {
363 spin_unlock(&sbinfo
->stat_lock
);
364 return ERR_PTR(-ENOSPC
);
366 sbinfo
->free_blocks
--;
367 inode
->i_blocks
+= BLOCKS_PER_PAGE
;
368 spin_unlock(&sbinfo
->stat_lock
);
371 spin_unlock(&info
->lock
);
372 page
= shmem_dir_alloc(mapping_gfp_mask(inode
->i_mapping
) | __GFP_ZERO
);
374 set_page_private(page
, 0);
375 spin_lock(&info
->lock
);
378 shmem_free_blocks(inode
, 1);
379 return ERR_PTR(-ENOMEM
);
381 if (sgp
!= SGP_WRITE
&&
382 ((loff_t
) index
<< PAGE_CACHE_SHIFT
) >= i_size_read(inode
)) {
383 entry
= ERR_PTR(-EINVAL
);
386 if (info
->next_index
<= index
)
387 info
->next_index
= index
+ 1;
390 /* another task gave its page, or truncated the file */
391 shmem_free_blocks(inode
, 1);
392 shmem_dir_free(page
);
394 if (info
->next_index
<= index
&& !IS_ERR(entry
))
395 info
->next_index
= index
+ 1;
400 * shmem_free_swp - free some swap entries in a directory
402 * @dir: pointer to the directory
403 * @edir: pointer after last entry of the directory
405 static int shmem_free_swp(swp_entry_t
*dir
, swp_entry_t
*edir
)
410 for (ptr
= dir
; ptr
< edir
; ptr
++) {
412 free_swap_and_cache(*ptr
);
413 *ptr
= (swp_entry_t
){0};
420 static int shmem_map_and_free_swp(struct page
*subdir
,
421 int offset
, int limit
, struct page
***dir
)
426 ptr
= shmem_swp_map(subdir
);
427 for (; offset
< limit
; offset
+= LATENCY_LIMIT
) {
428 int size
= limit
- offset
;
429 if (size
> LATENCY_LIMIT
)
430 size
= LATENCY_LIMIT
;
431 freed
+= shmem_free_swp(ptr
+offset
, ptr
+offset
+size
);
432 if (need_resched()) {
433 shmem_swp_unmap(ptr
);
435 shmem_dir_unmap(*dir
);
439 ptr
= shmem_swp_map(subdir
);
442 shmem_swp_unmap(ptr
);
446 static void shmem_free_pages(struct list_head
*next
)
452 page
= container_of(next
, struct page
, lru
);
454 shmem_dir_free(page
);
456 if (freed
>= LATENCY_LIMIT
) {
463 static void shmem_truncate_range(struct inode
*inode
, loff_t start
, loff_t end
)
465 struct shmem_inode_info
*info
= SHMEM_I(inode
);
470 unsigned long diroff
;
476 LIST_HEAD(pages_to_free
);
477 long nr_pages_to_free
= 0;
478 long nr_swaps_freed
= 0;
483 inode
->i_ctime
= inode
->i_mtime
= CURRENT_TIME
;
484 idx
= (start
+ PAGE_CACHE_SIZE
- 1) >> PAGE_CACHE_SHIFT
;
485 if (idx
>= info
->next_index
)
488 spin_lock(&info
->lock
);
489 info
->flags
|= SHMEM_TRUNCATE
;
490 if (likely(end
== (loff_t
) -1)) {
491 limit
= info
->next_index
;
492 info
->next_index
= idx
;
494 limit
= (end
+ PAGE_CACHE_SIZE
- 1) >> PAGE_CACHE_SHIFT
;
495 if (limit
> info
->next_index
)
496 limit
= info
->next_index
;
500 topdir
= info
->i_indirect
;
501 if (topdir
&& idx
<= SHMEM_NR_DIRECT
&& !punch_hole
) {
502 info
->i_indirect
= NULL
;
504 list_add(&topdir
->lru
, &pages_to_free
);
506 spin_unlock(&info
->lock
);
508 if (info
->swapped
&& idx
< SHMEM_NR_DIRECT
) {
509 ptr
= info
->i_direct
;
511 if (size
> SHMEM_NR_DIRECT
)
512 size
= SHMEM_NR_DIRECT
;
513 nr_swaps_freed
= shmem_free_swp(ptr
+idx
, ptr
+size
);
518 BUG_ON(limit
<= SHMEM_NR_DIRECT
);
519 limit
-= SHMEM_NR_DIRECT
;
520 idx
= (idx
> SHMEM_NR_DIRECT
)? (idx
- SHMEM_NR_DIRECT
): 0;
521 offset
= idx
% ENTRIES_PER_PAGE
;
524 dir
= shmem_dir_map(topdir
);
525 stage
= ENTRIES_PER_PAGEPAGE
/2;
526 if (idx
< ENTRIES_PER_PAGEPAGE
/2) {
528 diroff
= idx
/ENTRIES_PER_PAGE
;
530 dir
+= ENTRIES_PER_PAGE
/2;
531 dir
+= (idx
- ENTRIES_PER_PAGEPAGE
/2)/ENTRIES_PER_PAGEPAGE
;
533 stage
+= ENTRIES_PER_PAGEPAGE
;
536 diroff
= ((idx
- ENTRIES_PER_PAGEPAGE
/2) %
537 ENTRIES_PER_PAGEPAGE
) / ENTRIES_PER_PAGE
;
538 if (!diroff
&& !offset
) {
541 list_add(&middir
->lru
, &pages_to_free
);
543 shmem_dir_unmap(dir
);
544 dir
= shmem_dir_map(middir
);
552 for (; idx
< limit
; idx
+= ENTRIES_PER_PAGE
, diroff
++) {
553 if (unlikely(idx
== stage
)) {
554 shmem_dir_unmap(dir
);
555 dir
= shmem_dir_map(topdir
) +
556 ENTRIES_PER_PAGE
/2 + idx
/ENTRIES_PER_PAGEPAGE
;
559 idx
+= ENTRIES_PER_PAGEPAGE
;
563 stage
= idx
+ ENTRIES_PER_PAGEPAGE
;
567 list_add(&middir
->lru
, &pages_to_free
);
568 shmem_dir_unmap(dir
);
570 dir
= shmem_dir_map(middir
);
573 subdir
= dir
[diroff
];
574 if (subdir
&& page_private(subdir
)) {
576 if (size
> ENTRIES_PER_PAGE
)
577 size
= ENTRIES_PER_PAGE
;
578 freed
= shmem_map_and_free_swp(subdir
,
581 dir
= shmem_dir_map(middir
);
582 nr_swaps_freed
+= freed
;
584 spin_lock(&info
->lock
);
585 set_page_private(subdir
, page_private(subdir
) - freed
);
587 spin_unlock(&info
->lock
);
589 BUG_ON(page_private(subdir
) > offset
);
593 else if (subdir
&& !page_private(subdir
)) {
596 list_add(&subdir
->lru
, &pages_to_free
);
600 shmem_dir_unmap(dir
);
602 if (inode
->i_mapping
->nrpages
&& (info
->flags
& SHMEM_PAGEIN
)) {
604 * Call truncate_inode_pages again: racing shmem_unuse_inode
605 * may have swizzled a page in from swap since vmtruncate or
606 * generic_delete_inode did it, before we lowered next_index.
607 * Also, though shmem_getpage checks i_size before adding to
608 * cache, no recheck after: so fix the narrow window there too.
610 truncate_inode_pages_range(inode
->i_mapping
, start
, end
);
613 spin_lock(&info
->lock
);
614 info
->flags
&= ~SHMEM_TRUNCATE
;
615 info
->swapped
-= nr_swaps_freed
;
616 if (nr_pages_to_free
)
617 shmem_free_blocks(inode
, nr_pages_to_free
);
618 shmem_recalc_inode(inode
);
619 spin_unlock(&info
->lock
);
622 * Empty swap vector directory pages to be freed?
624 if (!list_empty(&pages_to_free
)) {
625 pages_to_free
.prev
->next
= NULL
;
626 shmem_free_pages(pages_to_free
.next
);
630 static void shmem_truncate(struct inode
*inode
)
632 shmem_truncate_range(inode
, inode
->i_size
, (loff_t
)-1);
635 static int shmem_notify_change(struct dentry
*dentry
, struct iattr
*attr
)
637 struct inode
*inode
= dentry
->d_inode
;
638 struct page
*page
= NULL
;
641 if (attr
->ia_valid
& ATTR_SIZE
) {
642 if (attr
->ia_size
< inode
->i_size
) {
644 * If truncating down to a partial page, then
645 * if that page is already allocated, hold it
646 * in memory until the truncation is over, so
647 * truncate_partial_page cannnot miss it were
648 * it assigned to swap.
650 if (attr
->ia_size
& (PAGE_CACHE_SIZE
-1)) {
651 (void) shmem_getpage(inode
,
652 attr
->ia_size
>>PAGE_CACHE_SHIFT
,
653 &page
, SGP_READ
, NULL
);
656 * Reset SHMEM_PAGEIN flag so that shmem_truncate can
657 * detect if any pages might have been added to cache
658 * after truncate_inode_pages. But we needn't bother
659 * if it's being fully truncated to zero-length: the
660 * nrpages check is efficient enough in that case.
663 struct shmem_inode_info
*info
= SHMEM_I(inode
);
664 spin_lock(&info
->lock
);
665 info
->flags
&= ~SHMEM_PAGEIN
;
666 spin_unlock(&info
->lock
);
671 error
= inode_change_ok(inode
, attr
);
673 error
= inode_setattr(inode
, attr
);
675 page_cache_release(page
);
679 static void shmem_delete_inode(struct inode
*inode
)
681 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
682 struct shmem_inode_info
*info
= SHMEM_I(inode
);
684 if (inode
->i_op
->truncate
== shmem_truncate
) {
685 truncate_inode_pages(inode
->i_mapping
, 0);
686 shmem_unacct_size(info
->flags
, inode
->i_size
);
688 shmem_truncate(inode
);
689 if (!list_empty(&info
->swaplist
)) {
690 spin_lock(&shmem_swaplist_lock
);
691 list_del_init(&info
->swaplist
);
692 spin_unlock(&shmem_swaplist_lock
);
695 BUG_ON(inode
->i_blocks
);
696 if (sbinfo
->max_inodes
) {
697 spin_lock(&sbinfo
->stat_lock
);
698 sbinfo
->free_inodes
++;
699 spin_unlock(&sbinfo
->stat_lock
);
704 static inline int shmem_find_swp(swp_entry_t entry
, swp_entry_t
*dir
, swp_entry_t
*edir
)
708 for (ptr
= dir
; ptr
< edir
; ptr
++) {
709 if (ptr
->val
== entry
.val
)
715 static int shmem_unuse_inode(struct shmem_inode_info
*info
, swp_entry_t entry
, struct page
*page
)
728 ptr
= info
->i_direct
;
729 spin_lock(&info
->lock
);
730 limit
= info
->next_index
;
732 if (size
> SHMEM_NR_DIRECT
)
733 size
= SHMEM_NR_DIRECT
;
734 offset
= shmem_find_swp(entry
, ptr
, ptr
+size
);
736 shmem_swp_balance_unmap();
739 if (!info
->i_indirect
)
742 dir
= shmem_dir_map(info
->i_indirect
);
743 stage
= SHMEM_NR_DIRECT
+ ENTRIES_PER_PAGEPAGE
/2;
745 for (idx
= SHMEM_NR_DIRECT
; idx
< limit
; idx
+= ENTRIES_PER_PAGE
, dir
++) {
746 if (unlikely(idx
== stage
)) {
747 shmem_dir_unmap(dir
-1);
748 dir
= shmem_dir_map(info
->i_indirect
) +
749 ENTRIES_PER_PAGE
/2 + idx
/ENTRIES_PER_PAGEPAGE
;
752 idx
+= ENTRIES_PER_PAGEPAGE
;
756 stage
= idx
+ ENTRIES_PER_PAGEPAGE
;
758 shmem_dir_unmap(dir
);
759 dir
= shmem_dir_map(subdir
);
762 if (subdir
&& page_private(subdir
)) {
763 ptr
= shmem_swp_map(subdir
);
765 if (size
> ENTRIES_PER_PAGE
)
766 size
= ENTRIES_PER_PAGE
;
767 offset
= shmem_find_swp(entry
, ptr
, ptr
+size
);
769 shmem_dir_unmap(dir
);
772 shmem_swp_unmap(ptr
);
776 shmem_dir_unmap(dir
-1);
778 spin_unlock(&info
->lock
);
782 inode
= &info
->vfs_inode
;
783 if (move_from_swap_cache(page
, idx
, inode
->i_mapping
) == 0) {
784 info
->flags
|= SHMEM_PAGEIN
;
785 shmem_swp_set(info
, ptr
+ offset
, 0);
787 shmem_swp_unmap(ptr
);
788 spin_unlock(&info
->lock
);
790 * Decrement swap count even when the entry is left behind:
791 * try_to_unuse will skip over mms, then reincrement count.
798 * shmem_unuse() search for an eventually swapped out shmem page.
800 int shmem_unuse(swp_entry_t entry
, struct page
*page
)
802 struct list_head
*p
, *next
;
803 struct shmem_inode_info
*info
;
806 spin_lock(&shmem_swaplist_lock
);
807 list_for_each_safe(p
, next
, &shmem_swaplist
) {
808 info
= list_entry(p
, struct shmem_inode_info
, swaplist
);
810 list_del_init(&info
->swaplist
);
811 else if (shmem_unuse_inode(info
, entry
, page
)) {
812 /* move head to start search for next from here */
813 list_move_tail(&shmem_swaplist
, &info
->swaplist
);
818 spin_unlock(&shmem_swaplist_lock
);
823 * Move the page from the page cache to the swap cache.
825 static int shmem_writepage(struct page
*page
, struct writeback_control
*wbc
)
827 struct shmem_inode_info
*info
;
828 swp_entry_t
*entry
, swap
;
829 struct address_space
*mapping
;
833 BUG_ON(!PageLocked(page
));
834 BUG_ON(page_mapped(page
));
836 mapping
= page
->mapping
;
838 inode
= mapping
->host
;
839 info
= SHMEM_I(inode
);
840 if (info
->flags
& VM_LOCKED
)
842 swap
= get_swap_page();
846 spin_lock(&info
->lock
);
847 shmem_recalc_inode(inode
);
848 if (index
>= info
->next_index
) {
849 BUG_ON(!(info
->flags
& SHMEM_TRUNCATE
));
852 entry
= shmem_swp_entry(info
, index
, NULL
);
856 if (move_to_swap_cache(page
, swap
) == 0) {
857 shmem_swp_set(info
, entry
, swap
.val
);
858 shmem_swp_unmap(entry
);
859 spin_unlock(&info
->lock
);
860 if (list_empty(&info
->swaplist
)) {
861 spin_lock(&shmem_swaplist_lock
);
862 /* move instead of add in case we're racing */
863 list_move_tail(&info
->swaplist
, &shmem_swaplist
);
864 spin_unlock(&shmem_swaplist_lock
);
870 shmem_swp_unmap(entry
);
872 spin_unlock(&info
->lock
);
875 set_page_dirty(page
);
876 return AOP_WRITEPAGE_ACTIVATE
; /* Return with the page locked */
880 static inline int shmem_parse_mpol(char *value
, int *policy
, nodemask_t
*policy_nodes
)
882 char *nodelist
= strchr(value
, ':');
886 /* NUL-terminate policy string */
888 if (nodelist_parse(nodelist
, *policy_nodes
))
891 if (!strcmp(value
, "default")) {
892 *policy
= MPOL_DEFAULT
;
893 /* Don't allow a nodelist */
896 } else if (!strcmp(value
, "prefer")) {
897 *policy
= MPOL_PREFERRED
;
898 /* Insist on a nodelist of one node only */
900 char *rest
= nodelist
;
901 while (isdigit(*rest
))
906 } else if (!strcmp(value
, "bind")) {
908 /* Insist on a nodelist */
911 } else if (!strcmp(value
, "interleave")) {
912 *policy
= MPOL_INTERLEAVE
;
913 /* Default to nodes online if no nodelist */
915 *policy_nodes
= node_online_map
;
919 /* Restore string for error message */
925 static struct page
*shmem_swapin_async(struct shared_policy
*p
,
926 swp_entry_t entry
, unsigned long idx
)
929 struct vm_area_struct pvma
;
931 /* Create a pseudo vma that just contains the policy */
932 memset(&pvma
, 0, sizeof(struct vm_area_struct
));
933 pvma
.vm_end
= PAGE_SIZE
;
935 pvma
.vm_policy
= mpol_shared_policy_lookup(p
, idx
);
936 page
= read_swap_cache_async(entry
, &pvma
, 0);
937 mpol_free(pvma
.vm_policy
);
941 struct page
*shmem_swapin(struct shmem_inode_info
*info
, swp_entry_t entry
,
944 struct shared_policy
*p
= &info
->policy
;
947 unsigned long offset
;
949 num
= valid_swaphandles(entry
, &offset
);
950 for (i
= 0; i
< num
; offset
++, i
++) {
951 page
= shmem_swapin_async(p
,
952 swp_entry(swp_type(entry
), offset
), idx
);
955 page_cache_release(page
);
957 lru_add_drain(); /* Push any new pages onto the LRU now */
958 return shmem_swapin_async(p
, entry
, idx
);
962 shmem_alloc_page(gfp_t gfp
, struct shmem_inode_info
*info
,
965 struct vm_area_struct pvma
;
968 memset(&pvma
, 0, sizeof(struct vm_area_struct
));
969 pvma
.vm_policy
= mpol_shared_policy_lookup(&info
->policy
, idx
);
971 pvma
.vm_end
= PAGE_SIZE
;
972 page
= alloc_page_vma(gfp
| __GFP_ZERO
, &pvma
, 0);
973 mpol_free(pvma
.vm_policy
);
977 static inline int shmem_parse_mpol(char *value
, int *policy
, nodemask_t
*policy_nodes
)
982 static inline struct page
*
983 shmem_swapin(struct shmem_inode_info
*info
,swp_entry_t entry
,unsigned long idx
)
985 swapin_readahead(entry
, 0, NULL
);
986 return read_swap_cache_async(entry
, NULL
, 0);
989 static inline struct page
*
990 shmem_alloc_page(gfp_t gfp
,struct shmem_inode_info
*info
, unsigned long idx
)
992 return alloc_page(gfp
| __GFP_ZERO
);
997 * shmem_getpage - either get the page from swap or allocate a new one
999 * If we allocate a new one we do not mark it dirty. That's up to the
1000 * vm. If we swap it in we mark it dirty since we also free the swap
1001 * entry since a page cannot live in both the swap and page cache
1003 static int shmem_getpage(struct inode
*inode
, unsigned long idx
,
1004 struct page
**pagep
, enum sgp_type sgp
, int *type
)
1006 struct address_space
*mapping
= inode
->i_mapping
;
1007 struct shmem_inode_info
*info
= SHMEM_I(inode
);
1008 struct shmem_sb_info
*sbinfo
;
1009 struct page
*filepage
= *pagep
;
1010 struct page
*swappage
;
1015 if (idx
>= SHMEM_MAX_INDEX
)
1018 * Normally, filepage is NULL on entry, and either found
1019 * uptodate immediately, or allocated and zeroed, or read
1020 * in under swappage, which is then assigned to filepage.
1021 * But shmem_prepare_write passes in a locked filepage,
1022 * which may be found not uptodate by other callers too,
1023 * and may need to be copied from the swappage read in.
1027 filepage
= find_lock_page(mapping
, idx
);
1028 if (filepage
&& PageUptodate(filepage
))
1031 if (sgp
== SGP_QUICK
)
1034 spin_lock(&info
->lock
);
1035 shmem_recalc_inode(inode
);
1036 entry
= shmem_swp_alloc(info
, idx
, sgp
);
1037 if (IS_ERR(entry
)) {
1038 spin_unlock(&info
->lock
);
1039 error
= PTR_ERR(entry
);
1045 /* Look it up and read it in.. */
1046 swappage
= lookup_swap_cache(swap
);
1048 shmem_swp_unmap(entry
);
1049 spin_unlock(&info
->lock
);
1050 /* here we actually do the io */
1051 if (type
&& *type
== VM_FAULT_MINOR
) {
1052 inc_page_state(pgmajfault
);
1053 *type
= VM_FAULT_MAJOR
;
1055 swappage
= shmem_swapin(info
, swap
, idx
);
1057 spin_lock(&info
->lock
);
1058 entry
= shmem_swp_alloc(info
, idx
, sgp
);
1060 error
= PTR_ERR(entry
);
1062 if (entry
->val
== swap
.val
)
1064 shmem_swp_unmap(entry
);
1066 spin_unlock(&info
->lock
);
1071 wait_on_page_locked(swappage
);
1072 page_cache_release(swappage
);
1076 /* We have to do this with page locked to prevent races */
1077 if (TestSetPageLocked(swappage
)) {
1078 shmem_swp_unmap(entry
);
1079 spin_unlock(&info
->lock
);
1080 wait_on_page_locked(swappage
);
1081 page_cache_release(swappage
);
1084 if (PageWriteback(swappage
)) {
1085 shmem_swp_unmap(entry
);
1086 spin_unlock(&info
->lock
);
1087 wait_on_page_writeback(swappage
);
1088 unlock_page(swappage
);
1089 page_cache_release(swappage
);
1092 if (!PageUptodate(swappage
)) {
1093 shmem_swp_unmap(entry
);
1094 spin_unlock(&info
->lock
);
1095 unlock_page(swappage
);
1096 page_cache_release(swappage
);
1102 shmem_swp_set(info
, entry
, 0);
1103 shmem_swp_unmap(entry
);
1104 delete_from_swap_cache(swappage
);
1105 spin_unlock(&info
->lock
);
1106 copy_highpage(filepage
, swappage
);
1107 unlock_page(swappage
);
1108 page_cache_release(swappage
);
1109 flush_dcache_page(filepage
);
1110 SetPageUptodate(filepage
);
1111 set_page_dirty(filepage
);
1113 } else if (!(error
= move_from_swap_cache(
1114 swappage
, idx
, mapping
))) {
1115 info
->flags
|= SHMEM_PAGEIN
;
1116 shmem_swp_set(info
, entry
, 0);
1117 shmem_swp_unmap(entry
);
1118 spin_unlock(&info
->lock
);
1119 filepage
= swappage
;
1122 shmem_swp_unmap(entry
);
1123 spin_unlock(&info
->lock
);
1124 unlock_page(swappage
);
1125 page_cache_release(swappage
);
1126 if (error
== -ENOMEM
) {
1127 /* let kswapd refresh zone for GFP_ATOMICs */
1128 blk_congestion_wait(WRITE
, HZ
/50);
1132 } else if (sgp
== SGP_READ
&& !filepage
) {
1133 shmem_swp_unmap(entry
);
1134 filepage
= find_get_page(mapping
, idx
);
1136 (!PageUptodate(filepage
) || TestSetPageLocked(filepage
))) {
1137 spin_unlock(&info
->lock
);
1138 wait_on_page_locked(filepage
);
1139 page_cache_release(filepage
);
1143 spin_unlock(&info
->lock
);
1145 shmem_swp_unmap(entry
);
1146 sbinfo
= SHMEM_SB(inode
->i_sb
);
1147 if (sbinfo
->max_blocks
) {
1148 spin_lock(&sbinfo
->stat_lock
);
1149 if (sbinfo
->free_blocks
== 0 ||
1150 shmem_acct_block(info
->flags
)) {
1151 spin_unlock(&sbinfo
->stat_lock
);
1152 spin_unlock(&info
->lock
);
1156 sbinfo
->free_blocks
--;
1157 inode
->i_blocks
+= BLOCKS_PER_PAGE
;
1158 spin_unlock(&sbinfo
->stat_lock
);
1159 } else if (shmem_acct_block(info
->flags
)) {
1160 spin_unlock(&info
->lock
);
1166 spin_unlock(&info
->lock
);
1167 filepage
= shmem_alloc_page(mapping_gfp_mask(mapping
),
1171 shmem_unacct_blocks(info
->flags
, 1);
1172 shmem_free_blocks(inode
, 1);
1177 spin_lock(&info
->lock
);
1178 entry
= shmem_swp_alloc(info
, idx
, sgp
);
1180 error
= PTR_ERR(entry
);
1183 shmem_swp_unmap(entry
);
1185 if (error
|| swap
.val
|| 0 != add_to_page_cache_lru(
1186 filepage
, mapping
, idx
, GFP_ATOMIC
)) {
1187 spin_unlock(&info
->lock
);
1188 page_cache_release(filepage
);
1189 shmem_unacct_blocks(info
->flags
, 1);
1190 shmem_free_blocks(inode
, 1);
1196 info
->flags
|= SHMEM_PAGEIN
;
1200 spin_unlock(&info
->lock
);
1201 flush_dcache_page(filepage
);
1202 SetPageUptodate(filepage
);
1205 if (*pagep
!= filepage
) {
1206 unlock_page(filepage
);
1212 if (*pagep
!= filepage
) {
1213 unlock_page(filepage
);
1214 page_cache_release(filepage
);
1219 struct page
*shmem_nopage(struct vm_area_struct
*vma
, unsigned long address
, int *type
)
1221 struct inode
*inode
= vma
->vm_file
->f_dentry
->d_inode
;
1222 struct page
*page
= NULL
;
1226 idx
= (address
- vma
->vm_start
) >> PAGE_SHIFT
;
1227 idx
+= vma
->vm_pgoff
;
1228 idx
>>= PAGE_CACHE_SHIFT
- PAGE_SHIFT
;
1229 if (((loff_t
) idx
<< PAGE_CACHE_SHIFT
) >= i_size_read(inode
))
1230 return NOPAGE_SIGBUS
;
1232 error
= shmem_getpage(inode
, idx
, &page
, SGP_CACHE
, type
);
1234 return (error
== -ENOMEM
)? NOPAGE_OOM
: NOPAGE_SIGBUS
;
1236 mark_page_accessed(page
);
1240 static int shmem_populate(struct vm_area_struct
*vma
,
1241 unsigned long addr
, unsigned long len
,
1242 pgprot_t prot
, unsigned long pgoff
, int nonblock
)
1244 struct inode
*inode
= vma
->vm_file
->f_dentry
->d_inode
;
1245 struct mm_struct
*mm
= vma
->vm_mm
;
1246 enum sgp_type sgp
= nonblock
? SGP_QUICK
: SGP_CACHE
;
1249 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >> PAGE_SHIFT
;
1250 if (pgoff
>= size
|| pgoff
+ (len
>> PAGE_SHIFT
) > size
)
1253 while ((long) len
> 0) {
1254 struct page
*page
= NULL
;
1257 * Will need changing if PAGE_CACHE_SIZE != PAGE_SIZE
1259 err
= shmem_getpage(inode
, pgoff
, &page
, sgp
, NULL
);
1262 /* Page may still be null, but only if nonblock was set. */
1264 mark_page_accessed(page
);
1265 err
= install_page(mm
, vma
, addr
, page
, prot
);
1267 page_cache_release(page
);
1270 } else if (vma
->vm_flags
& VM_NONLINEAR
) {
1271 /* No page was found just because we can't read it in
1272 * now (being here implies nonblock != 0), but the page
1273 * may exist, so set the PTE to fault it in later. */
1274 err
= install_file_pte(mm
, vma
, addr
, pgoff
, prot
);
1287 int shmem_set_policy(struct vm_area_struct
*vma
, struct mempolicy
*new)
1289 struct inode
*i
= vma
->vm_file
->f_dentry
->d_inode
;
1290 return mpol_set_shared_policy(&SHMEM_I(i
)->policy
, vma
, new);
1294 shmem_get_policy(struct vm_area_struct
*vma
, unsigned long addr
)
1296 struct inode
*i
= vma
->vm_file
->f_dentry
->d_inode
;
1299 idx
= ((addr
- vma
->vm_start
) >> PAGE_SHIFT
) + vma
->vm_pgoff
;
1300 return mpol_shared_policy_lookup(&SHMEM_I(i
)->policy
, idx
);
1304 int shmem_lock(struct file
*file
, int lock
, struct user_struct
*user
)
1306 struct inode
*inode
= file
->f_dentry
->d_inode
;
1307 struct shmem_inode_info
*info
= SHMEM_I(inode
);
1308 int retval
= -ENOMEM
;
1310 spin_lock(&info
->lock
);
1311 if (lock
&& !(info
->flags
& VM_LOCKED
)) {
1312 if (!user_shm_lock(inode
->i_size
, user
))
1314 info
->flags
|= VM_LOCKED
;
1316 if (!lock
&& (info
->flags
& VM_LOCKED
) && user
) {
1317 user_shm_unlock(inode
->i_size
, user
);
1318 info
->flags
&= ~VM_LOCKED
;
1322 spin_unlock(&info
->lock
);
1326 int shmem_mmap(struct file
*file
, struct vm_area_struct
*vma
)
1328 file_accessed(file
);
1329 vma
->vm_ops
= &shmem_vm_ops
;
1333 static struct inode
*
1334 shmem_get_inode(struct super_block
*sb
, int mode
, dev_t dev
)
1336 struct inode
*inode
;
1337 struct shmem_inode_info
*info
;
1338 struct shmem_sb_info
*sbinfo
= SHMEM_SB(sb
);
1340 if (sbinfo
->max_inodes
) {
1341 spin_lock(&sbinfo
->stat_lock
);
1342 if (!sbinfo
->free_inodes
) {
1343 spin_unlock(&sbinfo
->stat_lock
);
1346 sbinfo
->free_inodes
--;
1347 spin_unlock(&sbinfo
->stat_lock
);
1350 inode
= new_inode(sb
);
1352 inode
->i_mode
= mode
;
1353 inode
->i_uid
= current
->fsuid
;
1354 inode
->i_gid
= current
->fsgid
;
1355 inode
->i_blksize
= PAGE_CACHE_SIZE
;
1356 inode
->i_blocks
= 0;
1357 inode
->i_mapping
->a_ops
= &shmem_aops
;
1358 inode
->i_mapping
->backing_dev_info
= &shmem_backing_dev_info
;
1359 inode
->i_atime
= inode
->i_mtime
= inode
->i_ctime
= CURRENT_TIME
;
1360 info
= SHMEM_I(inode
);
1361 memset(info
, 0, (char *)inode
- (char *)info
);
1362 spin_lock_init(&info
->lock
);
1363 INIT_LIST_HEAD(&info
->swaplist
);
1365 switch (mode
& S_IFMT
) {
1367 init_special_inode(inode
, mode
, dev
);
1370 inode
->i_op
= &shmem_inode_operations
;
1371 inode
->i_fop
= &shmem_file_operations
;
1372 mpol_shared_policy_init(&info
->policy
, sbinfo
->policy
,
1373 &sbinfo
->policy_nodes
);
1377 /* Some things misbehave if size == 0 on a directory */
1378 inode
->i_size
= 2 * BOGO_DIRENT_SIZE
;
1379 inode
->i_op
= &shmem_dir_inode_operations
;
1380 inode
->i_fop
= &simple_dir_operations
;
1384 * Must not load anything in the rbtree,
1385 * mpol_free_shared_policy will not be called.
1387 mpol_shared_policy_init(&info
->policy
, MPOL_DEFAULT
,
1391 } else if (sbinfo
->max_inodes
) {
1392 spin_lock(&sbinfo
->stat_lock
);
1393 sbinfo
->free_inodes
++;
1394 spin_unlock(&sbinfo
->stat_lock
);
1400 static struct inode_operations shmem_symlink_inode_operations
;
1401 static struct inode_operations shmem_symlink_inline_operations
;
1404 * Normally tmpfs makes no use of shmem_prepare_write, but it
1405 * lets a tmpfs file be used read-write below the loop driver.
1408 shmem_prepare_write(struct file
*file
, struct page
*page
, unsigned offset
, unsigned to
)
1410 struct inode
*inode
= page
->mapping
->host
;
1411 return shmem_getpage(inode
, page
->index
, &page
, SGP_WRITE
, NULL
);
1415 shmem_file_write(struct file
*file
, const char __user
*buf
, size_t count
, loff_t
*ppos
)
1417 struct inode
*inode
= file
->f_dentry
->d_inode
;
1419 unsigned long written
;
1422 if ((ssize_t
) count
< 0)
1425 if (!access_ok(VERIFY_READ
, buf
, count
))
1428 mutex_lock(&inode
->i_mutex
);
1433 err
= generic_write_checks(file
, &pos
, &count
, 0);
1437 err
= remove_suid(file
->f_dentry
);
1441 inode
->i_ctime
= inode
->i_mtime
= CURRENT_TIME
;
1444 struct page
*page
= NULL
;
1445 unsigned long bytes
, index
, offset
;
1449 offset
= (pos
& (PAGE_CACHE_SIZE
-1)); /* Within page */
1450 index
= pos
>> PAGE_CACHE_SHIFT
;
1451 bytes
= PAGE_CACHE_SIZE
- offset
;
1456 * We don't hold page lock across copy from user -
1457 * what would it guard against? - so no deadlock here.
1458 * But it still may be a good idea to prefault below.
1461 err
= shmem_getpage(inode
, index
, &page
, SGP_WRITE
, NULL
);
1466 if (PageHighMem(page
)) {
1467 volatile unsigned char dummy
;
1468 __get_user(dummy
, buf
);
1469 __get_user(dummy
, buf
+ bytes
- 1);
1471 kaddr
= kmap_atomic(page
, KM_USER0
);
1472 left
= __copy_from_user_inatomic(kaddr
+ offset
,
1474 kunmap_atomic(kaddr
, KM_USER0
);
1478 left
= __copy_from_user(kaddr
+ offset
, buf
, bytes
);
1486 if (pos
> inode
->i_size
)
1487 i_size_write(inode
, pos
);
1489 flush_dcache_page(page
);
1490 set_page_dirty(page
);
1491 mark_page_accessed(page
);
1492 page_cache_release(page
);
1502 * Our dirty pages are not counted in nr_dirty,
1503 * and we do not attempt to balance dirty pages.
1513 mutex_unlock(&inode
->i_mutex
);
1517 static void do_shmem_file_read(struct file
*filp
, loff_t
*ppos
, read_descriptor_t
*desc
, read_actor_t actor
)
1519 struct inode
*inode
= filp
->f_dentry
->d_inode
;
1520 struct address_space
*mapping
= inode
->i_mapping
;
1521 unsigned long index
, offset
;
1523 index
= *ppos
>> PAGE_CACHE_SHIFT
;
1524 offset
= *ppos
& ~PAGE_CACHE_MASK
;
1527 struct page
*page
= NULL
;
1528 unsigned long end_index
, nr
, ret
;
1529 loff_t i_size
= i_size_read(inode
);
1531 end_index
= i_size
>> PAGE_CACHE_SHIFT
;
1532 if (index
> end_index
)
1534 if (index
== end_index
) {
1535 nr
= i_size
& ~PAGE_CACHE_MASK
;
1540 desc
->error
= shmem_getpage(inode
, index
, &page
, SGP_READ
, NULL
);
1542 if (desc
->error
== -EINVAL
)
1548 * We must evaluate after, since reads (unlike writes)
1549 * are called without i_mutex protection against truncate
1551 nr
= PAGE_CACHE_SIZE
;
1552 i_size
= i_size_read(inode
);
1553 end_index
= i_size
>> PAGE_CACHE_SHIFT
;
1554 if (index
== end_index
) {
1555 nr
= i_size
& ~PAGE_CACHE_MASK
;
1558 page_cache_release(page
);
1566 * If users can be writing to this page using arbitrary
1567 * virtual addresses, take care about potential aliasing
1568 * before reading the page on the kernel side.
1570 if (mapping_writably_mapped(mapping
))
1571 flush_dcache_page(page
);
1573 * Mark the page accessed if we read the beginning.
1576 mark_page_accessed(page
);
1578 page
= ZERO_PAGE(0);
1579 page_cache_get(page
);
1583 * Ok, we have the page, and it's up-to-date, so
1584 * now we can copy it to user space...
1586 * The actor routine returns how many bytes were actually used..
1587 * NOTE! This may not be the same as how much of a user buffer
1588 * we filled up (we may be padding etc), so we can only update
1589 * "pos" here (the actor routine has to update the user buffer
1590 * pointers and the remaining count).
1592 ret
= actor(desc
, page
, offset
, nr
);
1594 index
+= offset
>> PAGE_CACHE_SHIFT
;
1595 offset
&= ~PAGE_CACHE_MASK
;
1597 page_cache_release(page
);
1598 if (ret
!= nr
|| !desc
->count
)
1604 *ppos
= ((loff_t
) index
<< PAGE_CACHE_SHIFT
) + offset
;
1605 file_accessed(filp
);
1608 static ssize_t
shmem_file_read(struct file
*filp
, char __user
*buf
, size_t count
, loff_t
*ppos
)
1610 read_descriptor_t desc
;
1612 if ((ssize_t
) count
< 0)
1614 if (!access_ok(VERIFY_WRITE
, buf
, count
))
1624 do_shmem_file_read(filp
, ppos
, &desc
, file_read_actor
);
1626 return desc
.written
;
1630 static ssize_t
shmem_file_sendfile(struct file
*in_file
, loff_t
*ppos
,
1631 size_t count
, read_actor_t actor
, void *target
)
1633 read_descriptor_t desc
;
1640 desc
.arg
.data
= target
;
1643 do_shmem_file_read(in_file
, ppos
, &desc
, actor
);
1645 return desc
.written
;
1649 static int shmem_statfs(struct dentry
*dentry
, struct kstatfs
*buf
)
1651 struct shmem_sb_info
*sbinfo
= SHMEM_SB(dentry
->d_sb
);
1653 buf
->f_type
= TMPFS_MAGIC
;
1654 buf
->f_bsize
= PAGE_CACHE_SIZE
;
1655 buf
->f_namelen
= NAME_MAX
;
1656 spin_lock(&sbinfo
->stat_lock
);
1657 if (sbinfo
->max_blocks
) {
1658 buf
->f_blocks
= sbinfo
->max_blocks
;
1659 buf
->f_bavail
= buf
->f_bfree
= sbinfo
->free_blocks
;
1661 if (sbinfo
->max_inodes
) {
1662 buf
->f_files
= sbinfo
->max_inodes
;
1663 buf
->f_ffree
= sbinfo
->free_inodes
;
1665 /* else leave those fields 0 like simple_statfs */
1666 spin_unlock(&sbinfo
->stat_lock
);
1671 * File creation. Allocate an inode, and we're done..
1674 shmem_mknod(struct inode
*dir
, struct dentry
*dentry
, int mode
, dev_t dev
)
1676 struct inode
*inode
= shmem_get_inode(dir
->i_sb
, mode
, dev
);
1677 int error
= -ENOSPC
;
1680 error
= security_inode_init_security(inode
, dir
, NULL
, NULL
,
1683 if (error
!= -EOPNOTSUPP
) {
1689 if (dir
->i_mode
& S_ISGID
) {
1690 inode
->i_gid
= dir
->i_gid
;
1692 inode
->i_mode
|= S_ISGID
;
1694 dir
->i_size
+= BOGO_DIRENT_SIZE
;
1695 dir
->i_ctime
= dir
->i_mtime
= CURRENT_TIME
;
1696 d_instantiate(dentry
, inode
);
1697 dget(dentry
); /* Extra count - pin the dentry in core */
1702 static int shmem_mkdir(struct inode
*dir
, struct dentry
*dentry
, int mode
)
1706 if ((error
= shmem_mknod(dir
, dentry
, mode
| S_IFDIR
, 0)))
1712 static int shmem_create(struct inode
*dir
, struct dentry
*dentry
, int mode
,
1713 struct nameidata
*nd
)
1715 return shmem_mknod(dir
, dentry
, mode
| S_IFREG
, 0);
1721 static int shmem_link(struct dentry
*old_dentry
, struct inode
*dir
, struct dentry
*dentry
)
1723 struct inode
*inode
= old_dentry
->d_inode
;
1724 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
1727 * No ordinary (disk based) filesystem counts links as inodes;
1728 * but each new link needs a new dentry, pinning lowmem, and
1729 * tmpfs dentries cannot be pruned until they are unlinked.
1731 if (sbinfo
->max_inodes
) {
1732 spin_lock(&sbinfo
->stat_lock
);
1733 if (!sbinfo
->free_inodes
) {
1734 spin_unlock(&sbinfo
->stat_lock
);
1737 sbinfo
->free_inodes
--;
1738 spin_unlock(&sbinfo
->stat_lock
);
1741 dir
->i_size
+= BOGO_DIRENT_SIZE
;
1742 inode
->i_ctime
= dir
->i_ctime
= dir
->i_mtime
= CURRENT_TIME
;
1744 atomic_inc(&inode
->i_count
); /* New dentry reference */
1745 dget(dentry
); /* Extra pinning count for the created dentry */
1746 d_instantiate(dentry
, inode
);
1750 static int shmem_unlink(struct inode
*dir
, struct dentry
*dentry
)
1752 struct inode
*inode
= dentry
->d_inode
;
1754 if (inode
->i_nlink
> 1 && !S_ISDIR(inode
->i_mode
)) {
1755 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
1756 if (sbinfo
->max_inodes
) {
1757 spin_lock(&sbinfo
->stat_lock
);
1758 sbinfo
->free_inodes
++;
1759 spin_unlock(&sbinfo
->stat_lock
);
1763 dir
->i_size
-= BOGO_DIRENT_SIZE
;
1764 inode
->i_ctime
= dir
->i_ctime
= dir
->i_mtime
= CURRENT_TIME
;
1766 dput(dentry
); /* Undo the count from "create" - this does all the work */
1770 static int shmem_rmdir(struct inode
*dir
, struct dentry
*dentry
)
1772 if (!simple_empty(dentry
))
1775 dentry
->d_inode
->i_nlink
--;
1777 return shmem_unlink(dir
, dentry
);
1781 * The VFS layer already does all the dentry stuff for rename,
1782 * we just have to decrement the usage count for the target if
1783 * it exists so that the VFS layer correctly free's it when it
1786 static int shmem_rename(struct inode
*old_dir
, struct dentry
*old_dentry
, struct inode
*new_dir
, struct dentry
*new_dentry
)
1788 struct inode
*inode
= old_dentry
->d_inode
;
1789 int they_are_dirs
= S_ISDIR(inode
->i_mode
);
1791 if (!simple_empty(new_dentry
))
1794 if (new_dentry
->d_inode
) {
1795 (void) shmem_unlink(new_dir
, new_dentry
);
1798 } else if (they_are_dirs
) {
1803 old_dir
->i_size
-= BOGO_DIRENT_SIZE
;
1804 new_dir
->i_size
+= BOGO_DIRENT_SIZE
;
1805 old_dir
->i_ctime
= old_dir
->i_mtime
=
1806 new_dir
->i_ctime
= new_dir
->i_mtime
=
1807 inode
->i_ctime
= CURRENT_TIME
;
1811 static int shmem_symlink(struct inode
*dir
, struct dentry
*dentry
, const char *symname
)
1815 struct inode
*inode
;
1816 struct page
*page
= NULL
;
1818 struct shmem_inode_info
*info
;
1820 len
= strlen(symname
) + 1;
1821 if (len
> PAGE_CACHE_SIZE
)
1822 return -ENAMETOOLONG
;
1824 inode
= shmem_get_inode(dir
->i_sb
, S_IFLNK
|S_IRWXUGO
, 0);
1828 error
= security_inode_init_security(inode
, dir
, NULL
, NULL
,
1831 if (error
!= -EOPNOTSUPP
) {
1838 info
= SHMEM_I(inode
);
1839 inode
->i_size
= len
-1;
1840 if (len
<= (char *)inode
- (char *)info
) {
1842 memcpy(info
, symname
, len
);
1843 inode
->i_op
= &shmem_symlink_inline_operations
;
1845 error
= shmem_getpage(inode
, 0, &page
, SGP_WRITE
, NULL
);
1850 inode
->i_op
= &shmem_symlink_inode_operations
;
1851 kaddr
= kmap_atomic(page
, KM_USER0
);
1852 memcpy(kaddr
, symname
, len
);
1853 kunmap_atomic(kaddr
, KM_USER0
);
1854 set_page_dirty(page
);
1855 page_cache_release(page
);
1857 if (dir
->i_mode
& S_ISGID
)
1858 inode
->i_gid
= dir
->i_gid
;
1859 dir
->i_size
+= BOGO_DIRENT_SIZE
;
1860 dir
->i_ctime
= dir
->i_mtime
= CURRENT_TIME
;
1861 d_instantiate(dentry
, inode
);
1866 static void *shmem_follow_link_inline(struct dentry
*dentry
, struct nameidata
*nd
)
1868 nd_set_link(nd
, (char *)SHMEM_I(dentry
->d_inode
));
1872 static void *shmem_follow_link(struct dentry
*dentry
, struct nameidata
*nd
)
1874 struct page
*page
= NULL
;
1875 int res
= shmem_getpage(dentry
->d_inode
, 0, &page
, SGP_READ
, NULL
);
1876 nd_set_link(nd
, res
? ERR_PTR(res
) : kmap(page
));
1880 static void shmem_put_link(struct dentry
*dentry
, struct nameidata
*nd
, void *cookie
)
1882 if (!IS_ERR(nd_get_link(nd
))) {
1883 struct page
*page
= cookie
;
1885 mark_page_accessed(page
);
1886 page_cache_release(page
);
1890 static struct inode_operations shmem_symlink_inline_operations
= {
1891 .readlink
= generic_readlink
,
1892 .follow_link
= shmem_follow_link_inline
,
1895 static struct inode_operations shmem_symlink_inode_operations
= {
1896 .truncate
= shmem_truncate
,
1897 .readlink
= generic_readlink
,
1898 .follow_link
= shmem_follow_link
,
1899 .put_link
= shmem_put_link
,
1902 static int shmem_parse_options(char *options
, int *mode
, uid_t
*uid
,
1903 gid_t
*gid
, unsigned long *blocks
, unsigned long *inodes
,
1904 int *policy
, nodemask_t
*policy_nodes
)
1906 char *this_char
, *value
, *rest
;
1908 while (options
!= NULL
) {
1909 this_char
= options
;
1912 * NUL-terminate this option: unfortunately,
1913 * mount options form a comma-separated list,
1914 * but mpol's nodelist may also contain commas.
1916 options
= strchr(options
, ',');
1917 if (options
== NULL
)
1920 if (!isdigit(*options
)) {
1927 if ((value
= strchr(this_char
,'=')) != NULL
) {
1931 "tmpfs: No value for mount option '%s'\n",
1936 if (!strcmp(this_char
,"size")) {
1937 unsigned long long size
;
1938 size
= memparse(value
,&rest
);
1940 size
<<= PAGE_SHIFT
;
1941 size
*= totalram_pages
;
1947 *blocks
= size
>> PAGE_CACHE_SHIFT
;
1948 } else if (!strcmp(this_char
,"nr_blocks")) {
1949 *blocks
= memparse(value
,&rest
);
1952 } else if (!strcmp(this_char
,"nr_inodes")) {
1953 *inodes
= memparse(value
,&rest
);
1956 } else if (!strcmp(this_char
,"mode")) {
1959 *mode
= simple_strtoul(value
,&rest
,8);
1962 } else if (!strcmp(this_char
,"uid")) {
1965 *uid
= simple_strtoul(value
,&rest
,0);
1968 } else if (!strcmp(this_char
,"gid")) {
1971 *gid
= simple_strtoul(value
,&rest
,0);
1974 } else if (!strcmp(this_char
,"mpol")) {
1975 if (shmem_parse_mpol(value
,policy
,policy_nodes
))
1978 printk(KERN_ERR
"tmpfs: Bad mount option %s\n",
1986 printk(KERN_ERR
"tmpfs: Bad value '%s' for mount option '%s'\n",
1992 static int shmem_remount_fs(struct super_block
*sb
, int *flags
, char *data
)
1994 struct shmem_sb_info
*sbinfo
= SHMEM_SB(sb
);
1995 unsigned long max_blocks
= sbinfo
->max_blocks
;
1996 unsigned long max_inodes
= sbinfo
->max_inodes
;
1997 int policy
= sbinfo
->policy
;
1998 nodemask_t policy_nodes
= sbinfo
->policy_nodes
;
1999 unsigned long blocks
;
2000 unsigned long inodes
;
2001 int error
= -EINVAL
;
2003 if (shmem_parse_options(data
, NULL
, NULL
, NULL
, &max_blocks
,
2004 &max_inodes
, &policy
, &policy_nodes
))
2007 spin_lock(&sbinfo
->stat_lock
);
2008 blocks
= sbinfo
->max_blocks
- sbinfo
->free_blocks
;
2009 inodes
= sbinfo
->max_inodes
- sbinfo
->free_inodes
;
2010 if (max_blocks
< blocks
)
2012 if (max_inodes
< inodes
)
2015 * Those tests also disallow limited->unlimited while any are in
2016 * use, so i_blocks will always be zero when max_blocks is zero;
2017 * but we must separately disallow unlimited->limited, because
2018 * in that case we have no record of how much is already in use.
2020 if (max_blocks
&& !sbinfo
->max_blocks
)
2022 if (max_inodes
&& !sbinfo
->max_inodes
)
2026 sbinfo
->max_blocks
= max_blocks
;
2027 sbinfo
->free_blocks
= max_blocks
- blocks
;
2028 sbinfo
->max_inodes
= max_inodes
;
2029 sbinfo
->free_inodes
= max_inodes
- inodes
;
2030 sbinfo
->policy
= policy
;
2031 sbinfo
->policy_nodes
= policy_nodes
;
2033 spin_unlock(&sbinfo
->stat_lock
);
2038 static void shmem_put_super(struct super_block
*sb
)
2040 kfree(sb
->s_fs_info
);
2041 sb
->s_fs_info
= NULL
;
2044 static int shmem_fill_super(struct super_block
*sb
,
2045 void *data
, int silent
)
2047 struct inode
*inode
;
2048 struct dentry
*root
;
2049 int mode
= S_IRWXUGO
| S_ISVTX
;
2050 uid_t uid
= current
->fsuid
;
2051 gid_t gid
= current
->fsgid
;
2053 struct shmem_sb_info
*sbinfo
;
2054 unsigned long blocks
= 0;
2055 unsigned long inodes
= 0;
2056 int policy
= MPOL_DEFAULT
;
2057 nodemask_t policy_nodes
= node_online_map
;
2061 * Per default we only allow half of the physical ram per
2062 * tmpfs instance, limiting inodes to one per page of lowmem;
2063 * but the internal instance is left unlimited.
2065 if (!(sb
->s_flags
& MS_NOUSER
)) {
2066 blocks
= totalram_pages
/ 2;
2067 inodes
= totalram_pages
- totalhigh_pages
;
2068 if (inodes
> blocks
)
2070 if (shmem_parse_options(data
, &mode
, &uid
, &gid
, &blocks
,
2071 &inodes
, &policy
, &policy_nodes
))
2075 sb
->s_flags
|= MS_NOUSER
;
2078 /* Round up to L1_CACHE_BYTES to resist false sharing */
2079 sbinfo
= kmalloc(max((int)sizeof(struct shmem_sb_info
),
2080 L1_CACHE_BYTES
), GFP_KERNEL
);
2084 spin_lock_init(&sbinfo
->stat_lock
);
2085 sbinfo
->max_blocks
= blocks
;
2086 sbinfo
->free_blocks
= blocks
;
2087 sbinfo
->max_inodes
= inodes
;
2088 sbinfo
->free_inodes
= inodes
;
2089 sbinfo
->policy
= policy
;
2090 sbinfo
->policy_nodes
= policy_nodes
;
2092 sb
->s_fs_info
= sbinfo
;
2093 sb
->s_maxbytes
= SHMEM_MAX_BYTES
;
2094 sb
->s_blocksize
= PAGE_CACHE_SIZE
;
2095 sb
->s_blocksize_bits
= PAGE_CACHE_SHIFT
;
2096 sb
->s_magic
= TMPFS_MAGIC
;
2097 sb
->s_op
= &shmem_ops
;
2098 sb
->s_time_gran
= 1;
2100 inode
= shmem_get_inode(sb
, S_IFDIR
| mode
, 0);
2105 root
= d_alloc_root(inode
);
2114 shmem_put_super(sb
);
2118 static struct kmem_cache
*shmem_inode_cachep
;
2120 static struct inode
*shmem_alloc_inode(struct super_block
*sb
)
2122 struct shmem_inode_info
*p
;
2123 p
= (struct shmem_inode_info
*)kmem_cache_alloc(shmem_inode_cachep
, SLAB_KERNEL
);
2126 return &p
->vfs_inode
;
2129 static void shmem_destroy_inode(struct inode
*inode
)
2131 if ((inode
->i_mode
& S_IFMT
) == S_IFREG
) {
2132 /* only struct inode is valid if it's an inline symlink */
2133 mpol_free_shared_policy(&SHMEM_I(inode
)->policy
);
2135 kmem_cache_free(shmem_inode_cachep
, SHMEM_I(inode
));
2138 static void init_once(void *foo
, struct kmem_cache
*cachep
,
2139 unsigned long flags
)
2141 struct shmem_inode_info
*p
= (struct shmem_inode_info
*) foo
;
2143 if ((flags
& (SLAB_CTOR_VERIFY
|SLAB_CTOR_CONSTRUCTOR
)) ==
2144 SLAB_CTOR_CONSTRUCTOR
) {
2145 inode_init_once(&p
->vfs_inode
);
2149 static int init_inodecache(void)
2151 shmem_inode_cachep
= kmem_cache_create("shmem_inode_cache",
2152 sizeof(struct shmem_inode_info
),
2153 0, 0, init_once
, NULL
);
2154 if (shmem_inode_cachep
== NULL
)
2159 static void destroy_inodecache(void)
2161 if (kmem_cache_destroy(shmem_inode_cachep
))
2162 printk(KERN_INFO
"shmem_inode_cache: not all structures were freed\n");
2165 static struct address_space_operations shmem_aops
= {
2166 .writepage
= shmem_writepage
,
2167 .set_page_dirty
= __set_page_dirty_nobuffers
,
2169 .prepare_write
= shmem_prepare_write
,
2170 .commit_write
= simple_commit_write
,
2172 .migratepage
= migrate_page
,
2175 static struct file_operations shmem_file_operations
= {
2178 .llseek
= generic_file_llseek
,
2179 .read
= shmem_file_read
,
2180 .write
= shmem_file_write
,
2181 .fsync
= simple_sync_file
,
2182 .sendfile
= shmem_file_sendfile
,
2186 static struct inode_operations shmem_inode_operations
= {
2187 .truncate
= shmem_truncate
,
2188 .setattr
= shmem_notify_change
,
2189 .truncate_range
= shmem_truncate_range
,
2192 static struct inode_operations shmem_dir_inode_operations
= {
2194 .create
= shmem_create
,
2195 .lookup
= simple_lookup
,
2197 .unlink
= shmem_unlink
,
2198 .symlink
= shmem_symlink
,
2199 .mkdir
= shmem_mkdir
,
2200 .rmdir
= shmem_rmdir
,
2201 .mknod
= shmem_mknod
,
2202 .rename
= shmem_rename
,
2206 static struct super_operations shmem_ops
= {
2207 .alloc_inode
= shmem_alloc_inode
,
2208 .destroy_inode
= shmem_destroy_inode
,
2210 .statfs
= shmem_statfs
,
2211 .remount_fs
= shmem_remount_fs
,
2213 .delete_inode
= shmem_delete_inode
,
2214 .drop_inode
= generic_delete_inode
,
2215 .put_super
= shmem_put_super
,
2218 static struct vm_operations_struct shmem_vm_ops
= {
2219 .nopage
= shmem_nopage
,
2220 .populate
= shmem_populate
,
2222 .set_policy
= shmem_set_policy
,
2223 .get_policy
= shmem_get_policy
,
2228 static int shmem_get_sb(struct file_system_type
*fs_type
,
2229 int flags
, const char *dev_name
, void *data
, struct vfsmount
*mnt
)
2231 return get_sb_nodev(fs_type
, flags
, data
, shmem_fill_super
, mnt
);
2234 static struct file_system_type tmpfs_fs_type
= {
2235 .owner
= THIS_MODULE
,
2237 .get_sb
= shmem_get_sb
,
2238 .kill_sb
= kill_litter_super
,
2240 static struct vfsmount
*shm_mnt
;
2242 static int __init
init_tmpfs(void)
2246 error
= init_inodecache();
2250 error
= register_filesystem(&tmpfs_fs_type
);
2252 printk(KERN_ERR
"Could not register tmpfs\n");
2256 devfs_mk_dir("shm");
2258 shm_mnt
= vfs_kern_mount(&tmpfs_fs_type
, MS_NOUSER
,
2259 tmpfs_fs_type
.name
, NULL
);
2260 if (IS_ERR(shm_mnt
)) {
2261 error
= PTR_ERR(shm_mnt
);
2262 printk(KERN_ERR
"Could not kern_mount tmpfs\n");
2268 unregister_filesystem(&tmpfs_fs_type
);
2270 destroy_inodecache();
2272 shm_mnt
= ERR_PTR(error
);
2275 module_init(init_tmpfs
)
2278 * shmem_file_setup - get an unlinked file living in tmpfs
2280 * @name: name for dentry (to be seen in /proc/<pid>/maps
2281 * @size: size to be set for the file
2284 struct file
*shmem_file_setup(char *name
, loff_t size
, unsigned long flags
)
2288 struct inode
*inode
;
2289 struct dentry
*dentry
, *root
;
2292 if (IS_ERR(shm_mnt
))
2293 return (void *)shm_mnt
;
2295 if (size
< 0 || size
> SHMEM_MAX_BYTES
)
2296 return ERR_PTR(-EINVAL
);
2298 if (shmem_acct_size(flags
, size
))
2299 return ERR_PTR(-ENOMEM
);
2303 this.len
= strlen(name
);
2304 this.hash
= 0; /* will go */
2305 root
= shm_mnt
->mnt_root
;
2306 dentry
= d_alloc(root
, &this);
2311 file
= get_empty_filp();
2316 inode
= shmem_get_inode(root
->d_sb
, S_IFREG
| S_IRWXUGO
, 0);
2320 SHMEM_I(inode
)->flags
= flags
& VM_ACCOUNT
;
2321 d_instantiate(dentry
, inode
);
2322 inode
->i_size
= size
;
2323 inode
->i_nlink
= 0; /* It is unlinked */
2324 file
->f_vfsmnt
= mntget(shm_mnt
);
2325 file
->f_dentry
= dentry
;
2326 file
->f_mapping
= inode
->i_mapping
;
2327 file
->f_op
= &shmem_file_operations
;
2328 file
->f_mode
= FMODE_WRITE
| FMODE_READ
;
2336 shmem_unacct_size(flags
, size
);
2337 return ERR_PTR(error
);
2341 * shmem_zero_setup - setup a shared anonymous mapping
2343 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2345 int shmem_zero_setup(struct vm_area_struct
*vma
)
2348 loff_t size
= vma
->vm_end
- vma
->vm_start
;
2350 file
= shmem_file_setup("dev/zero", size
, vma
->vm_flags
);
2352 return PTR_ERR(file
);
2356 vma
->vm_file
= file
;
2357 vma
->vm_ops
= &shmem_vm_ops
;