2 * Resizable virtual memory filesystem for Linux.
4 * Copyright (C) 2000 Linus Torvalds.
6 * 2000-2001 Christoph Rohland
9 * Copyright (C) 2002-2005 Hugh Dickins.
10 * Copyright (C) 2002-2005 VERITAS Software Corporation.
11 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 * Extended attribute support for tmpfs:
14 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
15 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17 * This file is released under the GPL.
21 * This virtual memory filesystem is heavily based on the ramfs. It
22 * extends ramfs by the ability to use swap and honor resource limits
23 * which makes it a completely usable filesystem.
26 #include <linux/config.h>
27 #include <linux/module.h>
28 #include <linux/init.h>
29 #include <linux/devfs_fs_kernel.h>
32 #include <linux/mman.h>
33 #include <linux/file.h>
34 #include <linux/swap.h>
35 #include <linux/pagemap.h>
36 #include <linux/string.h>
37 #include <linux/slab.h>
38 #include <linux/backing-dev.h>
39 #include <linux/shmem_fs.h>
40 #include <linux/mount.h>
41 #include <linux/writeback.h>
42 #include <linux/vfs.h>
43 #include <linux/blkdev.h>
44 #include <linux/security.h>
45 #include <linux/swapops.h>
46 #include <linux/mempolicy.h>
47 #include <linux/namei.h>
48 #include <linux/ctype.h>
49 #include <asm/uaccess.h>
50 #include <asm/div64.h>
51 #include <asm/pgtable.h>
53 /* This magic number is used in glibc for posix shared memory */
54 #define TMPFS_MAGIC 0x01021994
56 #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
57 #define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
58 #define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
60 #define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
61 #define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
63 #define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
65 /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
66 #define SHMEM_PAGEIN VM_READ
67 #define SHMEM_TRUNCATE VM_WRITE
69 /* Definition to limit shmem_truncate's steps between cond_rescheds */
70 #define LATENCY_LIMIT 64
72 /* Pretend that each entry is of this size in directory's i_size */
73 #define BOGO_DIRENT_SIZE 20
75 /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
77 SGP_QUICK
, /* don't try more than file page cache lookup */
78 SGP_READ
, /* don't exceed i_size, don't allocate page */
79 SGP_CACHE
, /* don't exceed i_size, may allocate page */
80 SGP_WRITE
, /* may exceed i_size, may allocate page */
83 static int shmem_getpage(struct inode
*inode
, unsigned long idx
,
84 struct page
**pagep
, enum sgp_type sgp
, int *type
);
86 static inline struct page
*shmem_dir_alloc(gfp_t gfp_mask
)
89 * The above definition of ENTRIES_PER_PAGE, and the use of
90 * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
91 * might be reconsidered if it ever diverges from PAGE_SIZE.
93 return alloc_pages(gfp_mask
, PAGE_CACHE_SHIFT
-PAGE_SHIFT
);
96 static inline void shmem_dir_free(struct page
*page
)
98 __free_pages(page
, PAGE_CACHE_SHIFT
-PAGE_SHIFT
);
101 static struct page
**shmem_dir_map(struct page
*page
)
103 return (struct page
**)kmap_atomic(page
, KM_USER0
);
106 static inline void shmem_dir_unmap(struct page
**dir
)
108 kunmap_atomic(dir
, KM_USER0
);
111 static swp_entry_t
*shmem_swp_map(struct page
*page
)
113 return (swp_entry_t
*)kmap_atomic(page
, KM_USER1
);
116 static inline void shmem_swp_balance_unmap(void)
119 * When passing a pointer to an i_direct entry, to code which
120 * also handles indirect entries and so will shmem_swp_unmap,
121 * we must arrange for the preempt count to remain in balance.
122 * What kmap_atomic of a lowmem page does depends on config
123 * and architecture, so pretend to kmap_atomic some lowmem page.
125 (void) kmap_atomic(ZERO_PAGE(0), KM_USER1
);
128 static inline void shmem_swp_unmap(swp_entry_t
*entry
)
130 kunmap_atomic(entry
, KM_USER1
);
133 static inline struct shmem_sb_info
*SHMEM_SB(struct super_block
*sb
)
135 return sb
->s_fs_info
;
139 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
140 * for shared memory and for shared anonymous (/dev/zero) mappings
141 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
142 * consistent with the pre-accounting of private mappings ...
144 static inline int shmem_acct_size(unsigned long flags
, loff_t size
)
146 return (flags
& VM_ACCOUNT
)?
147 security_vm_enough_memory(VM_ACCT(size
)): 0;
150 static inline void shmem_unacct_size(unsigned long flags
, loff_t size
)
152 if (flags
& VM_ACCOUNT
)
153 vm_unacct_memory(VM_ACCT(size
));
157 * ... whereas tmpfs objects are accounted incrementally as
158 * pages are allocated, in order to allow huge sparse files.
159 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
160 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
162 static inline int shmem_acct_block(unsigned long flags
)
164 return (flags
& VM_ACCOUNT
)?
165 0: security_vm_enough_memory(VM_ACCT(PAGE_CACHE_SIZE
));
168 static inline void shmem_unacct_blocks(unsigned long flags
, long pages
)
170 if (!(flags
& VM_ACCOUNT
))
171 vm_unacct_memory(pages
* VM_ACCT(PAGE_CACHE_SIZE
));
174 static struct super_operations shmem_ops
;
175 static struct address_space_operations shmem_aops
;
176 static struct file_operations shmem_file_operations
;
177 static struct inode_operations shmem_inode_operations
;
178 static struct inode_operations shmem_dir_inode_operations
;
179 static struct vm_operations_struct shmem_vm_ops
;
181 static struct backing_dev_info shmem_backing_dev_info __read_mostly
= {
182 .ra_pages
= 0, /* No readahead */
183 .capabilities
= BDI_CAP_NO_ACCT_DIRTY
| BDI_CAP_NO_WRITEBACK
,
184 .unplug_io_fn
= default_unplug_io_fn
,
187 static LIST_HEAD(shmem_swaplist
);
188 static DEFINE_SPINLOCK(shmem_swaplist_lock
);
190 static void shmem_free_blocks(struct inode
*inode
, long pages
)
192 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
193 if (sbinfo
->max_blocks
) {
194 spin_lock(&sbinfo
->stat_lock
);
195 sbinfo
->free_blocks
+= pages
;
196 inode
->i_blocks
-= pages
*BLOCKS_PER_PAGE
;
197 spin_unlock(&sbinfo
->stat_lock
);
202 * shmem_recalc_inode - recalculate the size of an inode
204 * @inode: inode to recalc
206 * We have to calculate the free blocks since the mm can drop
207 * undirtied hole pages behind our back.
209 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
210 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
212 * It has to be called with the spinlock held.
214 static void shmem_recalc_inode(struct inode
*inode
)
216 struct shmem_inode_info
*info
= SHMEM_I(inode
);
219 freed
= info
->alloced
- info
->swapped
- inode
->i_mapping
->nrpages
;
221 info
->alloced
-= freed
;
222 shmem_unacct_blocks(info
->flags
, freed
);
223 shmem_free_blocks(inode
, freed
);
228 * shmem_swp_entry - find the swap vector position in the info structure
230 * @info: info structure for the inode
231 * @index: index of the page to find
232 * @page: optional page to add to the structure. Has to be preset to
235 * If there is no space allocated yet it will return NULL when
236 * page is NULL, else it will use the page for the needed block,
237 * setting it to NULL on return to indicate that it has been used.
239 * The swap vector is organized the following way:
241 * There are SHMEM_NR_DIRECT entries directly stored in the
242 * shmem_inode_info structure. So small files do not need an addional
245 * For pages with index > SHMEM_NR_DIRECT there is the pointer
246 * i_indirect which points to a page which holds in the first half
247 * doubly indirect blocks, in the second half triple indirect blocks:
249 * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
250 * following layout (for SHMEM_NR_DIRECT == 16):
252 * i_indirect -> dir --> 16-19
265 static swp_entry_t
*shmem_swp_entry(struct shmem_inode_info
*info
, unsigned long index
, struct page
**page
)
267 unsigned long offset
;
271 if (index
< SHMEM_NR_DIRECT
) {
272 shmem_swp_balance_unmap();
273 return info
->i_direct
+index
;
275 if (!info
->i_indirect
) {
277 info
->i_indirect
= *page
;
280 return NULL
; /* need another page */
283 index
-= SHMEM_NR_DIRECT
;
284 offset
= index
% ENTRIES_PER_PAGE
;
285 index
/= ENTRIES_PER_PAGE
;
286 dir
= shmem_dir_map(info
->i_indirect
);
288 if (index
>= ENTRIES_PER_PAGE
/2) {
289 index
-= ENTRIES_PER_PAGE
/2;
290 dir
+= ENTRIES_PER_PAGE
/2 + index
/ENTRIES_PER_PAGE
;
291 index
%= ENTRIES_PER_PAGE
;
298 shmem_dir_unmap(dir
);
299 return NULL
; /* need another page */
301 shmem_dir_unmap(dir
);
302 dir
= shmem_dir_map(subdir
);
308 if (!page
|| !(subdir
= *page
)) {
309 shmem_dir_unmap(dir
);
310 return NULL
; /* need a page */
315 shmem_dir_unmap(dir
);
316 return shmem_swp_map(subdir
) + offset
;
319 static void shmem_swp_set(struct shmem_inode_info
*info
, swp_entry_t
*entry
, unsigned long value
)
321 long incdec
= value
? 1: -1;
324 info
->swapped
+= incdec
;
325 if ((unsigned long)(entry
- info
->i_direct
) >= SHMEM_NR_DIRECT
) {
326 struct page
*page
= kmap_atomic_to_page(entry
);
327 set_page_private(page
, page_private(page
) + incdec
);
332 * shmem_swp_alloc - get the position of the swap entry for the page.
333 * If it does not exist allocate the entry.
335 * @info: info structure for the inode
336 * @index: index of the page to find
337 * @sgp: check and recheck i_size? skip allocation?
339 static swp_entry_t
*shmem_swp_alloc(struct shmem_inode_info
*info
, unsigned long index
, enum sgp_type sgp
)
341 struct inode
*inode
= &info
->vfs_inode
;
342 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
343 struct page
*page
= NULL
;
346 if (sgp
!= SGP_WRITE
&&
347 ((loff_t
) index
<< PAGE_CACHE_SHIFT
) >= i_size_read(inode
))
348 return ERR_PTR(-EINVAL
);
350 while (!(entry
= shmem_swp_entry(info
, index
, &page
))) {
352 return shmem_swp_map(ZERO_PAGE(0));
354 * Test free_blocks against 1 not 0, since we have 1 data
355 * page (and perhaps indirect index pages) yet to allocate:
356 * a waste to allocate index if we cannot allocate data.
358 if (sbinfo
->max_blocks
) {
359 spin_lock(&sbinfo
->stat_lock
);
360 if (sbinfo
->free_blocks
<= 1) {
361 spin_unlock(&sbinfo
->stat_lock
);
362 return ERR_PTR(-ENOSPC
);
364 sbinfo
->free_blocks
--;
365 inode
->i_blocks
+= BLOCKS_PER_PAGE
;
366 spin_unlock(&sbinfo
->stat_lock
);
369 spin_unlock(&info
->lock
);
370 page
= shmem_dir_alloc(mapping_gfp_mask(inode
->i_mapping
) | __GFP_ZERO
);
372 set_page_private(page
, 0);
373 spin_lock(&info
->lock
);
376 shmem_free_blocks(inode
, 1);
377 return ERR_PTR(-ENOMEM
);
379 if (sgp
!= SGP_WRITE
&&
380 ((loff_t
) index
<< PAGE_CACHE_SHIFT
) >= i_size_read(inode
)) {
381 entry
= ERR_PTR(-EINVAL
);
384 if (info
->next_index
<= index
)
385 info
->next_index
= index
+ 1;
388 /* another task gave its page, or truncated the file */
389 shmem_free_blocks(inode
, 1);
390 shmem_dir_free(page
);
392 if (info
->next_index
<= index
&& !IS_ERR(entry
))
393 info
->next_index
= index
+ 1;
398 * shmem_free_swp - free some swap entries in a directory
400 * @dir: pointer to the directory
401 * @edir: pointer after last entry of the directory
403 static int shmem_free_swp(swp_entry_t
*dir
, swp_entry_t
*edir
)
408 for (ptr
= dir
; ptr
< edir
; ptr
++) {
410 free_swap_and_cache(*ptr
);
411 *ptr
= (swp_entry_t
){0};
418 static int shmem_map_and_free_swp(struct page
*subdir
,
419 int offset
, int limit
, struct page
***dir
)
424 ptr
= shmem_swp_map(subdir
);
425 for (; offset
< limit
; offset
+= LATENCY_LIMIT
) {
426 int size
= limit
- offset
;
427 if (size
> LATENCY_LIMIT
)
428 size
= LATENCY_LIMIT
;
429 freed
+= shmem_free_swp(ptr
+offset
, ptr
+offset
+size
);
430 if (need_resched()) {
431 shmem_swp_unmap(ptr
);
433 shmem_dir_unmap(*dir
);
437 ptr
= shmem_swp_map(subdir
);
440 shmem_swp_unmap(ptr
);
444 static void shmem_free_pages(struct list_head
*next
)
450 page
= container_of(next
, struct page
, lru
);
452 shmem_dir_free(page
);
454 if (freed
>= LATENCY_LIMIT
) {
461 static void shmem_truncate_range(struct inode
*inode
, loff_t start
, loff_t end
)
463 struct shmem_inode_info
*info
= SHMEM_I(inode
);
468 unsigned long diroff
;
474 LIST_HEAD(pages_to_free
);
475 long nr_pages_to_free
= 0;
476 long nr_swaps_freed
= 0;
481 inode
->i_ctime
= inode
->i_mtime
= CURRENT_TIME
;
482 idx
= (start
+ PAGE_CACHE_SIZE
- 1) >> PAGE_CACHE_SHIFT
;
483 if (idx
>= info
->next_index
)
486 spin_lock(&info
->lock
);
487 info
->flags
|= SHMEM_TRUNCATE
;
488 if (likely(end
== (loff_t
) -1)) {
489 limit
= info
->next_index
;
490 info
->next_index
= idx
;
492 limit
= (end
+ PAGE_CACHE_SIZE
- 1) >> PAGE_CACHE_SHIFT
;
493 if (limit
> info
->next_index
)
494 limit
= info
->next_index
;
498 topdir
= info
->i_indirect
;
499 if (topdir
&& idx
<= SHMEM_NR_DIRECT
&& !punch_hole
) {
500 info
->i_indirect
= NULL
;
502 list_add(&topdir
->lru
, &pages_to_free
);
504 spin_unlock(&info
->lock
);
506 if (info
->swapped
&& idx
< SHMEM_NR_DIRECT
) {
507 ptr
= info
->i_direct
;
509 if (size
> SHMEM_NR_DIRECT
)
510 size
= SHMEM_NR_DIRECT
;
511 nr_swaps_freed
= shmem_free_swp(ptr
+idx
, ptr
+size
);
516 BUG_ON(limit
<= SHMEM_NR_DIRECT
);
517 limit
-= SHMEM_NR_DIRECT
;
518 idx
= (idx
> SHMEM_NR_DIRECT
)? (idx
- SHMEM_NR_DIRECT
): 0;
519 offset
= idx
% ENTRIES_PER_PAGE
;
522 dir
= shmem_dir_map(topdir
);
523 stage
= ENTRIES_PER_PAGEPAGE
/2;
524 if (idx
< ENTRIES_PER_PAGEPAGE
/2) {
526 diroff
= idx
/ENTRIES_PER_PAGE
;
528 dir
+= ENTRIES_PER_PAGE
/2;
529 dir
+= (idx
- ENTRIES_PER_PAGEPAGE
/2)/ENTRIES_PER_PAGEPAGE
;
531 stage
+= ENTRIES_PER_PAGEPAGE
;
534 diroff
= ((idx
- ENTRIES_PER_PAGEPAGE
/2) %
535 ENTRIES_PER_PAGEPAGE
) / ENTRIES_PER_PAGE
;
536 if (!diroff
&& !offset
) {
539 list_add(&middir
->lru
, &pages_to_free
);
541 shmem_dir_unmap(dir
);
542 dir
= shmem_dir_map(middir
);
550 for (; idx
< limit
; idx
+= ENTRIES_PER_PAGE
, diroff
++) {
551 if (unlikely(idx
== stage
)) {
552 shmem_dir_unmap(dir
);
553 dir
= shmem_dir_map(topdir
) +
554 ENTRIES_PER_PAGE
/2 + idx
/ENTRIES_PER_PAGEPAGE
;
557 idx
+= ENTRIES_PER_PAGEPAGE
;
561 stage
= idx
+ ENTRIES_PER_PAGEPAGE
;
565 list_add(&middir
->lru
, &pages_to_free
);
566 shmem_dir_unmap(dir
);
568 dir
= shmem_dir_map(middir
);
571 subdir
= dir
[diroff
];
572 if (subdir
&& page_private(subdir
)) {
574 if (size
> ENTRIES_PER_PAGE
)
575 size
= ENTRIES_PER_PAGE
;
576 freed
= shmem_map_and_free_swp(subdir
,
579 dir
= shmem_dir_map(middir
);
580 nr_swaps_freed
+= freed
;
582 spin_lock(&info
->lock
);
583 set_page_private(subdir
, page_private(subdir
) - freed
);
585 spin_unlock(&info
->lock
);
587 BUG_ON(page_private(subdir
) > offset
);
591 else if (subdir
&& !page_private(subdir
)) {
594 list_add(&subdir
->lru
, &pages_to_free
);
598 shmem_dir_unmap(dir
);
600 if (inode
->i_mapping
->nrpages
&& (info
->flags
& SHMEM_PAGEIN
)) {
602 * Call truncate_inode_pages again: racing shmem_unuse_inode
603 * may have swizzled a page in from swap since vmtruncate or
604 * generic_delete_inode did it, before we lowered next_index.
605 * Also, though shmem_getpage checks i_size before adding to
606 * cache, no recheck after: so fix the narrow window there too.
608 truncate_inode_pages_range(inode
->i_mapping
, start
, end
);
611 spin_lock(&info
->lock
);
612 info
->flags
&= ~SHMEM_TRUNCATE
;
613 info
->swapped
-= nr_swaps_freed
;
614 if (nr_pages_to_free
)
615 shmem_free_blocks(inode
, nr_pages_to_free
);
616 shmem_recalc_inode(inode
);
617 spin_unlock(&info
->lock
);
620 * Empty swap vector directory pages to be freed?
622 if (!list_empty(&pages_to_free
)) {
623 pages_to_free
.prev
->next
= NULL
;
624 shmem_free_pages(pages_to_free
.next
);
628 static void shmem_truncate(struct inode
*inode
)
630 shmem_truncate_range(inode
, inode
->i_size
, (loff_t
)-1);
633 static int shmem_notify_change(struct dentry
*dentry
, struct iattr
*attr
)
635 struct inode
*inode
= dentry
->d_inode
;
636 struct page
*page
= NULL
;
639 if (attr
->ia_valid
& ATTR_SIZE
) {
640 if (attr
->ia_size
< inode
->i_size
) {
642 * If truncating down to a partial page, then
643 * if that page is already allocated, hold it
644 * in memory until the truncation is over, so
645 * truncate_partial_page cannnot miss it were
646 * it assigned to swap.
648 if (attr
->ia_size
& (PAGE_CACHE_SIZE
-1)) {
649 (void) shmem_getpage(inode
,
650 attr
->ia_size
>>PAGE_CACHE_SHIFT
,
651 &page
, SGP_READ
, NULL
);
654 * Reset SHMEM_PAGEIN flag so that shmem_truncate can
655 * detect if any pages might have been added to cache
656 * after truncate_inode_pages. But we needn't bother
657 * if it's being fully truncated to zero-length: the
658 * nrpages check is efficient enough in that case.
661 struct shmem_inode_info
*info
= SHMEM_I(inode
);
662 spin_lock(&info
->lock
);
663 info
->flags
&= ~SHMEM_PAGEIN
;
664 spin_unlock(&info
->lock
);
669 error
= inode_change_ok(inode
, attr
);
671 error
= inode_setattr(inode
, attr
);
673 page_cache_release(page
);
677 static void shmem_delete_inode(struct inode
*inode
)
679 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
680 struct shmem_inode_info
*info
= SHMEM_I(inode
);
682 if (inode
->i_op
->truncate
== shmem_truncate
) {
683 truncate_inode_pages(inode
->i_mapping
, 0);
684 shmem_unacct_size(info
->flags
, inode
->i_size
);
686 shmem_truncate(inode
);
687 if (!list_empty(&info
->swaplist
)) {
688 spin_lock(&shmem_swaplist_lock
);
689 list_del_init(&info
->swaplist
);
690 spin_unlock(&shmem_swaplist_lock
);
693 BUG_ON(inode
->i_blocks
);
694 if (sbinfo
->max_inodes
) {
695 spin_lock(&sbinfo
->stat_lock
);
696 sbinfo
->free_inodes
++;
697 spin_unlock(&sbinfo
->stat_lock
);
702 static inline int shmem_find_swp(swp_entry_t entry
, swp_entry_t
*dir
, swp_entry_t
*edir
)
706 for (ptr
= dir
; ptr
< edir
; ptr
++) {
707 if (ptr
->val
== entry
.val
)
713 static int shmem_unuse_inode(struct shmem_inode_info
*info
, swp_entry_t entry
, struct page
*page
)
726 ptr
= info
->i_direct
;
727 spin_lock(&info
->lock
);
728 limit
= info
->next_index
;
730 if (size
> SHMEM_NR_DIRECT
)
731 size
= SHMEM_NR_DIRECT
;
732 offset
= shmem_find_swp(entry
, ptr
, ptr
+size
);
734 shmem_swp_balance_unmap();
737 if (!info
->i_indirect
)
740 dir
= shmem_dir_map(info
->i_indirect
);
741 stage
= SHMEM_NR_DIRECT
+ ENTRIES_PER_PAGEPAGE
/2;
743 for (idx
= SHMEM_NR_DIRECT
; idx
< limit
; idx
+= ENTRIES_PER_PAGE
, dir
++) {
744 if (unlikely(idx
== stage
)) {
745 shmem_dir_unmap(dir
-1);
746 dir
= shmem_dir_map(info
->i_indirect
) +
747 ENTRIES_PER_PAGE
/2 + idx
/ENTRIES_PER_PAGEPAGE
;
750 idx
+= ENTRIES_PER_PAGEPAGE
;
754 stage
= idx
+ ENTRIES_PER_PAGEPAGE
;
756 shmem_dir_unmap(dir
);
757 dir
= shmem_dir_map(subdir
);
760 if (subdir
&& page_private(subdir
)) {
761 ptr
= shmem_swp_map(subdir
);
763 if (size
> ENTRIES_PER_PAGE
)
764 size
= ENTRIES_PER_PAGE
;
765 offset
= shmem_find_swp(entry
, ptr
, ptr
+size
);
767 shmem_dir_unmap(dir
);
770 shmem_swp_unmap(ptr
);
774 shmem_dir_unmap(dir
-1);
776 spin_unlock(&info
->lock
);
780 inode
= &info
->vfs_inode
;
781 if (move_from_swap_cache(page
, idx
, inode
->i_mapping
) == 0) {
782 info
->flags
|= SHMEM_PAGEIN
;
783 shmem_swp_set(info
, ptr
+ offset
, 0);
785 shmem_swp_unmap(ptr
);
786 spin_unlock(&info
->lock
);
788 * Decrement swap count even when the entry is left behind:
789 * try_to_unuse will skip over mms, then reincrement count.
796 * shmem_unuse() search for an eventually swapped out shmem page.
798 int shmem_unuse(swp_entry_t entry
, struct page
*page
)
800 struct list_head
*p
, *next
;
801 struct shmem_inode_info
*info
;
804 spin_lock(&shmem_swaplist_lock
);
805 list_for_each_safe(p
, next
, &shmem_swaplist
) {
806 info
= list_entry(p
, struct shmem_inode_info
, swaplist
);
808 list_del_init(&info
->swaplist
);
809 else if (shmem_unuse_inode(info
, entry
, page
)) {
810 /* move head to start search for next from here */
811 list_move_tail(&shmem_swaplist
, &info
->swaplist
);
816 spin_unlock(&shmem_swaplist_lock
);
821 * Move the page from the page cache to the swap cache.
823 static int shmem_writepage(struct page
*page
, struct writeback_control
*wbc
)
825 struct shmem_inode_info
*info
;
826 swp_entry_t
*entry
, swap
;
827 struct address_space
*mapping
;
831 BUG_ON(!PageLocked(page
));
832 BUG_ON(page_mapped(page
));
834 mapping
= page
->mapping
;
836 inode
= mapping
->host
;
837 info
= SHMEM_I(inode
);
838 if (info
->flags
& VM_LOCKED
)
840 swap
= get_swap_page();
844 spin_lock(&info
->lock
);
845 shmem_recalc_inode(inode
);
846 if (index
>= info
->next_index
) {
847 BUG_ON(!(info
->flags
& SHMEM_TRUNCATE
));
850 entry
= shmem_swp_entry(info
, index
, NULL
);
854 if (move_to_swap_cache(page
, swap
) == 0) {
855 shmem_swp_set(info
, entry
, swap
.val
);
856 shmem_swp_unmap(entry
);
857 spin_unlock(&info
->lock
);
858 if (list_empty(&info
->swaplist
)) {
859 spin_lock(&shmem_swaplist_lock
);
860 /* move instead of add in case we're racing */
861 list_move_tail(&info
->swaplist
, &shmem_swaplist
);
862 spin_unlock(&shmem_swaplist_lock
);
868 shmem_swp_unmap(entry
);
870 spin_unlock(&info
->lock
);
873 set_page_dirty(page
);
874 return AOP_WRITEPAGE_ACTIVATE
; /* Return with the page locked */
878 static int shmem_parse_mpol(char *value
, int *policy
, nodemask_t
*policy_nodes
)
880 char *nodelist
= strchr(value
, ':');
884 /* NUL-terminate policy string */
886 if (nodelist_parse(nodelist
, *policy_nodes
))
889 if (!strcmp(value
, "default")) {
890 *policy
= MPOL_DEFAULT
;
891 /* Don't allow a nodelist */
894 } else if (!strcmp(value
, "prefer")) {
895 *policy
= MPOL_PREFERRED
;
896 /* Insist on a nodelist of one node only */
898 char *rest
= nodelist
;
899 while (isdigit(*rest
))
904 } else if (!strcmp(value
, "bind")) {
906 /* Insist on a nodelist */
909 } else if (!strcmp(value
, "interleave")) {
910 *policy
= MPOL_INTERLEAVE
;
911 /* Default to nodes online if no nodelist */
913 *policy_nodes
= node_online_map
;
917 /* Restore string for error message */
923 static struct page
*shmem_swapin_async(struct shared_policy
*p
,
924 swp_entry_t entry
, unsigned long idx
)
927 struct vm_area_struct pvma
;
929 /* Create a pseudo vma that just contains the policy */
930 memset(&pvma
, 0, sizeof(struct vm_area_struct
));
931 pvma
.vm_end
= PAGE_SIZE
;
933 pvma
.vm_policy
= mpol_shared_policy_lookup(p
, idx
);
934 page
= read_swap_cache_async(entry
, &pvma
, 0);
935 mpol_free(pvma
.vm_policy
);
939 struct page
*shmem_swapin(struct shmem_inode_info
*info
, swp_entry_t entry
,
942 struct shared_policy
*p
= &info
->policy
;
945 unsigned long offset
;
947 num
= valid_swaphandles(entry
, &offset
);
948 for (i
= 0; i
< num
; offset
++, i
++) {
949 page
= shmem_swapin_async(p
,
950 swp_entry(swp_type(entry
), offset
), idx
);
953 page_cache_release(page
);
955 lru_add_drain(); /* Push any new pages onto the LRU now */
956 return shmem_swapin_async(p
, entry
, idx
);
960 shmem_alloc_page(gfp_t gfp
, struct shmem_inode_info
*info
,
963 struct vm_area_struct pvma
;
966 memset(&pvma
, 0, sizeof(struct vm_area_struct
));
967 pvma
.vm_policy
= mpol_shared_policy_lookup(&info
->policy
, idx
);
969 pvma
.vm_end
= PAGE_SIZE
;
970 page
= alloc_page_vma(gfp
| __GFP_ZERO
, &pvma
, 0);
971 mpol_free(pvma
.vm_policy
);
975 static inline int shmem_parse_mpol(char *value
, int *policy
, nodemask_t
*policy_nodes
)
980 static inline struct page
*
981 shmem_swapin(struct shmem_inode_info
*info
,swp_entry_t entry
,unsigned long idx
)
983 swapin_readahead(entry
, 0, NULL
);
984 return read_swap_cache_async(entry
, NULL
, 0);
987 static inline struct page
*
988 shmem_alloc_page(gfp_t gfp
,struct shmem_inode_info
*info
, unsigned long idx
)
990 return alloc_page(gfp
| __GFP_ZERO
);
995 * shmem_getpage - either get the page from swap or allocate a new one
997 * If we allocate a new one we do not mark it dirty. That's up to the
998 * vm. If we swap it in we mark it dirty since we also free the swap
999 * entry since a page cannot live in both the swap and page cache
1001 static int shmem_getpage(struct inode
*inode
, unsigned long idx
,
1002 struct page
**pagep
, enum sgp_type sgp
, int *type
)
1004 struct address_space
*mapping
= inode
->i_mapping
;
1005 struct shmem_inode_info
*info
= SHMEM_I(inode
);
1006 struct shmem_sb_info
*sbinfo
;
1007 struct page
*filepage
= *pagep
;
1008 struct page
*swappage
;
1013 if (idx
>= SHMEM_MAX_INDEX
)
1016 * Normally, filepage is NULL on entry, and either found
1017 * uptodate immediately, or allocated and zeroed, or read
1018 * in under swappage, which is then assigned to filepage.
1019 * But shmem_prepare_write passes in a locked filepage,
1020 * which may be found not uptodate by other callers too,
1021 * and may need to be copied from the swappage read in.
1025 filepage
= find_lock_page(mapping
, idx
);
1026 if (filepage
&& PageUptodate(filepage
))
1029 if (sgp
== SGP_QUICK
)
1032 spin_lock(&info
->lock
);
1033 shmem_recalc_inode(inode
);
1034 entry
= shmem_swp_alloc(info
, idx
, sgp
);
1035 if (IS_ERR(entry
)) {
1036 spin_unlock(&info
->lock
);
1037 error
= PTR_ERR(entry
);
1043 /* Look it up and read it in.. */
1044 swappage
= lookup_swap_cache(swap
);
1046 shmem_swp_unmap(entry
);
1047 spin_unlock(&info
->lock
);
1048 /* here we actually do the io */
1049 if (type
&& *type
== VM_FAULT_MINOR
) {
1050 inc_page_state(pgmajfault
);
1051 *type
= VM_FAULT_MAJOR
;
1053 swappage
= shmem_swapin(info
, swap
, idx
);
1055 spin_lock(&info
->lock
);
1056 entry
= shmem_swp_alloc(info
, idx
, sgp
);
1058 error
= PTR_ERR(entry
);
1060 if (entry
->val
== swap
.val
)
1062 shmem_swp_unmap(entry
);
1064 spin_unlock(&info
->lock
);
1069 wait_on_page_locked(swappage
);
1070 page_cache_release(swappage
);
1074 /* We have to do this with page locked to prevent races */
1075 if (TestSetPageLocked(swappage
)) {
1076 shmem_swp_unmap(entry
);
1077 spin_unlock(&info
->lock
);
1078 wait_on_page_locked(swappage
);
1079 page_cache_release(swappage
);
1082 if (!PageSwapCache(swappage
)) {
1083 /* Page migration has occured */
1084 shmem_swp_unmap(entry
);
1085 spin_unlock(&info
->lock
);
1086 unlock_page(swappage
);
1087 page_cache_release(swappage
);
1090 if (PageWriteback(swappage
)) {
1091 shmem_swp_unmap(entry
);
1092 spin_unlock(&info
->lock
);
1093 wait_on_page_writeback(swappage
);
1094 unlock_page(swappage
);
1095 page_cache_release(swappage
);
1098 if (!PageUptodate(swappage
)) {
1099 shmem_swp_unmap(entry
);
1100 spin_unlock(&info
->lock
);
1101 unlock_page(swappage
);
1102 page_cache_release(swappage
);
1108 shmem_swp_set(info
, entry
, 0);
1109 shmem_swp_unmap(entry
);
1110 delete_from_swap_cache(swappage
);
1111 spin_unlock(&info
->lock
);
1112 copy_highpage(filepage
, swappage
);
1113 unlock_page(swappage
);
1114 page_cache_release(swappage
);
1115 flush_dcache_page(filepage
);
1116 SetPageUptodate(filepage
);
1117 set_page_dirty(filepage
);
1119 } else if (!(error
= move_from_swap_cache(
1120 swappage
, idx
, mapping
))) {
1121 info
->flags
|= SHMEM_PAGEIN
;
1122 shmem_swp_set(info
, entry
, 0);
1123 shmem_swp_unmap(entry
);
1124 spin_unlock(&info
->lock
);
1125 filepage
= swappage
;
1128 shmem_swp_unmap(entry
);
1129 spin_unlock(&info
->lock
);
1130 unlock_page(swappage
);
1131 page_cache_release(swappage
);
1132 if (error
== -ENOMEM
) {
1133 /* let kswapd refresh zone for GFP_ATOMICs */
1134 blk_congestion_wait(WRITE
, HZ
/50);
1138 } else if (sgp
== SGP_READ
&& !filepage
) {
1139 shmem_swp_unmap(entry
);
1140 filepage
= find_get_page(mapping
, idx
);
1142 (!PageUptodate(filepage
) || TestSetPageLocked(filepage
))) {
1143 spin_unlock(&info
->lock
);
1144 wait_on_page_locked(filepage
);
1145 page_cache_release(filepage
);
1149 spin_unlock(&info
->lock
);
1151 shmem_swp_unmap(entry
);
1152 sbinfo
= SHMEM_SB(inode
->i_sb
);
1153 if (sbinfo
->max_blocks
) {
1154 spin_lock(&sbinfo
->stat_lock
);
1155 if (sbinfo
->free_blocks
== 0 ||
1156 shmem_acct_block(info
->flags
)) {
1157 spin_unlock(&sbinfo
->stat_lock
);
1158 spin_unlock(&info
->lock
);
1162 sbinfo
->free_blocks
--;
1163 inode
->i_blocks
+= BLOCKS_PER_PAGE
;
1164 spin_unlock(&sbinfo
->stat_lock
);
1165 } else if (shmem_acct_block(info
->flags
)) {
1166 spin_unlock(&info
->lock
);
1172 spin_unlock(&info
->lock
);
1173 filepage
= shmem_alloc_page(mapping_gfp_mask(mapping
),
1177 shmem_unacct_blocks(info
->flags
, 1);
1178 shmem_free_blocks(inode
, 1);
1183 spin_lock(&info
->lock
);
1184 entry
= shmem_swp_alloc(info
, idx
, sgp
);
1186 error
= PTR_ERR(entry
);
1189 shmem_swp_unmap(entry
);
1191 if (error
|| swap
.val
|| 0 != add_to_page_cache_lru(
1192 filepage
, mapping
, idx
, GFP_ATOMIC
)) {
1193 spin_unlock(&info
->lock
);
1194 page_cache_release(filepage
);
1195 shmem_unacct_blocks(info
->flags
, 1);
1196 shmem_free_blocks(inode
, 1);
1202 info
->flags
|= SHMEM_PAGEIN
;
1206 spin_unlock(&info
->lock
);
1207 flush_dcache_page(filepage
);
1208 SetPageUptodate(filepage
);
1211 if (*pagep
!= filepage
) {
1212 unlock_page(filepage
);
1218 if (*pagep
!= filepage
) {
1219 unlock_page(filepage
);
1220 page_cache_release(filepage
);
1225 struct page
*shmem_nopage(struct vm_area_struct
*vma
, unsigned long address
, int *type
)
1227 struct inode
*inode
= vma
->vm_file
->f_dentry
->d_inode
;
1228 struct page
*page
= NULL
;
1232 idx
= (address
- vma
->vm_start
) >> PAGE_SHIFT
;
1233 idx
+= vma
->vm_pgoff
;
1234 idx
>>= PAGE_CACHE_SHIFT
- PAGE_SHIFT
;
1235 if (((loff_t
) idx
<< PAGE_CACHE_SHIFT
) >= i_size_read(inode
))
1236 return NOPAGE_SIGBUS
;
1238 error
= shmem_getpage(inode
, idx
, &page
, SGP_CACHE
, type
);
1240 return (error
== -ENOMEM
)? NOPAGE_OOM
: NOPAGE_SIGBUS
;
1242 mark_page_accessed(page
);
1246 static int shmem_populate(struct vm_area_struct
*vma
,
1247 unsigned long addr
, unsigned long len
,
1248 pgprot_t prot
, unsigned long pgoff
, int nonblock
)
1250 struct inode
*inode
= vma
->vm_file
->f_dentry
->d_inode
;
1251 struct mm_struct
*mm
= vma
->vm_mm
;
1252 enum sgp_type sgp
= nonblock
? SGP_QUICK
: SGP_CACHE
;
1255 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >> PAGE_SHIFT
;
1256 if (pgoff
>= size
|| pgoff
+ (len
>> PAGE_SHIFT
) > size
)
1259 while ((long) len
> 0) {
1260 struct page
*page
= NULL
;
1263 * Will need changing if PAGE_CACHE_SIZE != PAGE_SIZE
1265 err
= shmem_getpage(inode
, pgoff
, &page
, sgp
, NULL
);
1268 /* Page may still be null, but only if nonblock was set. */
1270 mark_page_accessed(page
);
1271 err
= install_page(mm
, vma
, addr
, page
, prot
);
1273 page_cache_release(page
);
1276 } else if (vma
->vm_flags
& VM_NONLINEAR
) {
1277 /* No page was found just because we can't read it in
1278 * now (being here implies nonblock != 0), but the page
1279 * may exist, so set the PTE to fault it in later. */
1280 err
= install_file_pte(mm
, vma
, addr
, pgoff
, prot
);
1293 int shmem_set_policy(struct vm_area_struct
*vma
, struct mempolicy
*new)
1295 struct inode
*i
= vma
->vm_file
->f_dentry
->d_inode
;
1296 return mpol_set_shared_policy(&SHMEM_I(i
)->policy
, vma
, new);
1300 shmem_get_policy(struct vm_area_struct
*vma
, unsigned long addr
)
1302 struct inode
*i
= vma
->vm_file
->f_dentry
->d_inode
;
1305 idx
= ((addr
- vma
->vm_start
) >> PAGE_SHIFT
) + vma
->vm_pgoff
;
1306 return mpol_shared_policy_lookup(&SHMEM_I(i
)->policy
, idx
);
1310 int shmem_lock(struct file
*file
, int lock
, struct user_struct
*user
)
1312 struct inode
*inode
= file
->f_dentry
->d_inode
;
1313 struct shmem_inode_info
*info
= SHMEM_I(inode
);
1314 int retval
= -ENOMEM
;
1316 spin_lock(&info
->lock
);
1317 if (lock
&& !(info
->flags
& VM_LOCKED
)) {
1318 if (!user_shm_lock(inode
->i_size
, user
))
1320 info
->flags
|= VM_LOCKED
;
1322 if (!lock
&& (info
->flags
& VM_LOCKED
) && user
) {
1323 user_shm_unlock(inode
->i_size
, user
);
1324 info
->flags
&= ~VM_LOCKED
;
1328 spin_unlock(&info
->lock
);
1332 int shmem_mmap(struct file
*file
, struct vm_area_struct
*vma
)
1334 file_accessed(file
);
1335 vma
->vm_ops
= &shmem_vm_ops
;
1339 static struct inode
*
1340 shmem_get_inode(struct super_block
*sb
, int mode
, dev_t dev
)
1342 struct inode
*inode
;
1343 struct shmem_inode_info
*info
;
1344 struct shmem_sb_info
*sbinfo
= SHMEM_SB(sb
);
1346 if (sbinfo
->max_inodes
) {
1347 spin_lock(&sbinfo
->stat_lock
);
1348 if (!sbinfo
->free_inodes
) {
1349 spin_unlock(&sbinfo
->stat_lock
);
1352 sbinfo
->free_inodes
--;
1353 spin_unlock(&sbinfo
->stat_lock
);
1356 inode
= new_inode(sb
);
1358 inode
->i_mode
= mode
;
1359 inode
->i_uid
= current
->fsuid
;
1360 inode
->i_gid
= current
->fsgid
;
1361 inode
->i_blksize
= PAGE_CACHE_SIZE
;
1362 inode
->i_blocks
= 0;
1363 inode
->i_mapping
->a_ops
= &shmem_aops
;
1364 inode
->i_mapping
->backing_dev_info
= &shmem_backing_dev_info
;
1365 inode
->i_atime
= inode
->i_mtime
= inode
->i_ctime
= CURRENT_TIME
;
1366 info
= SHMEM_I(inode
);
1367 memset(info
, 0, (char *)inode
- (char *)info
);
1368 spin_lock_init(&info
->lock
);
1369 INIT_LIST_HEAD(&info
->swaplist
);
1371 switch (mode
& S_IFMT
) {
1373 init_special_inode(inode
, mode
, dev
);
1376 inode
->i_op
= &shmem_inode_operations
;
1377 inode
->i_fop
= &shmem_file_operations
;
1378 mpol_shared_policy_init(&info
->policy
, sbinfo
->policy
,
1379 &sbinfo
->policy_nodes
);
1383 /* Some things misbehave if size == 0 on a directory */
1384 inode
->i_size
= 2 * BOGO_DIRENT_SIZE
;
1385 inode
->i_op
= &shmem_dir_inode_operations
;
1386 inode
->i_fop
= &simple_dir_operations
;
1390 * Must not load anything in the rbtree,
1391 * mpol_free_shared_policy will not be called.
1393 mpol_shared_policy_init(&info
->policy
, MPOL_DEFAULT
,
1397 } else if (sbinfo
->max_inodes
) {
1398 spin_lock(&sbinfo
->stat_lock
);
1399 sbinfo
->free_inodes
++;
1400 spin_unlock(&sbinfo
->stat_lock
);
1406 static struct inode_operations shmem_symlink_inode_operations
;
1407 static struct inode_operations shmem_symlink_inline_operations
;
1410 * Normally tmpfs makes no use of shmem_prepare_write, but it
1411 * lets a tmpfs file be used read-write below the loop driver.
1414 shmem_prepare_write(struct file
*file
, struct page
*page
, unsigned offset
, unsigned to
)
1416 struct inode
*inode
= page
->mapping
->host
;
1417 return shmem_getpage(inode
, page
->index
, &page
, SGP_WRITE
, NULL
);
1421 shmem_file_write(struct file
*file
, const char __user
*buf
, size_t count
, loff_t
*ppos
)
1423 struct inode
*inode
= file
->f_dentry
->d_inode
;
1425 unsigned long written
;
1428 if ((ssize_t
) count
< 0)
1431 if (!access_ok(VERIFY_READ
, buf
, count
))
1434 mutex_lock(&inode
->i_mutex
);
1439 err
= generic_write_checks(file
, &pos
, &count
, 0);
1443 err
= remove_suid(file
->f_dentry
);
1447 inode
->i_ctime
= inode
->i_mtime
= CURRENT_TIME
;
1450 struct page
*page
= NULL
;
1451 unsigned long bytes
, index
, offset
;
1455 offset
= (pos
& (PAGE_CACHE_SIZE
-1)); /* Within page */
1456 index
= pos
>> PAGE_CACHE_SHIFT
;
1457 bytes
= PAGE_CACHE_SIZE
- offset
;
1462 * We don't hold page lock across copy from user -
1463 * what would it guard against? - so no deadlock here.
1464 * But it still may be a good idea to prefault below.
1467 err
= shmem_getpage(inode
, index
, &page
, SGP_WRITE
, NULL
);
1472 if (PageHighMem(page
)) {
1473 volatile unsigned char dummy
;
1474 __get_user(dummy
, buf
);
1475 __get_user(dummy
, buf
+ bytes
- 1);
1477 kaddr
= kmap_atomic(page
, KM_USER0
);
1478 left
= __copy_from_user_inatomic(kaddr
+ offset
,
1480 kunmap_atomic(kaddr
, KM_USER0
);
1484 left
= __copy_from_user(kaddr
+ offset
, buf
, bytes
);
1492 if (pos
> inode
->i_size
)
1493 i_size_write(inode
, pos
);
1495 flush_dcache_page(page
);
1496 set_page_dirty(page
);
1497 mark_page_accessed(page
);
1498 page_cache_release(page
);
1508 * Our dirty pages are not counted in nr_dirty,
1509 * and we do not attempt to balance dirty pages.
1519 mutex_unlock(&inode
->i_mutex
);
1523 static void do_shmem_file_read(struct file
*filp
, loff_t
*ppos
, read_descriptor_t
*desc
, read_actor_t actor
)
1525 struct inode
*inode
= filp
->f_dentry
->d_inode
;
1526 struct address_space
*mapping
= inode
->i_mapping
;
1527 unsigned long index
, offset
;
1529 index
= *ppos
>> PAGE_CACHE_SHIFT
;
1530 offset
= *ppos
& ~PAGE_CACHE_MASK
;
1533 struct page
*page
= NULL
;
1534 unsigned long end_index
, nr
, ret
;
1535 loff_t i_size
= i_size_read(inode
);
1537 end_index
= i_size
>> PAGE_CACHE_SHIFT
;
1538 if (index
> end_index
)
1540 if (index
== end_index
) {
1541 nr
= i_size
& ~PAGE_CACHE_MASK
;
1546 desc
->error
= shmem_getpage(inode
, index
, &page
, SGP_READ
, NULL
);
1548 if (desc
->error
== -EINVAL
)
1554 * We must evaluate after, since reads (unlike writes)
1555 * are called without i_mutex protection against truncate
1557 nr
= PAGE_CACHE_SIZE
;
1558 i_size
= i_size_read(inode
);
1559 end_index
= i_size
>> PAGE_CACHE_SHIFT
;
1560 if (index
== end_index
) {
1561 nr
= i_size
& ~PAGE_CACHE_MASK
;
1564 page_cache_release(page
);
1572 * If users can be writing to this page using arbitrary
1573 * virtual addresses, take care about potential aliasing
1574 * before reading the page on the kernel side.
1576 if (mapping_writably_mapped(mapping
))
1577 flush_dcache_page(page
);
1579 * Mark the page accessed if we read the beginning.
1582 mark_page_accessed(page
);
1584 page
= ZERO_PAGE(0);
1585 page_cache_get(page
);
1589 * Ok, we have the page, and it's up-to-date, so
1590 * now we can copy it to user space...
1592 * The actor routine returns how many bytes were actually used..
1593 * NOTE! This may not be the same as how much of a user buffer
1594 * we filled up (we may be padding etc), so we can only update
1595 * "pos" here (the actor routine has to update the user buffer
1596 * pointers and the remaining count).
1598 ret
= actor(desc
, page
, offset
, nr
);
1600 index
+= offset
>> PAGE_CACHE_SHIFT
;
1601 offset
&= ~PAGE_CACHE_MASK
;
1603 page_cache_release(page
);
1604 if (ret
!= nr
|| !desc
->count
)
1610 *ppos
= ((loff_t
) index
<< PAGE_CACHE_SHIFT
) + offset
;
1611 file_accessed(filp
);
1614 static ssize_t
shmem_file_read(struct file
*filp
, char __user
*buf
, size_t count
, loff_t
*ppos
)
1616 read_descriptor_t desc
;
1618 if ((ssize_t
) count
< 0)
1620 if (!access_ok(VERIFY_WRITE
, buf
, count
))
1630 do_shmem_file_read(filp
, ppos
, &desc
, file_read_actor
);
1632 return desc
.written
;
1636 static ssize_t
shmem_file_sendfile(struct file
*in_file
, loff_t
*ppos
,
1637 size_t count
, read_actor_t actor
, void *target
)
1639 read_descriptor_t desc
;
1646 desc
.arg
.data
= target
;
1649 do_shmem_file_read(in_file
, ppos
, &desc
, actor
);
1651 return desc
.written
;
1655 static int shmem_statfs(struct super_block
*sb
, struct kstatfs
*buf
)
1657 struct shmem_sb_info
*sbinfo
= SHMEM_SB(sb
);
1659 buf
->f_type
= TMPFS_MAGIC
;
1660 buf
->f_bsize
= PAGE_CACHE_SIZE
;
1661 buf
->f_namelen
= NAME_MAX
;
1662 spin_lock(&sbinfo
->stat_lock
);
1663 if (sbinfo
->max_blocks
) {
1664 buf
->f_blocks
= sbinfo
->max_blocks
;
1665 buf
->f_bavail
= buf
->f_bfree
= sbinfo
->free_blocks
;
1667 if (sbinfo
->max_inodes
) {
1668 buf
->f_files
= sbinfo
->max_inodes
;
1669 buf
->f_ffree
= sbinfo
->free_inodes
;
1671 /* else leave those fields 0 like simple_statfs */
1672 spin_unlock(&sbinfo
->stat_lock
);
1677 * File creation. Allocate an inode, and we're done..
1680 shmem_mknod(struct inode
*dir
, struct dentry
*dentry
, int mode
, dev_t dev
)
1682 struct inode
*inode
= shmem_get_inode(dir
->i_sb
, mode
, dev
);
1683 int error
= -ENOSPC
;
1686 error
= security_inode_init_security(inode
, dir
, NULL
, NULL
,
1689 if (error
!= -EOPNOTSUPP
) {
1695 if (dir
->i_mode
& S_ISGID
) {
1696 inode
->i_gid
= dir
->i_gid
;
1698 inode
->i_mode
|= S_ISGID
;
1700 dir
->i_size
+= BOGO_DIRENT_SIZE
;
1701 dir
->i_ctime
= dir
->i_mtime
= CURRENT_TIME
;
1702 d_instantiate(dentry
, inode
);
1703 dget(dentry
); /* Extra count - pin the dentry in core */
1708 static int shmem_mkdir(struct inode
*dir
, struct dentry
*dentry
, int mode
)
1712 if ((error
= shmem_mknod(dir
, dentry
, mode
| S_IFDIR
, 0)))
1718 static int shmem_create(struct inode
*dir
, struct dentry
*dentry
, int mode
,
1719 struct nameidata
*nd
)
1721 return shmem_mknod(dir
, dentry
, mode
| S_IFREG
, 0);
1727 static int shmem_link(struct dentry
*old_dentry
, struct inode
*dir
, struct dentry
*dentry
)
1729 struct inode
*inode
= old_dentry
->d_inode
;
1730 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
1733 * No ordinary (disk based) filesystem counts links as inodes;
1734 * but each new link needs a new dentry, pinning lowmem, and
1735 * tmpfs dentries cannot be pruned until they are unlinked.
1737 if (sbinfo
->max_inodes
) {
1738 spin_lock(&sbinfo
->stat_lock
);
1739 if (!sbinfo
->free_inodes
) {
1740 spin_unlock(&sbinfo
->stat_lock
);
1743 sbinfo
->free_inodes
--;
1744 spin_unlock(&sbinfo
->stat_lock
);
1747 dir
->i_size
+= BOGO_DIRENT_SIZE
;
1748 inode
->i_ctime
= dir
->i_ctime
= dir
->i_mtime
= CURRENT_TIME
;
1750 atomic_inc(&inode
->i_count
); /* New dentry reference */
1751 dget(dentry
); /* Extra pinning count for the created dentry */
1752 d_instantiate(dentry
, inode
);
1756 static int shmem_unlink(struct inode
*dir
, struct dentry
*dentry
)
1758 struct inode
*inode
= dentry
->d_inode
;
1760 if (inode
->i_nlink
> 1 && !S_ISDIR(inode
->i_mode
)) {
1761 struct shmem_sb_info
*sbinfo
= SHMEM_SB(inode
->i_sb
);
1762 if (sbinfo
->max_inodes
) {
1763 spin_lock(&sbinfo
->stat_lock
);
1764 sbinfo
->free_inodes
++;
1765 spin_unlock(&sbinfo
->stat_lock
);
1769 dir
->i_size
-= BOGO_DIRENT_SIZE
;
1770 inode
->i_ctime
= dir
->i_ctime
= dir
->i_mtime
= CURRENT_TIME
;
1772 dput(dentry
); /* Undo the count from "create" - this does all the work */
1776 static int shmem_rmdir(struct inode
*dir
, struct dentry
*dentry
)
1778 if (!simple_empty(dentry
))
1782 return shmem_unlink(dir
, dentry
);
1786 * The VFS layer already does all the dentry stuff for rename,
1787 * we just have to decrement the usage count for the target if
1788 * it exists so that the VFS layer correctly free's it when it
1791 static int shmem_rename(struct inode
*old_dir
, struct dentry
*old_dentry
, struct inode
*new_dir
, struct dentry
*new_dentry
)
1793 struct inode
*inode
= old_dentry
->d_inode
;
1794 int they_are_dirs
= S_ISDIR(inode
->i_mode
);
1796 if (!simple_empty(new_dentry
))
1799 if (new_dentry
->d_inode
) {
1800 (void) shmem_unlink(new_dir
, new_dentry
);
1803 } else if (they_are_dirs
) {
1808 old_dir
->i_size
-= BOGO_DIRENT_SIZE
;
1809 new_dir
->i_size
+= BOGO_DIRENT_SIZE
;
1810 old_dir
->i_ctime
= old_dir
->i_mtime
=
1811 new_dir
->i_ctime
= new_dir
->i_mtime
=
1812 inode
->i_ctime
= CURRENT_TIME
;
1816 static int shmem_symlink(struct inode
*dir
, struct dentry
*dentry
, const char *symname
)
1820 struct inode
*inode
;
1821 struct page
*page
= NULL
;
1823 struct shmem_inode_info
*info
;
1825 len
= strlen(symname
) + 1;
1826 if (len
> PAGE_CACHE_SIZE
)
1827 return -ENAMETOOLONG
;
1829 inode
= shmem_get_inode(dir
->i_sb
, S_IFLNK
|S_IRWXUGO
, 0);
1833 error
= security_inode_init_security(inode
, dir
, NULL
, NULL
,
1836 if (error
!= -EOPNOTSUPP
) {
1843 info
= SHMEM_I(inode
);
1844 inode
->i_size
= len
-1;
1845 if (len
<= (char *)inode
- (char *)info
) {
1847 memcpy(info
, symname
, len
);
1848 inode
->i_op
= &shmem_symlink_inline_operations
;
1850 error
= shmem_getpage(inode
, 0, &page
, SGP_WRITE
, NULL
);
1855 inode
->i_op
= &shmem_symlink_inode_operations
;
1856 kaddr
= kmap_atomic(page
, KM_USER0
);
1857 memcpy(kaddr
, symname
, len
);
1858 kunmap_atomic(kaddr
, KM_USER0
);
1859 set_page_dirty(page
);
1860 page_cache_release(page
);
1862 if (dir
->i_mode
& S_ISGID
)
1863 inode
->i_gid
= dir
->i_gid
;
1864 dir
->i_size
+= BOGO_DIRENT_SIZE
;
1865 dir
->i_ctime
= dir
->i_mtime
= CURRENT_TIME
;
1866 d_instantiate(dentry
, inode
);
1871 static void *shmem_follow_link_inline(struct dentry
*dentry
, struct nameidata
*nd
)
1873 nd_set_link(nd
, (char *)SHMEM_I(dentry
->d_inode
));
1877 static void *shmem_follow_link(struct dentry
*dentry
, struct nameidata
*nd
)
1879 struct page
*page
= NULL
;
1880 int res
= shmem_getpage(dentry
->d_inode
, 0, &page
, SGP_READ
, NULL
);
1881 nd_set_link(nd
, res
? ERR_PTR(res
) : kmap(page
));
1885 static void shmem_put_link(struct dentry
*dentry
, struct nameidata
*nd
, void *cookie
)
1887 if (!IS_ERR(nd_get_link(nd
))) {
1888 struct page
*page
= cookie
;
1890 mark_page_accessed(page
);
1891 page_cache_release(page
);
1895 static struct inode_operations shmem_symlink_inline_operations
= {
1896 .readlink
= generic_readlink
,
1897 .follow_link
= shmem_follow_link_inline
,
1900 static struct inode_operations shmem_symlink_inode_operations
= {
1901 .truncate
= shmem_truncate
,
1902 .readlink
= generic_readlink
,
1903 .follow_link
= shmem_follow_link
,
1904 .put_link
= shmem_put_link
,
1907 static int shmem_parse_options(char *options
, int *mode
, uid_t
*uid
,
1908 gid_t
*gid
, unsigned long *blocks
, unsigned long *inodes
,
1909 int *policy
, nodemask_t
*policy_nodes
)
1911 char *this_char
, *value
, *rest
;
1913 while (options
!= NULL
) {
1914 this_char
= options
;
1917 * NUL-terminate this option: unfortunately,
1918 * mount options form a comma-separated list,
1919 * but mpol's nodelist may also contain commas.
1921 options
= strchr(options
, ',');
1922 if (options
== NULL
)
1925 if (!isdigit(*options
)) {
1932 if ((value
= strchr(this_char
,'=')) != NULL
) {
1936 "tmpfs: No value for mount option '%s'\n",
1941 if (!strcmp(this_char
,"size")) {
1942 unsigned long long size
;
1943 size
= memparse(value
,&rest
);
1945 size
<<= PAGE_SHIFT
;
1946 size
*= totalram_pages
;
1952 *blocks
= size
>> PAGE_CACHE_SHIFT
;
1953 } else if (!strcmp(this_char
,"nr_blocks")) {
1954 *blocks
= memparse(value
,&rest
);
1957 } else if (!strcmp(this_char
,"nr_inodes")) {
1958 *inodes
= memparse(value
,&rest
);
1961 } else if (!strcmp(this_char
,"mode")) {
1964 *mode
= simple_strtoul(value
,&rest
,8);
1967 } else if (!strcmp(this_char
,"uid")) {
1970 *uid
= simple_strtoul(value
,&rest
,0);
1973 } else if (!strcmp(this_char
,"gid")) {
1976 *gid
= simple_strtoul(value
,&rest
,0);
1979 } else if (!strcmp(this_char
,"mpol")) {
1980 if (shmem_parse_mpol(value
,policy
,policy_nodes
))
1983 printk(KERN_ERR
"tmpfs: Bad mount option %s\n",
1991 printk(KERN_ERR
"tmpfs: Bad value '%s' for mount option '%s'\n",
1997 static int shmem_remount_fs(struct super_block
*sb
, int *flags
, char *data
)
1999 struct shmem_sb_info
*sbinfo
= SHMEM_SB(sb
);
2000 unsigned long max_blocks
= sbinfo
->max_blocks
;
2001 unsigned long max_inodes
= sbinfo
->max_inodes
;
2002 int policy
= sbinfo
->policy
;
2003 nodemask_t policy_nodes
= sbinfo
->policy_nodes
;
2004 unsigned long blocks
;
2005 unsigned long inodes
;
2006 int error
= -EINVAL
;
2008 if (shmem_parse_options(data
, NULL
, NULL
, NULL
, &max_blocks
,
2009 &max_inodes
, &policy
, &policy_nodes
))
2012 spin_lock(&sbinfo
->stat_lock
);
2013 blocks
= sbinfo
->max_blocks
- sbinfo
->free_blocks
;
2014 inodes
= sbinfo
->max_inodes
- sbinfo
->free_inodes
;
2015 if (max_blocks
< blocks
)
2017 if (max_inodes
< inodes
)
2020 * Those tests also disallow limited->unlimited while any are in
2021 * use, so i_blocks will always be zero when max_blocks is zero;
2022 * but we must separately disallow unlimited->limited, because
2023 * in that case we have no record of how much is already in use.
2025 if (max_blocks
&& !sbinfo
->max_blocks
)
2027 if (max_inodes
&& !sbinfo
->max_inodes
)
2031 sbinfo
->max_blocks
= max_blocks
;
2032 sbinfo
->free_blocks
= max_blocks
- blocks
;
2033 sbinfo
->max_inodes
= max_inodes
;
2034 sbinfo
->free_inodes
= max_inodes
- inodes
;
2035 sbinfo
->policy
= policy
;
2036 sbinfo
->policy_nodes
= policy_nodes
;
2038 spin_unlock(&sbinfo
->stat_lock
);
2043 static void shmem_put_super(struct super_block
*sb
)
2045 kfree(sb
->s_fs_info
);
2046 sb
->s_fs_info
= NULL
;
2049 static int shmem_fill_super(struct super_block
*sb
,
2050 void *data
, int silent
)
2052 struct inode
*inode
;
2053 struct dentry
*root
;
2054 int mode
= S_IRWXUGO
| S_ISVTX
;
2055 uid_t uid
= current
->fsuid
;
2056 gid_t gid
= current
->fsgid
;
2058 struct shmem_sb_info
*sbinfo
;
2059 unsigned long blocks
= 0;
2060 unsigned long inodes
= 0;
2061 int policy
= MPOL_DEFAULT
;
2062 nodemask_t policy_nodes
= node_online_map
;
2066 * Per default we only allow half of the physical ram per
2067 * tmpfs instance, limiting inodes to one per page of lowmem;
2068 * but the internal instance is left unlimited.
2070 if (!(sb
->s_flags
& MS_NOUSER
)) {
2071 blocks
= totalram_pages
/ 2;
2072 inodes
= totalram_pages
- totalhigh_pages
;
2073 if (inodes
> blocks
)
2075 if (shmem_parse_options(data
, &mode
, &uid
, &gid
, &blocks
,
2076 &inodes
, &policy
, &policy_nodes
))
2080 sb
->s_flags
|= MS_NOUSER
;
2083 /* Round up to L1_CACHE_BYTES to resist false sharing */
2084 sbinfo
= kmalloc(max((int)sizeof(struct shmem_sb_info
),
2085 L1_CACHE_BYTES
), GFP_KERNEL
);
2089 spin_lock_init(&sbinfo
->stat_lock
);
2090 sbinfo
->max_blocks
= blocks
;
2091 sbinfo
->free_blocks
= blocks
;
2092 sbinfo
->max_inodes
= inodes
;
2093 sbinfo
->free_inodes
= inodes
;
2094 sbinfo
->policy
= policy
;
2095 sbinfo
->policy_nodes
= policy_nodes
;
2097 sb
->s_fs_info
= sbinfo
;
2098 sb
->s_maxbytes
= SHMEM_MAX_BYTES
;
2099 sb
->s_blocksize
= PAGE_CACHE_SIZE
;
2100 sb
->s_blocksize_bits
= PAGE_CACHE_SHIFT
;
2101 sb
->s_magic
= TMPFS_MAGIC
;
2102 sb
->s_op
= &shmem_ops
;
2103 sb
->s_time_gran
= 1;
2105 inode
= shmem_get_inode(sb
, S_IFDIR
| mode
, 0);
2110 root
= d_alloc_root(inode
);
2119 shmem_put_super(sb
);
2123 static kmem_cache_t
*shmem_inode_cachep
;
2125 static struct inode
*shmem_alloc_inode(struct super_block
*sb
)
2127 struct shmem_inode_info
*p
;
2128 p
= (struct shmem_inode_info
*)kmem_cache_alloc(shmem_inode_cachep
, SLAB_KERNEL
);
2131 return &p
->vfs_inode
;
2134 static void shmem_destroy_inode(struct inode
*inode
)
2136 if ((inode
->i_mode
& S_IFMT
) == S_IFREG
) {
2137 /* only struct inode is valid if it's an inline symlink */
2138 mpol_free_shared_policy(&SHMEM_I(inode
)->policy
);
2140 kmem_cache_free(shmem_inode_cachep
, SHMEM_I(inode
));
2143 static void init_once(void *foo
, kmem_cache_t
*cachep
, unsigned long flags
)
2145 struct shmem_inode_info
*p
= (struct shmem_inode_info
*) foo
;
2147 if ((flags
& (SLAB_CTOR_VERIFY
|SLAB_CTOR_CONSTRUCTOR
)) ==
2148 SLAB_CTOR_CONSTRUCTOR
) {
2149 inode_init_once(&p
->vfs_inode
);
2153 static int init_inodecache(void)
2155 shmem_inode_cachep
= kmem_cache_create("shmem_inode_cache",
2156 sizeof(struct shmem_inode_info
),
2157 0, 0, init_once
, NULL
);
2158 if (shmem_inode_cachep
== NULL
)
2163 static void destroy_inodecache(void)
2165 if (kmem_cache_destroy(shmem_inode_cachep
))
2166 printk(KERN_INFO
"shmem_inode_cache: not all structures were freed\n");
2169 static struct address_space_operations shmem_aops
= {
2170 .writepage
= shmem_writepage
,
2171 .set_page_dirty
= __set_page_dirty_nobuffers
,
2173 .prepare_write
= shmem_prepare_write
,
2174 .commit_write
= simple_commit_write
,
2176 .migratepage
= migrate_page
,
2179 static struct file_operations shmem_file_operations
= {
2182 .llseek
= generic_file_llseek
,
2183 .read
= shmem_file_read
,
2184 .write
= shmem_file_write
,
2185 .fsync
= simple_sync_file
,
2186 .sendfile
= shmem_file_sendfile
,
2190 static struct inode_operations shmem_inode_operations
= {
2191 .truncate
= shmem_truncate
,
2192 .setattr
= shmem_notify_change
,
2193 .truncate_range
= shmem_truncate_range
,
2196 static struct inode_operations shmem_dir_inode_operations
= {
2198 .create
= shmem_create
,
2199 .lookup
= simple_lookup
,
2201 .unlink
= shmem_unlink
,
2202 .symlink
= shmem_symlink
,
2203 .mkdir
= shmem_mkdir
,
2204 .rmdir
= shmem_rmdir
,
2205 .mknod
= shmem_mknod
,
2206 .rename
= shmem_rename
,
2210 static struct super_operations shmem_ops
= {
2211 .alloc_inode
= shmem_alloc_inode
,
2212 .destroy_inode
= shmem_destroy_inode
,
2214 .statfs
= shmem_statfs
,
2215 .remount_fs
= shmem_remount_fs
,
2217 .delete_inode
= shmem_delete_inode
,
2218 .drop_inode
= generic_delete_inode
,
2219 .put_super
= shmem_put_super
,
2222 static struct vm_operations_struct shmem_vm_ops
= {
2223 .nopage
= shmem_nopage
,
2224 .populate
= shmem_populate
,
2226 .set_policy
= shmem_set_policy
,
2227 .get_policy
= shmem_get_policy
,
2232 static struct super_block
*shmem_get_sb(struct file_system_type
*fs_type
,
2233 int flags
, const char *dev_name
, void *data
)
2235 return get_sb_nodev(fs_type
, flags
, data
, shmem_fill_super
);
2238 static struct file_system_type tmpfs_fs_type
= {
2239 .owner
= THIS_MODULE
,
2241 .get_sb
= shmem_get_sb
,
2242 .kill_sb
= kill_litter_super
,
2244 static struct vfsmount
*shm_mnt
;
2246 static int __init
init_tmpfs(void)
2250 error
= init_inodecache();
2254 error
= register_filesystem(&tmpfs_fs_type
);
2256 printk(KERN_ERR
"Could not register tmpfs\n");
2260 devfs_mk_dir("shm");
2262 shm_mnt
= do_kern_mount(tmpfs_fs_type
.name
, MS_NOUSER
,
2263 tmpfs_fs_type
.name
, NULL
);
2264 if (IS_ERR(shm_mnt
)) {
2265 error
= PTR_ERR(shm_mnt
);
2266 printk(KERN_ERR
"Could not kern_mount tmpfs\n");
2272 unregister_filesystem(&tmpfs_fs_type
);
2274 destroy_inodecache();
2276 shm_mnt
= ERR_PTR(error
);
2279 module_init(init_tmpfs
)
2282 * shmem_file_setup - get an unlinked file living in tmpfs
2284 * @name: name for dentry (to be seen in /proc/<pid>/maps
2285 * @size: size to be set for the file
2288 struct file
*shmem_file_setup(char *name
, loff_t size
, unsigned long flags
)
2292 struct inode
*inode
;
2293 struct dentry
*dentry
, *root
;
2296 if (IS_ERR(shm_mnt
))
2297 return (void *)shm_mnt
;
2299 if (size
< 0 || size
> SHMEM_MAX_BYTES
)
2300 return ERR_PTR(-EINVAL
);
2302 if (shmem_acct_size(flags
, size
))
2303 return ERR_PTR(-ENOMEM
);
2307 this.len
= strlen(name
);
2308 this.hash
= 0; /* will go */
2309 root
= shm_mnt
->mnt_root
;
2310 dentry
= d_alloc(root
, &this);
2315 file
= get_empty_filp();
2320 inode
= shmem_get_inode(root
->d_sb
, S_IFREG
| S_IRWXUGO
, 0);
2324 SHMEM_I(inode
)->flags
= flags
& VM_ACCOUNT
;
2325 d_instantiate(dentry
, inode
);
2326 inode
->i_size
= size
;
2327 inode
->i_nlink
= 0; /* It is unlinked */
2328 file
->f_vfsmnt
= mntget(shm_mnt
);
2329 file
->f_dentry
= dentry
;
2330 file
->f_mapping
= inode
->i_mapping
;
2331 file
->f_op
= &shmem_file_operations
;
2332 file
->f_mode
= FMODE_WRITE
| FMODE_READ
;
2340 shmem_unacct_size(flags
, size
);
2341 return ERR_PTR(error
);
2345 * shmem_zero_setup - setup a shared anonymous mapping
2347 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2349 int shmem_zero_setup(struct vm_area_struct
*vma
)
2352 loff_t size
= vma
->vm_end
- vma
->vm_start
;
2354 file
= shmem_file_setup("dev/zero", size
, vma
->vm_flags
);
2356 return PTR_ERR(file
);
2360 vma
->vm_file
= file
;
2361 vma
->vm_ops
= &shmem_vm_ops
;