| blob | 834b2d785f1e2f2fdce59a608f28a94b02b5d82d |
1 /*
2 * mm/mmap.c
3 *
4 * Written by obz.
5 *
6 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
7 */
9 #include <linux/kernel.h>
10 #include <linux/slab.h>
11 #include <linux/backing-dev.h>
12 #include <linux/mm.h>
13 #include <linux/shm.h>
14 #include <linux/mman.h>
15 #include <linux/pagemap.h>
16 #include <linux/swap.h>
17 #include <linux/syscalls.h>
18 #include <linux/capability.h>
19 #include <linux/init.h>
20 #include <linux/file.h>
21 #include <linux/fs.h>
22 #include <linux/personality.h>
23 #include <linux/security.h>
24 #include <linux/hugetlb.h>
25 #include <linux/profile.h>
26 #include <linux/export.h>
27 #include <linux/mount.h>
28 #include <linux/mempolicy.h>
29 #include <linux/rmap.h>
30 #include <linux/mmu_notifier.h>
31 #include <linux/perf_event.h>
32 #include <linux/audit.h>
33 #include <linux/khugepaged.h>
34 #include <linux/uprobes.h>
35 #include <linux/rbtree_augmented.h>
36 #include <linux/sched/sysctl.h>
37 #include <linux/notifier.h>
38 #include <linux/memory.h>
40 #include <asm/uaccess.h>
41 #include <asm/cacheflush.h>
42 #include <asm/tlb.h>
43 #include <asm/mmu_context.h>
47 #ifndef arch_mmap_check
48 #define arch_mmap_check(addr, len, flags) (0)
49 #endif
51 #ifndef arch_rebalance_pgtables
52 #define arch_rebalance_pgtables(addr, len) (addr)
53 #endif
59 /* description of effects of mapping type and prot in current implementation.
60 * this is due to the limited x86 page protection hardware. The expected
61 * behavior is in parens:
62 *
63 * map_type prot
64 * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
65 * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
66 * w: (no) no w: (no) no w: (yes) yes w: (no) no
67 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
68 *
69 * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
70 * w: (no) no w: (no) no w: (copy) copy w: (no) no
71 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
72 *
73 */
77 };
80 {
84 }
92 /*
93 * Make sure vm_committed_as in one cacheline and not cacheline shared with
94 * other variables. It can be updated by several CPUs frequently.
95 */
98 /*
99 * The global memory commitment made in the system can be a metric
100 * that can be used to drive ballooning decisions when Linux is hosted
101 * as a guest. On Hyper-V, the host implements a policy engine for dynamically
102 * balancing memory across competing virtual machines that are hosted.
103 * Several metrics drive this policy engine including the guest reported
104 * memory commitment.
105 */
107 {
109 }
112 /*
113 * Check that a process has enough memory to allocate a new virtual
114 * mapping. 0 means there is enough memory for the allocation to
115 * succeed and -ENOMEM implies there is not.
116 *
117 * We currently support three overcommit policies, which are set via the
118 * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
119 *
120 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
121 * Additional code 2002 Jul 20 by Robert Love.
122 *
123 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
124 *
125 * Note this is a helper function intended to be used by LSMs which
126 * wish to use this logic.
127 */
129 {
134 /*
135 * Sometimes we want to use more memory than we have
136 */
144 /*
145 * shmem pages shouldn't be counted as free in this
146 * case, they can't be purged, only swapped out, and
147 * that won't affect the overall amount of available
148 * memory in the system.
149 */
154 /*
155 * Any slabs which are created with the
156 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
157 * which are reclaimable, under pressure. The dentry
158 * cache and most inode caches should fall into this
159 */
162 /*
163 * Leave reserved pages. The pages are not for anonymous pages.
164 */
167 else
170 /*
171 * Reserve some for root
172 */
180 }
183 /*
184 * Reserve some for root
185 */
189 /*
190 * Don't let a single process grow so big a user can't recover
191 */
195 }
199 error:
203 }
205 /*
206 * Requires inode->i_mapping->i_mmap_mutex
207 */
210 {
219 else
222 }
224 /*
225 * Unlink a file-based vm structure from its interval tree, to hide
226 * vma from rmap and vmtruncate before freeing its page tables.
227 */
229 {
237 }
238 }
240 /*
241 * Close a vm structure and free it, returning the next.
242 */
244 {
255 }
260 {
269 #ifdef CONFIG_COMPAT_BRK
270 /*
271 * CONFIG_COMPAT_BRK can still be overridden by setting
272 * randomize_va_space to 2, which will still cause mm->start_brk
273 * to be arbitrarily shifted
274 */
277 else
279 #else
281 #endif
285 /*
286 * Check against rlimit here. If this check is done later after the test
287 * of oldbrk with newbrk then it can escape the test and let the data
288 * segment grow beyond its set limit the in case where the limit is
289 * not page aligned -Ram Gupta
290 */
301 /* Always allow shrinking brk. */
306 }
308 /* Check against existing mmap mappings. */
312 /* Ok, looks good - let it rip. */
316 set_brk:
324 out:
328 }
331 {
341 }
347 }
349 }
351 #ifdef CONFIG_DEBUG_VM_RB
353 {
364 }
368 }
373 }
379 }
380 i++;
384 }
387 j++;
391 }
393 }
396 {
404 }
405 }
408 {
421 i++;
422 }
426 }
431 }
436 }
438 }
439 #else
440 #define validate_mm_rb(root, ignore) do { } while (0)
441 #define validate_mm(mm) do { } while (0)
442 #endif
447 /*
448 * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
449 * vma->vm_prev->vm_end values changed, without modifying the vma's position
450 * in the rbtree.
451 */
453 {
454 /*
455 * As it turns out, RB_DECLARE_CALLBACKS() already created a callback
456 * function that does exacltly what we want.
457 */
459 }
463 {
464 /* All rb_subtree_gap values must be consistent prior to insertion */
468 }
471 {
472 /*
473 * All rb_subtree_gap values must be consistent prior to erase,
474 * with the possible exception of the vma being erased.
475 */
478 /*
479 * Note rb_erase_augmented is a fairly large inline function,
480 * so make sure we instantiate it only once with our desired
481 * augmented rbtree callbacks.
482 */
484 }
486 /*
487 * vma has some anon_vma assigned, and is already inserted on that
488 * anon_vma's interval trees.
489 *
490 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
491 * vma must be removed from the anon_vma's interval trees using
492 * anon_vma_interval_tree_pre_update_vma().
493 *
494 * After the update, the vma will be reinserted using
495 * anon_vma_interval_tree_post_update_vma().
496 *
497 * The entire update must be protected by exclusive mmap_sem and by
498 * the root anon_vma's mutex.
499 */
502 {
507 }
511 {
516 }
521 {
534 /* Fail if an existing vma overlaps the area */
541 }
542 }
550 }
554 {
558 /* Find first overlaping mapping */
566 /* Iterate over the rest of the overlaps */
575 }
578 }
582 {
583 /* Update tracking information for the gap following the new vma. */
586 else
589 /*
590 * vma->vm_prev wasn't known when we followed the rbtree to find the
591 * correct insertion point for that vma. As a result, we could not
592 * update the vma vm_rb parents rb_subtree_gap values on the way down.
593 * So, we first insert the vma with a zero rb_subtree_gap value
594 * (to be consistent with what we did on the way down), and then
595 * immediately update the gap to the correct value. Finally we
596 * rebalance the rbtree after all augmented values have been set.
597 */
602 }
605 {
620 else
623 }
624 }
626 static void
630 {
633 }
638 {
655 }
657 /*
658 * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
659 * mm's list and rbtree. It has already been inserted into the interval tree.
660 */
662 {
671 }
676 {
685 }
687 /*
688 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
689 * is already present in an i_mmap tree without adjusting the tree.
690 * The following helper function should be used when such adjustments
691 * are necessary. The "insert" vma (if any) is to be inserted
692 * before we drop the necessary locks.
693 */
696 {
712 /*
713 * vma expands, overlapping all the next, and
714 * perhaps the one after too (mprotect case 6).
715 */
721 /*
722 * vma expands, overlapping part of the next:
723 * mprotect case 5 shifting the boundary up.
724 */
729 /*
730 * vma shrinks, and !insert tells it's not
731 * split_vma inserting another: so it must be
732 * mprotect case 4 shifting the boundary down.
733 */
737 }
739 /*
740 * Easily overlooked: when mprotect shifts the boundary,
741 * make sure the expanding vma has anon_vma set if the
742 * shrinking vma had, to cover any anon pages imported.
743 */
748 }
749 }
760 }
764 /*
765 * Put into interval tree now, so instantiated pages
766 * are visible to arm/parisc __flush_dcache_page
767 * throughout; but we cannot insert into address
768 * space until vma start or end is updated.
769 */
771 }
772 }
786 }
793 }
798 }
802 }
807 }
814 }
817 /*
818 * vma_merge has merged next into vma, and needs
819 * us to remove next before dropping the locks.
820 */
825 /*
826 * split_vma has split insert from vma, and needs
827 * us to insert it before dropping the locks
828 * (it may either follow vma or precede it).
829 */
839 }
840 }
847 }
856 }
862 }
868 /*
869 * In mprotect's case 6 (see comments on vma_merge),
870 * we must remove another next too. It would clutter
871 * up the code too much to do both in one go.
872 */
878 else
880 }
887 }
889 /*
890 * If the vma has a ->close operation then the driver probably needs to release
891 * per-vma resources, so we don't attempt to merge those.
892 */
895 {
903 }
908 {
909 /*
910 * The list_is_singular() test is to avoid merging VMA cloned from
911 * parents. This can improve scalability caused by anon_vma lock.
912 */
917 }
919 /*
920 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
921 * in front of (at a lower virtual address and file offset than) the vma.
922 *
923 * We cannot merge two vmas if they have differently assigned (non-NULL)
924 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
925 *
926 * We don't check here for the merged mmap wrapping around the end of pagecache
927 * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
928 * wrap, nor mmaps which cover the final page at index -1UL.
929 */
930 static int
933 {
938 }
940 }
942 /*
943 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
944 * beyond (at a higher virtual address and file offset than) the vma.
945 *
946 * We cannot merge two vmas if they have differently assigned (non-NULL)
947 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
948 */
949 static int
952 {
955 pgoff_t vm_pglen;
959 }
961 }
963 /*
964 * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
965 * whether that can be merged with its predecessor or its successor.
966 * Or both (it neatly fills a hole).
967 *
968 * In most cases - when called for mmap, brk or mremap - [addr,end) is
969 * certain not to be mapped by the time vma_merge is called; but when
970 * called for mprotect, it is certain to be already mapped (either at
971 * an offset within prev, or at the start of next), and the flags of
972 * this area are about to be changed to vm_flags - and the no-change
973 * case has already been eliminated.
974 *
975 * The following mprotect cases have to be considered, where AAAA is
976 * the area passed down from mprotect_fixup, never extending beyond one
977 * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
978 *
979 * AAAA AAAA AAAA AAAA
980 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
981 * cannot merge might become might become might become
982 * PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
983 * mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
984 * mremap move: PPPPNNNNNNNN 8
985 * AAAA
986 * PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
987 * might become case 1 below case 2 below case 3 below
988 *
989 * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
990 * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
991 */
997 {
1002 /*
1003 * We later require that vma->vm_flags == vm_flags,
1004 * so this tests vma->vm_flags & VM_SPECIAL, too.
1005 */
1011 else
1017 /*
1018 * Can it merge with the predecessor?
1019 */
1024 /*
1025 * OK, it can. Can we now merge in the successor as well?
1026 */
1033 /* cases 1, 6 */
1043 }
1045 /*
1046 * Can this new request be merged in front of next?
1047 */
1062 }
1065 }
1067 /*
1068 * Rough compatbility check to quickly see if it's even worth looking
1069 * at sharing an anon_vma.
1070 *
1071 * They need to have the same vm_file, and the flags can only differ
1072 * in things that mprotect may change.
1073 *
1074 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1075 * we can merge the two vma's. For example, we refuse to merge a vma if
1076 * there is a vm_ops->close() function, because that indicates that the
1077 * driver is doing some kind of reference counting. But that doesn't
1078 * really matter for the anon_vma sharing case.
1079 */
1081 {
1087 }
1089 /*
1090 * Do some basic sanity checking to see if we can re-use the anon_vma
1091 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1092 * the same as 'old', the other will be the new one that is trying
1093 * to share the anon_vma.
1094 *
1095 * NOTE! This runs with mm_sem held for reading, so it is possible that
1096 * the anon_vma of 'old' is concurrently in the process of being set up
1097 * by another page fault trying to merge _that_. But that's ok: if it
1098 * is being set up, that automatically means that it will be a singleton
1099 * acceptable for merging, so we can do all of this optimistically. But
1100 * we do that ACCESS_ONCE() to make sure that we never re-load the pointer.
1101 *
1102 * IOW: that the "list_is_singular()" test on the anon_vma_chain only
1103 * matters for the 'stable anon_vma' case (ie the thing we want to avoid
1104 * is to return an anon_vma that is "complex" due to having gone through
1105 * a fork).
1106 *
1107 * We also make sure that the two vma's are compatible (adjacent,
1108 * and with the same memory policies). That's all stable, even with just
1109 * a read lock on the mm_sem.
1110 */
1111 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
1112 {
1118 }
1120 }
1122 /*
1123 * find_mergeable_anon_vma is used by anon_vma_prepare, to check
1124 * neighbouring vmas for a suitable anon_vma, before it goes off
1125 * to allocate a new anon_vma. It checks because a repetitive
1126 * sequence of mprotects and faults may otherwise lead to distinct
1127 * anon_vmas being allocated, preventing vma merge in subsequent
1128 * mprotect.
1129 */
1131 {
1142 try_prev:
1150 none:
1151 /*
1152 * There's no absolute need to look only at touching neighbours:
1153 * we could search further afield for "compatible" anon_vmas.
1154 * But it would probably just be a waste of time searching,
1155 * or lead to too many vmas hanging off the same anon_vma.
1156 * We're trying to allow mprotect remerging later on,
1157 * not trying to minimize memory used for anon_vmas.
1158 */
1160 }
1162 #ifdef CONFIG_PROC_FS
1165 {
1166 const unsigned long stack_flags
1177 }
1180 /*
1181 * If a hint addr is less than mmap_min_addr change hint to be as
1182 * low as possible but still greater than mmap_min_addr
1183 */
1185 {
1191 }
1193 /*
1194 * The caller must hold down_write(¤t->mm->mmap_sem).
1195 */
1201 {
1203 vm_flags_t vm_flags;
1207 /*
1208 * Does the application expect PROT_READ to imply PROT_EXEC?
1209 *
1210 * (the exception is when the underlying filesystem is noexec
1211 * mounted, in which case we dont add PROT_EXEC.)
1212 */
1223 /* Careful about overflows.. */
1228 /* offset overflow? */
1232 /* Too many mappings? */
1236 /* Obtain the address to map to. we verify (or select) it and ensure
1237 * that it represents a valid section of the address space.
1238 */
1243 /* Do simple checking here so the lower-level routines won't have
1244 * to. we assume access permissions have been handled by the open
1245 * of the memory object, so we don't do any here.
1246 */
1254 /* mlock MCL_FUTURE? */
1263 }
1273 /*
1274 * Make sure we don't allow writing to an append-only
1275 * file..
1276 */
1280 /*
1281 * Make sure there are no mandatory locks on the file.
1282 */
1290 /* fall through */
1298 }
1308 }
1314 /*
1315 * Ignore pgoff.
1316 */
1321 /*
1322 * Set pgoff according to addr for anon_vma.
1323 */
1328 }
1329 }
1331 /*
1332 * Set 'VM_NORESERVE' if we should not account for the
1333 * memory use of this mapping.
1334 */
1336 /* We honor MAP_NORESERVE if allowed to overcommit */
1340 /* hugetlb applies strict overcommit unless MAP_NORESERVE */
1343 }
1351 }
1356 {
1379 /*
1380 * VM_NORESERVE is used because the reservations will be
1381 * taken when vm_ops->mmap() is called
1382 * A dummy user value is used because we are not locking
1383 * memory so no accounting is necessary
1384 */
1386 VM_NORESERVE,
1391 }
1396 out_fput:
1399 out:
1401 }
1403 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1411 };
1414 {
1424 }
1427 /*
1428 * Some shared mappigns will want the pages marked read-only
1429 * to track write events. If so, we'll downgrade vm_page_prot
1430 * to the private version (using protection_map[] without the
1431 * VM_SHARED bit).
1432 */
1434 {
1437 /* If it was private or non-writable, the write bit is already clear */
1441 /* The backer wishes to know when pages are first written to? */
1445 /* The open routine did something to the protections already? */
1450 /* Specialty mapping? */
1454 /* Can the mapping track the dirty pages? */
1457 }
1459 /*
1460 * We account for memory if it's a private writeable mapping,
1461 * not hugepages and VM_NORESERVE wasn't set.
1462 */
1464 {
1465 /*
1466 * hugetlb has its own accounting separate from the core VM
1467 * VM_HUGETLB may not be set yet so we cannot check for that flag.
1468 */
1473 }
1477 {
1484 /* Check against address space limit. */
1488 /*
1489 * MAP_FIXED may remove pages of mappings that intersects with
1490 * requested mapping. Account for the pages it would unmap.
1491 */
1499 }
1501 /* Clear old maps */
1503 munmap_back:
1508 }
1510 /*
1511 * Private writable mapping: check memory availability
1512 */
1518 }
1520 /*
1521 * Can we just expand an old mapping?
1522 */
1527 /*
1528 * Determine the object being mapped and call the appropriate
1529 * specific mapper. the address has already been validated, but
1530 * not unmapped, but the maps are removed from the list.
1531 */
1536 }
1551 }
1557 /* Can addr have changed??
1558 *
1559 * Answer: Yes, several device drivers can do it in their
1560 * f_op->mmap method. -DaveM
1561 * Bug: If addr is changed, prev, rb_link, rb_parent should
1562 * be updated for vma_link()
1563 */
1572 }
1577 /* Can vma->vm_page_prot have changed??
1578 *
1579 * Answer: Yes, drivers may have changed it in their
1580 * f_op->mmap method.
1581 *
1582 * Ensures that vmas marked as uncached stay that way.
1583 */
1587 }
1590 /* Once vma denies write, undo our temporary denial count */
1594 out:
1602 else
1604 }
1609 /*
1610 * New (or expanded) vma always get soft dirty status.
1611 * Otherwise user-space soft-dirty page tracker won't
1612 * be able to distinguish situation when vma area unmapped,
1613 * then new mapped in-place (which must be aimed as
1614 * a completely new data area).
1615 */
1620 unmap_and_free_vma:
1626 /* Undo any partial mapping done by a device driver. */
1629 free_vma:
1631 unacct_error:
1635 }
1638 {
1639 /*
1640 * We implement the search by looking for an rbtree node that
1641 * immediately follows a suitable gap. That is,
1642 * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
1643 * - gap_end = vma->vm_start >= info->low_limit + length;
1644 * - gap_end - gap_start >= length
1645 */
1651 /* Adjust search length to account for worst case alignment overhead */
1656 /* Adjust search limits by the desired length */
1665 /* Check if rbtree root looks promising */
1673 /* Visit left subtree if it looks promising */
1682 }
1683 }
1686 check_current:
1687 /* Check if current node has a suitable gap */
1693 /* Visit right subtree if it looks promising */
1701 }
1702 }
1704 /* Go back up the rbtree to find next candidate node */
1715 }
1716 }
1717 }
1719 check_highest:
1720 /* Check highest gap, which does not precede any rbtree node */
1726 found:
1727 /* We found a suitable gap. Clip it with the original low_limit. */
1731 /* Adjust gap address to the desired alignment */
1737 }
1740 {
1745 /* Adjust search length to account for worst case alignment overhead */
1750 /*
1751 * Adjust search limits by the desired length.
1752 * See implementation comment at top of unmapped_area().
1753 */
1763 /* Check highest gap, which does not precede any rbtree node */
1768 /* Check if rbtree root looks promising */
1776 /* Visit right subtree if it looks promising */
1785 }
1786 }
1788 check_current:
1789 /* Check if current node has a suitable gap */
1796 /* Visit left subtree if it looks promising */
1804 }
1805 }
1807 /* Go back up the rbtree to find next candidate node */
1818 }
1819 }
1820 }
1822 found:
1823 /* We found a suitable gap. Clip it with the original high_limit. */
1827 found_highest:
1828 /* Compute highest gap address at the desired alignment */
1835 }
1837 /* Get an address range which is currently unmapped.
1838 * For shmat() with addr=0.
1839 *
1840 * Ugly calling convention alert:
1841 * Return value with the low bits set means error value,
1842 * ie
1843 * if (ret & ~PAGE_MASK)
1844 * error = ret;
1845 *
1846 * This function "knows" that -ENOMEM has the bits set.
1847 */
1848 #ifndef HAVE_ARCH_UNMAPPED_AREA
1849 unsigned long
1852 {
1869 }
1877 }
1878 #endif
1880 /*
1881 * This mmap-allocator allocates new areas top-down from below the
1882 * stack's low limit (the base):
1883 */
1884 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1885 unsigned long
1889 {
1895 /* requested length too big for entire address space */
1902 /* requesting a specific address */
1909 }
1918 /*
1919 * A failed mmap() very likely causes application failure,
1920 * so fall back to the bottom-up function here. This scenario
1921 * can happen with large stack limits and large mmap()
1922 * allocations.
1923 */
1930 }
1933 }
1934 #endif
1936 unsigned long
1939 {
1947 /* Careful about overflows.. */
1966 }
1970 /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
1972 {
1975 /* Check the cache first. */
1976 /* (Cache hit rate is typically around 35%.) */
1997 }
2000 }
2002 }
2006 /*
2007 * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
2008 */
2012 {
2024 }
2025 }
2027 }
2029 /*
2030 * Verify that the stack growth is acceptable and
2031 * update accounting. This is shared with both the
2032 * grow-up and grow-down cases.
2033 */
2034 static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
2035 {
2040 /* address space limit tests */
2044 /* Stack limit test */
2048 /* mlock limit tests */
2057 }
2059 /* Check to ensure the stack will not grow into a hugetlb-only region */
2065 /*
2066 * Overcommit.. This must be the final test, as it will
2067 * update security statistics.
2068 */
2072 /* Ok, everything looks good - let it rip */
2077 }
2079 #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
2080 /*
2081 * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
2082 * vma is the last one with address > vma->vm_end. Have to extend vma.
2083 */
2085 {
2091 /*
2092 * We must make sure the anon_vma is allocated
2093 * so that the anon_vma locking is not a noop.
2094 */
2099 /*
2100 * vma->vm_start/vm_end cannot change under us because the caller
2101 * is required to hold the mmap_sem in read mode. We need the
2102 * anon_vma lock to serialize against concurrent expand_stacks.
2103 * Also guard against wrapping around to address 0.
2104 */
2110 }
2113 /* Somebody else might have raced and expanded it already */
2124 /*
2125 * vma_gap_update() doesn't support concurrent
2126 * updates, but we only hold a shared mmap_sem
2127 * lock here, so we need to protect against
2128 * concurrent vma expansions.
2129 * vma_lock_anon_vma() doesn't help here, as
2130 * we don't guarantee that all growable vmas
2131 * in a mm share the same root anon vma.
2132 * So, we reuse mm->page_table_lock to guard
2133 * against concurrent vma expansions.
2134 */
2141 else
2146 }
2147 }
2148 }
2153 }
2156 /*
2157 * vma is the first one with address < vma->vm_start. Have to extend vma.
2158 */
2161 {
2164 /*
2165 * We must make sure the anon_vma is allocated
2166 * so that the anon_vma locking is not a noop.
2167 */
2178 /*
2179 * vma->vm_start/vm_end cannot change under us because the caller
2180 * is required to hold the mmap_sem in read mode. We need the
2181 * anon_vma lock to serialize against concurrent expand_stacks.
2182 */
2184 /* Somebody else might have raced and expanded it already */
2195 /*
2196 * vma_gap_update() doesn't support concurrent
2197 * updates, but we only hold a shared mmap_sem
2198 * lock here, so we need to protect against
2199 * concurrent vma expansions.
2200 * vma_lock_anon_vma() doesn't help here, as
2201 * we don't guarantee that all growable vmas
2202 * in a mm share the same root anon vma.
2203 * So, we reuse mm->page_table_lock to guard
2204 * against concurrent vma expansions.
2205 */
2215 }
2216 }
2217 }
2222 }
2224 /*
2225 * Note how expand_stack() refuses to expand the stack all the way to
2226 * abut the next virtual mapping, *unless* that mapping itself is also
2227 * a stack mapping. We want to leave room for a guard page, after all
2228 * (the guard page itself is not added here, that is done by the
2229 * actual page faulting logic)
2230 *
2231 * This matches the behavior of the guard page logic (see mm/memory.c:
2232 * check_stack_guard_page()), which only allows the guard page to be
2233 * removed under these circumstances.
2234 */
2235 #ifdef CONFIG_STACK_GROWSUP
2237 {
2245 }
2247 }
2251 {
2263 }
2264 #else
2266 {
2274 }
2276 }
2280 {
2298 }
2299 #endif
2301 /*
2302 * Ok - we have the memory areas we should free on the vma list,
2303 * so release them, and do the vma updates.
2304 *
2305 * Called with the mm semaphore held.
2306 */
2308 {
2311 /* Update high watermark before we lower total_vm */
2323 }
2325 /*
2326 * Get rid of page table information in the indicated region.
2327 *
2328 * Called with the mm semaphore held.
2329 */
2333 {
2344 }
2346 /*
2347 * Create a list of vma's touched by the unmap, removing them from the mm's
2348 * vma list as we go..
2349 */
2350 static void
2353 {
2373 }
2375 /*
2376 * __split_vma() bypasses sysctl_max_map_count checking. We use this on the
2377 * munmap path where it doesn't make sense to fail.
2378 */
2381 {
2393 /* most fields are the same, copy all, and then fixup */
2403 }
2421 else
2424 /* Success. */
2428 /* Clean everything up if vma_adjust failed. */
2434 out_free_mpol:
2436 out_free_vma:
2438 out_err:
2440 }
2442 /*
2443 * Split a vma into two pieces at address 'addr', a new vma is allocated
2444 * either for the first part or the tail.
2445 */
2448 {
2453 }
2455 /* Munmap is split into 2 main parts -- this part which finds
2456 * what needs doing, and the areas themselves, which do the
2457 * work. This now handles partial unmappings.
2458 * Jeremy Fitzhardinge <jeremy@goop.org>
2459 */
2461 {
2471 /* Find the first overlapping VMA */
2476 /* we have start < vma->vm_end */
2478 /* if it doesn't overlap, we have nothing.. */
2483 /*
2484 * If we need to split any vma, do it now to save pain later.
2485 *
2486 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
2487 * unmapped vm_area_struct will remain in use: so lower split_vma
2488 * places tmp vma above, and higher split_vma places tmp vma below.
2489 */
2493 /*
2494 * Make sure that map_count on return from munmap() will
2495 * not exceed its limit; but let map_count go just above
2496 * its limit temporarily, to help free resources as expected.
2497 */
2505 }
2507 /* Does it split the last one? */
2513 }
2516 /*
2517 * unlock any mlock()ed ranges before detaching vmas
2518 */
2525 }
2527 }
2528 }
2530 /*
2531 * Remove the vma's, and unmap the actual pages
2532 */
2536 /* Fix up all other VM information */
2540 }
2543 {
2551 }
2555 {
2558 }
2561 {
2562 #ifdef CONFIG_DEBUG_VM
2566 }
2567 #endif
2568 }
2570 /*
2571 * this is really a simplified "do_mmap". it only handles
2572 * anonymous maps. eventually we may be able to do some
2573 * brk-specific accounting here.
2574 */
2576 {
2594 /*
2595 * mlock MCL_FUTURE?
2596 */
2605 }
2607 /*
2608 * mm->mmap_sem is required to protect against another thread
2609 * changing the mappings in case we sleep.
2610 */
2613 /*
2614 * Clear old maps. this also does some error checking for us
2615 */
2616 munmap_back:
2621 }
2623 /* Check against address space limits *after* clearing old maps... */
2633 /* Can we just expand an old private anonymous mapping? */
2639 /*
2640 * create a vma struct for an anonymous mapping
2641 */
2646 }
2656 out:
2663 }
2666 {
2678 }
2681 /* Release all mmaps. */
2683 {
2688 /* mm's last user has gone, and its about to be pulled down */
2697 }
2698 }
2709 /* update_hiwater_rss(mm) here? but nobody should be looking */
2710 /* Use -1 here to ensure all VMAs in the mm are unmapped */
2716 /*
2717 * Walk the list again, actually closing and freeing it,
2718 * with preemption enabled, without holding any MM locks.
2719 */
2724 }
2729 }
2731 /* Insert vm structure into process list sorted by address
2732 * and into the inode's i_mmap tree. If vm_file is non-NULL
2733 * then i_mmap_mutex is taken here.
2734 */
2736 {
2740 /*
2741 * The vm_pgoff of a purely anonymous vma should be irrelevant
2742 * until its first write fault, when page's anon_vma and index
2743 * are set. But now set the vm_pgoff it will almost certainly
2744 * end up with (unless mremap moves it elsewhere before that
2745 * first wfault), so /proc/pid/maps tells a consistent story.
2746 *
2747 * By setting it to reflect the virtual start address of the
2748 * vma, merges and splits can happen in a seamless way, just
2749 * using the existing file pgoff checks and manipulations.
2750 * Similarly in do_mmap_pgoff and in do_brk.
2751 */
2755 }
2765 }
2767 /*
2768 * Copy the vma structure to a new location in the same mm,
2769 * prior to moving page table entries, to effect an mremap move.
2770 */
2774 {
2782 /*
2783 * If anonymous vma has not yet been faulted, update new pgoff
2784 * to match new location, to increase its chance of merging.
2785 */
2789 }
2796 /*
2797 * Source vma may have been merged into new_vma
2798 */
2801 /*
2802 * The only way we can get a vma_merge with
2803 * self during an mremap is if the vma hasn't
2804 * been faulted in yet and we were allowed to
2805 * reset the dst vma->vm_pgoff to the
2806 * destination address of the mremap to allow
2807 * the merge to happen. mremap must change the
2808 * vm_pgoff linearity between src and dst vmas
2809 * (in turn preventing a vma_merge) to be
2810 * safe. It is only safe to keep the vm_pgoff
2811 * linear if there are no pages mapped yet.
2812 */
2815 }
2835 }
2836 }
2839 out_free_mempol:
2841 out_free_vma:
2844 }
2846 /*
2847 * Return true if the calling process may expand its vm space by the passed
2848 * number of pages
2849 */
2851 {
2860 }
2865 {
2866 pgoff_t pgoff;
2869 /*
2870 * special mappings have no vm_file, and in that case, the mm
2871 * uses vm_pgoff internally. So we have to subtract it from here.
2872 * We are allowed to do this because we are the mm; do not copy
2873 * this code into drivers!
2874 */
2878 pgoff--;
2885 }
2888 }
2890 /*
2891 * Having a close hook prevents vma merging regardless of flags.
2892 */
2894 {
2895 }
2900 };
2902 /*
2903 * Called with mm->mmap_sem held for writing.
2904 * Insert a new vma covering the given region, with the given flags.
2905 * Its pages are supplied by the given array of struct page *.
2906 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
2907 * The region past the last page supplied will always produce SIGBUS.
2908 * The array pointer and the pages it points to are assumed to stay alive
2909 * for as long as this mapping might exist.
2910 */
2914 {
2943 out:
2946 }
2951 {
2953 /*
2954 * The LSB of head.next can't change from under us
2955 * because we hold the mm_all_locks_mutex.
2956 */
2958 /*
2959 * We can safely modify head.next after taking the
2960 * anon_vma->root->rwsem. If some other vma in this mm shares
2961 * the same anon_vma we won't take it again.
2962 *
2963 * No need of atomic instructions here, head.next
2964 * can't change from under us thanks to the
2965 * anon_vma->root->rwsem.
2966 */
2970 }
2971 }
2974 {
2976 /*
2977 * AS_MM_ALL_LOCKS can't change from under us because
2978 * we hold the mm_all_locks_mutex.
2979 *
2980 * Operations on ->flags have to be atomic because
2981 * even if AS_MM_ALL_LOCKS is stable thanks to the
2982 * mm_all_locks_mutex, there may be other cpus
2983 * changing other bitflags in parallel to us.
2984 */
2988 }
2989 }
2991 /*
2992 * This operation locks against the VM for all pte/vma/mm related
2993 * operations that could ever happen on a certain mm. This includes
2994 * vmtruncate, try_to_unmap, and all page faults.
2995 *
2996 * The caller must take the mmap_sem in write mode before calling
2997 * mm_take_all_locks(). The caller isn't allowed to release the
2998 * mmap_sem until mm_drop_all_locks() returns.
2999 *
3000 * mmap_sem in write mode is required in order to block all operations
3001 * that could modify pagetables and free pages without need of
3002 * altering the vma layout (for example populate_range() with
3003 * nonlinear vmas). It's also needed in write mode to avoid new
3004 * anon_vmas to be associated with existing vmas.
3005 *
3006 * A single task can't take more than one mm_take_all_locks() in a row
3007 * or it would deadlock.
3008 *
3009 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3010 * mapping->flags avoid to take the same lock twice, if more than one
3011 * vma in this mm is backed by the same anon_vma or address_space.
3012 *
3013 * We can take all the locks in random order because the VM code
3014 * taking i_mmap_mutex or anon_vma->rwsem outside the mmap_sem never
3015 * takes more than one of them in a row. Secondly we're protected
3016 * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
3017 *
3018 * mm_take_all_locks() and mm_drop_all_locks are expensive operations
3019 * that may have to take thousand of locks.
3020 *
3021 * mm_take_all_locks() can fail if it's interrupted by signals.
3022 */
3024 {
3037 }
3045 }
3049 out_unlock:
3052 }
3055 {
3057 /*
3058 * The LSB of head.next can't change to 0 from under
3059 * us because we hold the mm_all_locks_mutex.
3060 *
3061 * We must however clear the bitflag before unlocking
3062 * the vma so the users using the anon_vma->rb_root will
3063 * never see our bitflag.
3064 *
3065 * No need of atomic instructions here, head.next
3066 * can't change from under us until we release the
3067 * anon_vma->root->rwsem.
3068 */
3073 }
3074 }
3077 {
3079 /*
3080 * AS_MM_ALL_LOCKS can't change to 0 from under us
3081 * because we hold the mm_all_locks_mutex.
3082 */
3087 }
3088 }
3090 /*
3091 * The mmap_sem cannot be released by the caller until
3092 * mm_drop_all_locks() returns.
3093 */
3095 {
3108 }
3111 }
3113 /*
3114 * initialise the VMA slab
3115 */
3117 {
3122 }
3124 /*
3125 * Initialise sysctl_user_reserve_kbytes.
3126 *
3127 * This is intended to prevent a user from starting a single memory hogging
3128 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
3129 * mode.
3130 *
3131 * The default value is min(3% of free memory, 128MB)
3132 * 128MB is enough to recover with sshd/login, bash, and top/kill.
3133 */
3135 {
3142 }
3145 /*
3146 * Initialise sysctl_admin_reserve_kbytes.
3147 *
3148 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
3149 * to log in and kill a memory hogging process.
3150 *
3151 * Systems with more than 256MB will reserve 8MB, enough to recover
3152 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
3153 * only reserve 3% of free pages by default.
3154 */
3156 {
3163 }
3166 /*
3167 * Reinititalise user and admin reserves if memory is added or removed.
3168 *
3169 * The default user reserve max is 128MB, and the default max for the
3170 * admin reserve is 8MB. These are usually, but not always, enough to
3171 * enable recovery from a memory hogging process using login/sshd, a shell,
3172 * and tools like top. It may make sense to increase or even disable the
3173 * reserve depending on the existence of swap or variations in the recovery
3174 * tools. So, the admin may have changed them.
3175 *
3176 * If memory is added and the reserves have been eliminated or increased above
3177 * the default max, then we'll trust the admin.
3178 *
3179 * If memory is removed and there isn't enough free memory, then we
3180 * need to reset the reserves.
3181 *
3182 * Otherwise keep the reserve set by the admin.
3183 */
3186 {
3191 /* Default max is 128MB. Leave alone if modified by operator. */
3196 /* Default max is 8MB. Leave alone if modified by operator. */
3208 sysctl_user_reserve_kbytes);
3209 }
3214 sysctl_admin_reserve_kbytes);
3215 }
3219 }
3221 }
3225 };
3228 {
3233 }