4 * (C) Copyright 1996 Linus Torvalds
7 #include <linux/slab.h>
8 #include <linux/smp_lock.h>
10 #include <linux/mman.h>
11 #include <linux/swap.h>
13 #include <asm/uaccess.h>
14 #include <asm/pgtable.h>
16 extern int vm_enough_memory(long pages
);
18 static inline pte_t
*get_one_pte(struct mm_struct
*mm
, unsigned long addr
)
24 pgd
= pgd_offset(mm
, addr
);
28 printk("move_one_page: bad source pgd (%08lx)\n", pgd_val(*pgd
));
33 pmd
= pmd_offset(pgd
, addr
);
37 printk("move_one_page: bad source pmd (%08lx)\n", pmd_val(*pmd
));
42 pte
= pte_offset(pmd
, addr
);
49 static inline pte_t
*alloc_one_pte(struct mm_struct
*mm
, unsigned long addr
)
54 pmd
= pmd_alloc(pgd_offset(mm
, addr
), addr
);
56 pte
= pte_alloc(pmd
, addr
);
60 static inline int copy_one_pte(pte_t
* src
, pte_t
* dst
)
76 static int move_one_page(struct mm_struct
*mm
, unsigned long old_addr
, unsigned long new_addr
)
81 src
= get_one_pte(mm
, old_addr
);
83 error
= copy_one_pte(src
, alloc_one_pte(mm
, new_addr
));
87 static int move_page_tables(struct mm_struct
* mm
,
88 unsigned long new_addr
, unsigned long old_addr
, unsigned long len
)
90 unsigned long offset
= len
;
92 flush_cache_range(mm
, old_addr
, old_addr
+ len
);
93 flush_tlb_range(mm
, old_addr
, old_addr
+ len
);
96 * This is not the clever way to do this, but we're taking the
97 * easy way out on the assumption that most remappings will be
98 * only a few pages.. This also makes error recovery easier.
102 if (move_one_page(mm
, old_addr
+ offset
, new_addr
+ offset
))
108 * Ok, the move failed because we didn't have enough pages for
109 * the new page table tree. This is unlikely, but we have to
110 * take the possibility into account. In that case we just move
111 * all the pages back (this will work, because we still have
112 * the old page tables)
115 flush_cache_range(mm
, new_addr
, new_addr
+ len
);
116 while ((offset
+= PAGE_SIZE
) < len
)
117 move_one_page(mm
, new_addr
+ offset
, old_addr
+ offset
);
118 zap_page_range(mm
, new_addr
, new_addr
+ len
);
119 flush_tlb_range(mm
, new_addr
, new_addr
+ len
);
123 static inline unsigned long move_vma(struct vm_area_struct
* vma
,
124 unsigned long addr
, unsigned long old_len
, unsigned long new_len
)
126 struct vm_area_struct
* new_vma
;
128 new_vma
= kmem_cache_alloc(vm_area_cachep
, SLAB_KERNEL
);
130 unsigned long new_addr
= get_unmapped_area(addr
, new_len
);
132 if (new_addr
&& !move_page_tables(current
->mm
, new_addr
, addr
, old_len
)) {
134 new_vma
->vm_start
= new_addr
;
135 new_vma
->vm_end
= new_addr
+new_len
;
136 new_vma
->vm_offset
= vma
->vm_offset
+ (addr
- vma
->vm_start
);
137 if (new_vma
->vm_file
)
138 new_vma
->vm_file
->f_count
++;
139 if (new_vma
->vm_ops
&& new_vma
->vm_ops
->open
)
140 new_vma
->vm_ops
->open(new_vma
);
141 insert_vm_struct(current
->mm
, new_vma
);
142 merge_segments(current
->mm
, new_vma
->vm_start
, new_vma
->vm_end
);
143 do_munmap(addr
, old_len
);
144 current
->mm
->total_vm
+= new_len
>> PAGE_SHIFT
;
145 if (new_vma
->vm_flags
& VM_LOCKED
) {
146 current
->mm
->locked_vm
+= new_len
>> PAGE_SHIFT
;
147 make_pages_present(new_vma
->vm_start
,
152 kmem_cache_free(vm_area_cachep
, new_vma
);
158 * Expand (or shrink) an existing mapping, potentially moving it at the
159 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
161 asmlinkage
unsigned long sys_mremap(unsigned long addr
,
162 unsigned long old_len
, unsigned long new_len
,
165 struct vm_area_struct
*vma
;
166 unsigned long ret
= -EINVAL
;
168 down(¤t
->mm
->mmap_sem
);
170 if (addr
& ~PAGE_MASK
)
172 old_len
= PAGE_ALIGN(old_len
);
173 new_len
= PAGE_ALIGN(new_len
);
176 * Always allow a shrinking remap: that just unmaps
177 * the unnecessary pages..
180 if (old_len
>= new_len
) {
181 do_munmap(addr
+new_len
, old_len
- new_len
);
186 * Ok, we need to grow..
189 vma
= find_vma(current
->mm
, addr
);
190 if (!vma
|| vma
->vm_start
> addr
)
192 /* We can't remap across vm area boundaries */
193 if (old_len
> vma
->vm_end
- addr
)
195 if (vma
->vm_flags
& VM_LOCKED
) {
196 unsigned long locked
= current
->mm
->locked_vm
<< PAGE_SHIFT
;
197 locked
+= new_len
- old_len
;
199 if (locked
> current
->rlim
[RLIMIT_MEMLOCK
].rlim_cur
)
203 if ((current
->mm
->total_vm
<< PAGE_SHIFT
) + (new_len
- old_len
)
204 > current
->rlim
[RLIMIT_AS
].rlim_cur
)
206 /* Private writable mapping? Check memory availability.. */
207 if ((vma
->vm_flags
& (VM_SHARED
| VM_WRITE
)) == VM_WRITE
&&
208 !(flags
& MAP_NORESERVE
) &&
209 !vm_enough_memory((new_len
- old_len
) >> PAGE_SHIFT
))
212 /* old_len exactly to the end of the area.. */
213 if (old_len
== vma
->vm_end
- addr
&&
214 (old_len
!= new_len
|| !(flags
& MREMAP_MAYMOVE
))) {
215 unsigned long max_addr
= TASK_SIZE
;
217 max_addr
= vma
->vm_next
->vm_start
;
218 /* can we just expand the current mapping? */
219 if (max_addr
- addr
>= new_len
) {
220 int pages
= (new_len
- old_len
) >> PAGE_SHIFT
;
221 vma
->vm_end
= addr
+ new_len
;
222 current
->mm
->total_vm
+= pages
;
223 if (vma
->vm_flags
& VM_LOCKED
) {
224 current
->mm
->locked_vm
+= pages
;
225 make_pages_present(addr
+ old_len
,
234 * We weren't able to just expand or shrink the area,
235 * we need to create a new one and move it..
237 if (flags
& MREMAP_MAYMOVE
)
238 ret
= move_vma(vma
, addr
, old_len
, new_len
);
243 up(¤t
->mm
->mmap_sem
);