4 * (C) Copyright 1996 Linus Torvalds
7 #include <linux/slab.h>
8 #include <linux/smp_lock.h>
10 #include <linux/mman.h>
11 #include <linux/swap.h>
13 #include <asm/uaccess.h>
14 #include <asm/pgtable.h>
16 extern int vm_enough_memory(long pages
);
18 static inline pte_t
*get_one_pte(struct mm_struct
*mm
, unsigned long addr
)
24 pgd
= pgd_offset(mm
, addr
);
28 printk("move_one_page: bad source pgd (%08lx)\n", pgd_val(*pgd
));
33 pmd
= pmd_offset(pgd
, addr
);
37 printk("move_one_page: bad source pmd (%08lx)\n", pmd_val(*pmd
));
42 pte
= pte_offset(pmd
, addr
);
49 static inline pte_t
*alloc_one_pte(struct mm_struct
*mm
, unsigned long addr
)
54 pmd
= pmd_alloc(pgd_offset(mm
, addr
), addr
);
56 pte
= pte_alloc(pmd
, addr
);
60 static inline int copy_one_pte(pte_t
* src
, pte_t
* dst
)
76 static int move_one_page(struct mm_struct
*mm
, unsigned long old_addr
, unsigned long new_addr
)
81 src
= get_one_pte(mm
, old_addr
);
83 error
= copy_one_pte(src
, alloc_one_pte(mm
, new_addr
));
87 static int move_page_tables(struct mm_struct
* mm
,
88 unsigned long new_addr
, unsigned long old_addr
, unsigned long len
)
90 unsigned long offset
= len
;
92 flush_cache_range(mm
, old_addr
, old_addr
+ len
);
93 flush_tlb_range(mm
, old_addr
, old_addr
+ len
);
96 * This is not the clever way to do this, but we're taking the
97 * easy way out on the assumption that most remappings will be
98 * only a few pages.. This also makes error recovery easier.
102 if (move_one_page(mm
, old_addr
+ offset
, new_addr
+ offset
))
108 * Ok, the move failed because we didn't have enough pages for
109 * the new page table tree. This is unlikely, but we have to
110 * take the possibility into account. In that case we just move
111 * all the pages back (this will work, because we still have
112 * the old page tables)
115 flush_cache_range(mm
, new_addr
, new_addr
+ len
);
116 while ((offset
+= PAGE_SIZE
) < len
)
117 move_one_page(mm
, new_addr
+ offset
, old_addr
+ offset
);
118 zap_page_range(mm
, new_addr
, new_addr
+ len
);
119 flush_tlb_range(mm
, new_addr
, new_addr
+ len
);
123 static inline unsigned long move_vma(struct vm_area_struct
* vma
,
124 unsigned long addr
, unsigned long old_len
, unsigned long new_len
)
126 struct vm_area_struct
* new_vma
;
128 new_vma
= kmem_cache_alloc(vm_area_cachep
, SLAB_KERNEL
);
130 unsigned long new_addr
= get_unmapped_area(addr
, new_len
);
132 if (new_addr
&& !move_page_tables(current
->mm
, new_addr
, addr
, old_len
)) {
134 new_vma
->vm_start
= new_addr
;
135 new_vma
->vm_end
= new_addr
+new_len
;
136 new_vma
->vm_offset
= vma
->vm_offset
+ (addr
- vma
->vm_start
);
138 if (new_vma
->vm_file
)
139 atomic_inc(&new_vma
->vm_file
->f_count
);
140 if (new_vma
->vm_ops
&& new_vma
->vm_ops
->open
)
141 new_vma
->vm_ops
->open(new_vma
);
142 insert_vm_struct(current
->mm
, new_vma
);
143 merge_segments(current
->mm
, new_vma
->vm_start
, new_vma
->vm_end
);
145 do_munmap(addr
, old_len
);
146 current
->mm
->total_vm
+= new_len
>> PAGE_SHIFT
;
147 if (new_vma
->vm_flags
& VM_LOCKED
) {
148 current
->mm
->locked_vm
+= new_len
>> PAGE_SHIFT
;
149 make_pages_present(new_vma
->vm_start
,
154 kmem_cache_free(vm_area_cachep
, new_vma
);
160 * Expand (or shrink) an existing mapping, potentially moving it at the
161 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
163 asmlinkage
unsigned long sys_mremap(unsigned long addr
,
164 unsigned long old_len
, unsigned long new_len
,
167 struct vm_area_struct
*vma
;
168 unsigned long ret
= -EINVAL
;
170 down(¤t
->mm
->mmap_sem
);
171 if (addr
& ~PAGE_MASK
)
173 old_len
= PAGE_ALIGN(old_len
);
174 new_len
= PAGE_ALIGN(new_len
);
177 * Always allow a shrinking remap: that just unmaps
178 * the unnecessary pages..
181 if (old_len
>= new_len
) {
182 do_munmap(addr
+new_len
, old_len
- new_len
);
187 * Ok, we need to grow..
190 vma
= find_vma(current
->mm
, addr
);
191 if (!vma
|| vma
->vm_start
> addr
)
193 /* We can't remap across vm area boundaries */
194 if (old_len
> vma
->vm_end
- addr
)
196 if (vma
->vm_flags
& VM_LOCKED
) {
197 unsigned long locked
= current
->mm
->locked_vm
<< PAGE_SHIFT
;
198 locked
+= new_len
- old_len
;
200 if (locked
> current
->rlim
[RLIMIT_MEMLOCK
].rlim_cur
)
204 if ((current
->mm
->total_vm
<< PAGE_SHIFT
) + (new_len
- old_len
)
205 > current
->rlim
[RLIMIT_AS
].rlim_cur
)
207 /* Private writable mapping? Check memory availability.. */
208 if ((vma
->vm_flags
& (VM_SHARED
| VM_WRITE
)) == VM_WRITE
&&
209 !(flags
& MAP_NORESERVE
) &&
210 !vm_enough_memory((new_len
- old_len
) >> PAGE_SHIFT
))
213 /* old_len exactly to the end of the area.. */
214 if (old_len
== vma
->vm_end
- addr
&&
215 (old_len
!= new_len
|| !(flags
& MREMAP_MAYMOVE
))) {
216 unsigned long max_addr
= TASK_SIZE
;
218 max_addr
= vma
->vm_next
->vm_start
;
219 /* can we just expand the current mapping? */
220 if (max_addr
- addr
>= new_len
) {
221 int pages
= (new_len
- old_len
) >> PAGE_SHIFT
;
222 vma
->vm_end
= addr
+ new_len
;
223 current
->mm
->total_vm
+= pages
;
224 if (vma
->vm_flags
& VM_LOCKED
) {
225 current
->mm
->locked_vm
+= pages
;
226 make_pages_present(addr
+ old_len
,
235 * We weren't able to just expand or shrink the area,
236 * we need to create a new one and move it..
238 if (flags
& MREMAP_MAYMOVE
)
239 ret
= move_vma(vma
, addr
, old_len
, new_len
);
243 up(¤t
->mm
->mmap_sem
);