4 * (C) Copyright 1996 Linus Torvalds
7 #include <linux/slab.h>
8 #include <linux/smp_lock.h>
10 #include <linux/mman.h>
11 #include <linux/swap.h>
13 #include <asm/uaccess.h>
14 #include <asm/pgtable.h>
16 extern int vm_enough_memory(long pages
);
18 static inline pte_t
*get_one_pte(struct mm_struct
*mm
, unsigned long addr
)
24 pgd
= pgd_offset(mm
, addr
);
28 printk("move_one_page: bad source pgd (%08lx)\n", pgd_val(*pgd
));
33 pmd
= pmd_offset(pgd
, addr
);
37 printk("move_one_page: bad source pmd (%08lx)\n", pmd_val(*pmd
));
42 pte
= pte_offset(pmd
, addr
);
49 static inline pte_t
*alloc_one_pte(struct mm_struct
*mm
, unsigned long addr
)
54 pmd
= pmd_alloc(pgd_offset(mm
, addr
), addr
);
56 pte
= pte_alloc(pmd
, addr
);
60 static inline int copy_one_pte(struct mm_struct
*mm
, pte_t
* src
, pte_t
* dst
)
65 spin_lock(&mm
->page_table_lock
);
75 spin_unlock(&mm
->page_table_lock
);
79 static int move_one_page(struct mm_struct
*mm
, unsigned long old_addr
, unsigned long new_addr
)
84 src
= get_one_pte(mm
, old_addr
);
86 error
= copy_one_pte(mm
, src
, alloc_one_pte(mm
, new_addr
));
90 static int move_page_tables(struct mm_struct
* mm
,
91 unsigned long new_addr
, unsigned long old_addr
, unsigned long len
)
93 unsigned long offset
= len
;
95 flush_cache_range(mm
, old_addr
, old_addr
+ len
);
96 flush_tlb_range(mm
, old_addr
, old_addr
+ len
);
99 * This is not the clever way to do this, but we're taking the
100 * easy way out on the assumption that most remappings will be
101 * only a few pages.. This also makes error recovery easier.
105 if (move_one_page(mm
, old_addr
+ offset
, new_addr
+ offset
))
111 * Ok, the move failed because we didn't have enough pages for
112 * the new page table tree. This is unlikely, but we have to
113 * take the possibility into account. In that case we just move
114 * all the pages back (this will work, because we still have
115 * the old page tables)
118 flush_cache_range(mm
, new_addr
, new_addr
+ len
);
119 while ((offset
+= PAGE_SIZE
) < len
)
120 move_one_page(mm
, new_addr
+ offset
, old_addr
+ offset
);
121 zap_page_range(mm
, new_addr
, new_addr
+ len
);
122 flush_tlb_range(mm
, new_addr
, new_addr
+ len
);
126 static inline unsigned long move_vma(struct vm_area_struct
* vma
,
127 unsigned long addr
, unsigned long old_len
, unsigned long new_len
)
129 struct vm_area_struct
* new_vma
;
131 new_vma
= kmem_cache_alloc(vm_area_cachep
, SLAB_KERNEL
);
133 unsigned long new_addr
= get_unmapped_area(addr
, new_len
);
135 if (new_addr
&& !move_page_tables(current
->mm
, new_addr
, addr
, old_len
)) {
137 new_vma
->vm_start
= new_addr
;
138 new_vma
->vm_end
= new_addr
+new_len
;
139 new_vma
->vm_offset
= vma
->vm_offset
+ (addr
- vma
->vm_start
);
140 if (new_vma
->vm_file
)
141 get_file(new_vma
->vm_file
);
142 if (new_vma
->vm_ops
&& new_vma
->vm_ops
->open
)
143 new_vma
->vm_ops
->open(new_vma
);
144 insert_vm_struct(current
->mm
, new_vma
);
145 merge_segments(current
->mm
, new_vma
->vm_start
, new_vma
->vm_end
);
146 do_munmap(addr
, old_len
);
147 current
->mm
->total_vm
+= new_len
>> PAGE_SHIFT
;
148 if (new_vma
->vm_flags
& VM_LOCKED
) {
149 current
->mm
->locked_vm
+= new_len
>> PAGE_SHIFT
;
150 make_pages_present(new_vma
->vm_start
,
155 kmem_cache_free(vm_area_cachep
, new_vma
);
161 * Expand (or shrink) an existing mapping, potentially moving it at the
162 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
164 asmlinkage
unsigned long sys_mremap(unsigned long addr
,
165 unsigned long old_len
, unsigned long new_len
,
168 struct vm_area_struct
*vma
;
169 unsigned long ret
= -EINVAL
;
171 down(¤t
->mm
->mmap_sem
);
172 if (addr
& ~PAGE_MASK
)
174 old_len
= PAGE_ALIGN(old_len
);
175 new_len
= PAGE_ALIGN(new_len
);
178 * Always allow a shrinking remap: that just unmaps
179 * the unnecessary pages..
182 if (old_len
>= new_len
) {
183 do_munmap(addr
+new_len
, old_len
- new_len
);
188 * Ok, we need to grow..
191 vma
= find_vma(current
->mm
, addr
);
192 if (!vma
|| vma
->vm_start
> addr
)
194 /* We can't remap across vm area boundaries */
195 if (old_len
> vma
->vm_end
- addr
)
197 if (vma
->vm_flags
& VM_LOCKED
) {
198 unsigned long locked
= current
->mm
->locked_vm
<< PAGE_SHIFT
;
199 locked
+= new_len
- old_len
;
201 if (locked
> current
->rlim
[RLIMIT_MEMLOCK
].rlim_cur
)
205 if ((current
->mm
->total_vm
<< PAGE_SHIFT
) + (new_len
- old_len
)
206 > current
->rlim
[RLIMIT_AS
].rlim_cur
)
208 /* Private writable mapping? Check memory availability.. */
209 if ((vma
->vm_flags
& (VM_SHARED
| VM_WRITE
)) == VM_WRITE
&&
210 !(flags
& MAP_NORESERVE
) &&
211 !vm_enough_memory((new_len
- old_len
) >> PAGE_SHIFT
))
214 /* old_len exactly to the end of the area.. */
215 if (old_len
== vma
->vm_end
- addr
&&
216 (old_len
!= new_len
|| !(flags
& MREMAP_MAYMOVE
))) {
217 unsigned long max_addr
= TASK_SIZE
;
219 max_addr
= vma
->vm_next
->vm_start
;
220 /* can we just expand the current mapping? */
221 if (max_addr
- addr
>= new_len
) {
222 int pages
= (new_len
- old_len
) >> PAGE_SHIFT
;
223 vma
->vm_end
= addr
+ new_len
;
224 current
->mm
->total_vm
+= pages
;
225 if (vma
->vm_flags
& VM_LOCKED
) {
226 current
->mm
->locked_vm
+= pages
;
227 make_pages_present(addr
+ old_len
,
236 * We weren't able to just expand or shrink the area,
237 * we need to create a new one and move it..
239 if (flags
& MREMAP_MAYMOVE
)
240 ret
= move_vma(vma
, addr
, old_len
, new_len
);
244 up(¤t
->mm
->mmap_sem
);