2 * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
3 * Licensed under the GPL
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <asm/pgtable.h>
10 #include <asm/tlbflush.h>
11 #include "as-layout.h"
16 struct host_vm_change
{
18 enum { NONE
, MMAP
, MUNMAP
, MPROTECT
} type
;
44 #define INIT_HVC(mm, force) \
45 ((struct host_vm_change) \
46 { .ops = { { .type = NONE } }, \
47 .id = &mm->context.id, \
52 static int do_ops(struct host_vm_change
*hvc
, int end
,
55 struct host_vm_op
*op
;
58 for (i
= 0; i
< end
&& !ret
; i
++) {
62 ret
= map(hvc
->id
, op
->u
.mmap
.addr
, op
->u
.mmap
.len
,
63 op
->u
.mmap
.prot
, op
->u
.mmap
.fd
,
64 op
->u
.mmap
.offset
, finished
, &hvc
->data
);
67 ret
= unmap(hvc
->id
, op
->u
.munmap
.addr
,
68 op
->u
.munmap
.len
, finished
, &hvc
->data
);
71 ret
= protect(hvc
->id
, op
->u
.mprotect
.addr
,
72 op
->u
.mprotect
.len
, op
->u
.mprotect
.prot
,
73 finished
, &hvc
->data
);
76 printk(KERN_ERR
"Unknown op type %d in do_ops\n",
85 static int add_mmap(unsigned long virt
, unsigned long phys
, unsigned long len
,
86 unsigned int prot
, struct host_vm_change
*hvc
)
89 struct host_vm_op
*last
;
92 fd
= phys_mapping(phys
, &offset
);
93 if (hvc
->index
!= 0) {
94 last
= &hvc
->ops
[hvc
->index
- 1];
95 if ((last
->type
== MMAP
) &&
96 (last
->u
.mmap
.addr
+ last
->u
.mmap
.len
== virt
) &&
97 (last
->u
.mmap
.prot
== prot
) && (last
->u
.mmap
.fd
== fd
) &&
98 (last
->u
.mmap
.offset
+ last
->u
.mmap
.len
== offset
)) {
99 last
->u
.mmap
.len
+= len
;
104 if (hvc
->index
== ARRAY_SIZE(hvc
->ops
)) {
105 ret
= do_ops(hvc
, ARRAY_SIZE(hvc
->ops
), 0);
109 hvc
->ops
[hvc
->index
++] = ((struct host_vm_op
)
111 .u
= { .mmap
= { .addr
= virt
,
120 static int add_munmap(unsigned long addr
, unsigned long len
,
121 struct host_vm_change
*hvc
)
123 struct host_vm_op
*last
;
126 if (hvc
->index
!= 0) {
127 last
= &hvc
->ops
[hvc
->index
- 1];
128 if ((last
->type
== MUNMAP
) &&
129 (last
->u
.munmap
.addr
+ last
->u
.mmap
.len
== addr
)) {
130 last
->u
.munmap
.len
+= len
;
135 if (hvc
->index
== ARRAY_SIZE(hvc
->ops
)) {
136 ret
= do_ops(hvc
, ARRAY_SIZE(hvc
->ops
), 0);
140 hvc
->ops
[hvc
->index
++] = ((struct host_vm_op
)
142 .u
= { .munmap
= { .addr
= addr
,
147 static int add_mprotect(unsigned long addr
, unsigned long len
,
148 unsigned int prot
, struct host_vm_change
*hvc
)
150 struct host_vm_op
*last
;
153 if (hvc
->index
!= 0) {
154 last
= &hvc
->ops
[hvc
->index
- 1];
155 if ((last
->type
== MPROTECT
) &&
156 (last
->u
.mprotect
.addr
+ last
->u
.mprotect
.len
== addr
) &&
157 (last
->u
.mprotect
.prot
== prot
)) {
158 last
->u
.mprotect
.len
+= len
;
163 if (hvc
->index
== ARRAY_SIZE(hvc
->ops
)) {
164 ret
= do_ops(hvc
, ARRAY_SIZE(hvc
->ops
), 0);
168 hvc
->ops
[hvc
->index
++] = ((struct host_vm_op
)
170 .u
= { .mprotect
= { .addr
= addr
,
176 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
178 static inline int update_pte_range(pmd_t
*pmd
, unsigned long addr
,
180 struct host_vm_change
*hvc
)
183 int r
, w
, x
, prot
, ret
= 0;
185 pte
= pte_offset_kernel(pmd
, addr
);
187 if ((addr
>= STUB_START
) && (addr
< STUB_END
))
193 if (!pte_young(*pte
)) {
196 } else if (!pte_dirty(*pte
))
199 prot
= ((r
? UM_PROT_READ
: 0) | (w
? UM_PROT_WRITE
: 0) |
200 (x
? UM_PROT_EXEC
: 0));
201 if (hvc
->force
|| pte_newpage(*pte
)) {
202 if (pte_present(*pte
))
203 ret
= add_mmap(addr
, pte_val(*pte
) & PAGE_MASK
,
204 PAGE_SIZE
, prot
, hvc
);
206 ret
= add_munmap(addr
, PAGE_SIZE
, hvc
);
207 } else if (pte_newprot(*pte
))
208 ret
= add_mprotect(addr
, PAGE_SIZE
, prot
, hvc
);
209 *pte
= pte_mkuptodate(*pte
);
210 } while (pte
++, addr
+= PAGE_SIZE
, ((addr
< end
) && !ret
));
214 static inline int update_pmd_range(pud_t
*pud
, unsigned long addr
,
216 struct host_vm_change
*hvc
)
222 pmd
= pmd_offset(pud
, addr
);
224 next
= pmd_addr_end(addr
, end
);
225 if (!pmd_present(*pmd
)) {
226 if (hvc
->force
|| pmd_newpage(*pmd
)) {
227 ret
= add_munmap(addr
, next
- addr
, hvc
);
228 pmd_mkuptodate(*pmd
);
231 else ret
= update_pte_range(pmd
, addr
, next
, hvc
);
232 } while (pmd
++, addr
= next
, ((addr
< end
) && !ret
));
236 static inline int update_pud_range(pgd_t
*pgd
, unsigned long addr
,
238 struct host_vm_change
*hvc
)
244 pud
= pud_offset(pgd
, addr
);
246 next
= pud_addr_end(addr
, end
);
247 if (!pud_present(*pud
)) {
248 if (hvc
->force
|| pud_newpage(*pud
)) {
249 ret
= add_munmap(addr
, next
- addr
, hvc
);
250 pud_mkuptodate(*pud
);
253 else ret
= update_pmd_range(pud
, addr
, next
, hvc
);
254 } while (pud
++, addr
= next
, ((addr
< end
) && !ret
));
258 void fix_range_common(struct mm_struct
*mm
, unsigned long start_addr
,
259 unsigned long end_addr
, int force
)
262 struct host_vm_change hvc
;
263 unsigned long addr
= start_addr
, next
;
266 hvc
= INIT_HVC(mm
, force
);
267 pgd
= pgd_offset(mm
, addr
);
269 next
= pgd_addr_end(addr
, end_addr
);
270 if (!pgd_present(*pgd
)) {
271 if (force
|| pgd_newpage(*pgd
)) {
272 ret
= add_munmap(addr
, next
- addr
, &hvc
);
273 pgd_mkuptodate(*pgd
);
276 else ret
= update_pud_range(pgd
, addr
, next
, &hvc
);
277 } while (pgd
++, addr
= next
, ((addr
< end_addr
) && !ret
));
280 ret
= do_ops(&hvc
, hvc
.index
, 1);
282 /* This is not an else because ret is modified above */
284 printk(KERN_ERR
"fix_range_common: failed, killing current "
286 force_sig(SIGKILL
, current
);
290 static int flush_tlb_kernel_range_common(unsigned long start
, unsigned long end
)
292 struct mm_struct
*mm
;
297 unsigned long addr
, last
;
298 int updated
= 0, err
;
301 for (addr
= start
; addr
< end
;) {
302 pgd
= pgd_offset(mm
, addr
);
303 if (!pgd_present(*pgd
)) {
304 last
= ADD_ROUND(addr
, PGDIR_SIZE
);
307 if (pgd_newpage(*pgd
)) {
309 err
= os_unmap_memory((void *) addr
,
312 panic("munmap failed, errno = %d\n",
319 pud
= pud_offset(pgd
, addr
);
320 if (!pud_present(*pud
)) {
321 last
= ADD_ROUND(addr
, PUD_SIZE
);
324 if (pud_newpage(*pud
)) {
326 err
= os_unmap_memory((void *) addr
,
329 panic("munmap failed, errno = %d\n",
336 pmd
= pmd_offset(pud
, addr
);
337 if (!pmd_present(*pmd
)) {
338 last
= ADD_ROUND(addr
, PMD_SIZE
);
341 if (pmd_newpage(*pmd
)) {
343 err
= os_unmap_memory((void *) addr
,
346 panic("munmap failed, errno = %d\n",
353 pte
= pte_offset_kernel(pmd
, addr
);
354 if (!pte_present(*pte
) || pte_newpage(*pte
)) {
356 err
= os_unmap_memory((void *) addr
,
359 panic("munmap failed, errno = %d\n",
361 if (pte_present(*pte
))
363 pte_val(*pte
) & PAGE_MASK
,
366 else if (pte_newprot(*pte
)) {
368 os_protect_memory((void *) addr
, PAGE_SIZE
, 1, 1, 1);
375 void flush_tlb_page(struct vm_area_struct
*vma
, unsigned long address
)
381 struct mm_struct
*mm
= vma
->vm_mm
;
383 int r
, w
, x
, prot
, err
= 0;
386 address
&= PAGE_MASK
;
387 pgd
= pgd_offset(mm
, address
);
388 if (!pgd_present(*pgd
))
391 pud
= pud_offset(pgd
, address
);
392 if (!pud_present(*pud
))
395 pmd
= pmd_offset(pud
, address
);
396 if (!pmd_present(*pmd
))
399 pte
= pte_offset_kernel(pmd
, address
);
404 if (!pte_young(*pte
)) {
407 } else if (!pte_dirty(*pte
)) {
411 mm_id
= &mm
->context
.id
;
412 prot
= ((r
? UM_PROT_READ
: 0) | (w
? UM_PROT_WRITE
: 0) |
413 (x
? UM_PROT_EXEC
: 0));
414 if (pte_newpage(*pte
)) {
415 if (pte_present(*pte
)) {
416 unsigned long long offset
;
419 fd
= phys_mapping(pte_val(*pte
) & PAGE_MASK
, &offset
);
420 err
= map(mm_id
, address
, PAGE_SIZE
, prot
, fd
, offset
,
423 else err
= unmap(mm_id
, address
, PAGE_SIZE
, 1, &flush
);
425 else if (pte_newprot(*pte
))
426 err
= protect(mm_id
, address
, PAGE_SIZE
, prot
, 1, &flush
);
431 *pte
= pte_mkuptodate(*pte
);
436 printk(KERN_ERR
"Failed to flush page for address 0x%lx\n", address
);
437 force_sig(SIGKILL
, current
);
440 pgd_t
*pgd_offset_proc(struct mm_struct
*mm
, unsigned long address
)
442 return pgd_offset(mm
, address
);
445 pud_t
*pud_offset_proc(pgd_t
*pgd
, unsigned long address
)
447 return pud_offset(pgd
, address
);
450 pmd_t
*pmd_offset_proc(pud_t
*pud
, unsigned long address
)
452 return pmd_offset(pud
, address
);
455 pte_t
*pte_offset_proc(pmd_t
*pmd
, unsigned long address
)
457 return pte_offset_kernel(pmd
, address
);
460 pte_t
*addr_pte(struct task_struct
*task
, unsigned long addr
)
462 pgd_t
*pgd
= pgd_offset(task
->mm
, addr
);
463 pud_t
*pud
= pud_offset(pgd
, addr
);
464 pmd_t
*pmd
= pmd_offset(pud
, addr
);
466 return pte_offset_map(pmd
, addr
);
469 void flush_tlb_all(void)
471 flush_tlb_mm(current
->mm
);
474 void flush_tlb_kernel_range(unsigned long start
, unsigned long end
)
476 flush_tlb_kernel_range_common(start
, end
);
479 void flush_tlb_kernel_vm(void)
481 flush_tlb_kernel_range_common(start_vm
, end_vm
);
484 void __flush_tlb_one(unsigned long addr
)
486 flush_tlb_kernel_range_common(addr
, addr
+ PAGE_SIZE
);
489 static void fix_range(struct mm_struct
*mm
, unsigned long start_addr
,
490 unsigned long end_addr
, int force
)
492 fix_range_common(mm
, start_addr
, end_addr
, force
);
495 void flush_tlb_range(struct vm_area_struct
*vma
, unsigned long start
,
498 if (vma
->vm_mm
== NULL
)
499 flush_tlb_kernel_range_common(start
, end
);
500 else fix_range(vma
->vm_mm
, start
, end
, 0);
502 EXPORT_SYMBOL(flush_tlb_range
);
504 void flush_tlb_mm_range(struct mm_struct
*mm
, unsigned long start
,
508 * Don't bother flushing if this address space is about to be
511 if (atomic_read(&mm
->mm_users
) == 0)
514 fix_range(mm
, start
, end
, 0);
517 void flush_tlb_mm(struct mm_struct
*mm
)
519 struct vm_area_struct
*vma
= mm
->mmap
;
521 while (vma
!= NULL
) {
522 fix_range(mm
, vma
->vm_start
, vma
->vm_end
, 0);
527 void force_flush_all(void)
529 struct mm_struct
*mm
= current
->mm
;
530 struct vm_area_struct
*vma
= mm
->mmap
;
532 while (vma
!= NULL
) {
533 fix_range(mm
, vma
->vm_start
, vma
->vm_end
, 1);