2 * mmap support for qemu
4 * Copyright (c) 2003 Fabrice Bellard
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
24 static pthread_mutex_t mmap_mutex
= PTHREAD_MUTEX_INITIALIZER
;
25 static __thread
int mmap_lock_count
;
29 if (mmap_lock_count
++ == 0) {
30 pthread_mutex_lock(&mmap_mutex
);
34 void mmap_unlock(void)
36 if (--mmap_lock_count
== 0) {
37 pthread_mutex_unlock(&mmap_mutex
);
41 bool have_mmap_lock(void)
43 return mmap_lock_count
> 0 ? true : false;
46 /* Grab lock to make sure things are in a consistent state after fork(). */
47 void mmap_fork_start(void)
51 pthread_mutex_lock(&mmap_mutex
);
54 void mmap_fork_end(int child
)
57 pthread_mutex_init(&mmap_mutex
, NULL
);
59 pthread_mutex_unlock(&mmap_mutex
);
63 * Validate target prot bitmask.
64 * Return the prot bitmask for the host in *HOST_PROT.
65 * Return 0 if the target prot bitmask is invalid, otherwise
66 * the internal qemu page_flags (which will include PAGE_VALID).
68 static int validate_prot_to_pageflags(int *host_prot
, int prot
)
70 int valid
= PROT_READ
| PROT_WRITE
| PROT_EXEC
| TARGET_PROT_SEM
;
71 int page_flags
= (prot
& PAGE_BITS
) | PAGE_VALID
;
74 * For the host, we need not pass anything except read/write/exec.
75 * While PROT_SEM is allowed by all hosts, it is also ignored, so
76 * don't bother transforming guest bit to host bit. Any other
77 * target-specific prot bits will not be understood by the host
78 * and will need to be encoded into page_flags for qemu emulation.
80 * Pages that are executable by the guest will never be executed
81 * by the host, but the host will need to be able to read them.
83 *host_prot
= (prot
& (PROT_READ
| PROT_WRITE
))
84 | (prot
& PROT_EXEC
? PROT_READ
: 0);
88 * The PROT_BTI bit is only accepted if the cpu supports the feature.
89 * Since this is the unusual case, don't bother checking unless
90 * the bit has been requested. If set and valid, record the bit
91 * within QEMU's page_flags.
93 if (prot
& TARGET_PROT_BTI
) {
94 ARMCPU
*cpu
= ARM_CPU(thread_cpu
);
95 if (cpu_isar_feature(aa64_bti
, cpu
)) {
96 valid
|= TARGET_PROT_BTI
;
97 page_flags
|= PAGE_BTI
;
102 return prot
& ~valid
? 0 : page_flags
;
105 /* NOTE: all the constants are the HOST ones, but addresses are target. */
106 int target_mprotect(abi_ulong start
, abi_ulong len
, int target_prot
)
108 abi_ulong end
, host_start
, host_end
, addr
;
109 int prot1
, ret
, page_flags
, host_prot
;
111 trace_target_mprotect(start
, len
, target_prot
);
113 if ((start
& ~TARGET_PAGE_MASK
) != 0) {
114 return -TARGET_EINVAL
;
116 page_flags
= validate_prot_to_pageflags(&host_prot
, target_prot
);
118 return -TARGET_EINVAL
;
120 len
= TARGET_PAGE_ALIGN(len
);
122 if (!guest_range_valid(start
, len
)) {
123 return -TARGET_ENOMEM
;
130 host_start
= start
& qemu_host_page_mask
;
131 host_end
= HOST_PAGE_ALIGN(end
);
132 if (start
> host_start
) {
133 /* handle host page containing start */
135 for (addr
= host_start
; addr
< start
; addr
+= TARGET_PAGE_SIZE
) {
136 prot1
|= page_get_flags(addr
);
138 if (host_end
== host_start
+ qemu_host_page_size
) {
139 for (addr
= end
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
140 prot1
|= page_get_flags(addr
);
144 ret
= mprotect(g2h(host_start
), qemu_host_page_size
,
149 host_start
+= qemu_host_page_size
;
151 if (end
< host_end
) {
153 for (addr
= end
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
154 prot1
|= page_get_flags(addr
);
156 ret
= mprotect(g2h(host_end
- qemu_host_page_size
),
157 qemu_host_page_size
, prot1
& PAGE_BITS
);
161 host_end
-= qemu_host_page_size
;
164 /* handle the pages in the middle */
165 if (host_start
< host_end
) {
166 ret
= mprotect(g2h(host_start
), host_end
- host_start
, host_prot
);
171 page_set_flags(start
, start
+ len
, page_flags
);
179 /* map an incomplete host page */
180 static int mmap_frag(abi_ulong real_start
,
181 abi_ulong start
, abi_ulong end
,
182 int prot
, int flags
, int fd
, abi_ulong offset
)
184 abi_ulong real_end
, addr
;
188 real_end
= real_start
+ qemu_host_page_size
;
189 host_start
= g2h(real_start
);
191 /* get the protection of the target pages outside the mapping */
193 for(addr
= real_start
; addr
< real_end
; addr
++) {
194 if (addr
< start
|| addr
>= end
)
195 prot1
|= page_get_flags(addr
);
199 /* no page was there, so we allocate one */
200 void *p
= mmap(host_start
, qemu_host_page_size
, prot
,
201 flags
| MAP_ANONYMOUS
, -1, 0);
208 prot_new
= prot
| prot1
;
209 if (!(flags
& MAP_ANONYMOUS
)) {
210 /* msync() won't work here, so we return an error if write is
211 possible while it is a shared mapping */
212 if ((flags
& MAP_TYPE
) == MAP_SHARED
&&
216 /* adjust protection to be able to read */
217 if (!(prot1
& PROT_WRITE
))
218 mprotect(host_start
, qemu_host_page_size
, prot1
| PROT_WRITE
);
220 /* read the corresponding file data */
221 if (pread(fd
, g2h(start
), end
- start
, offset
) == -1)
224 /* put final protection */
225 if (prot_new
!= (prot1
| PROT_WRITE
))
226 mprotect(host_start
, qemu_host_page_size
, prot_new
);
228 if (prot_new
!= prot1
) {
229 mprotect(host_start
, qemu_host_page_size
, prot_new
);
231 if (prot_new
& PROT_WRITE
) {
232 memset(g2h(start
), 0, end
- start
);
238 #if HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 64
239 #ifdef TARGET_AARCH64
240 # define TASK_UNMAPPED_BASE 0x5500000000
242 # define TASK_UNMAPPED_BASE (1ul << 38)
245 # define TASK_UNMAPPED_BASE 0x40000000
247 abi_ulong mmap_next_start
= TASK_UNMAPPED_BASE
;
249 unsigned long last_brk
;
251 /* Subroutine of mmap_find_vma, used when we have pre-allocated a chunk
252 of guest address space. */
253 static abi_ulong
mmap_find_vma_reserved(abi_ulong start
, abi_ulong size
,
256 abi_ulong addr
, end_addr
, incr
= qemu_host_page_size
;
260 if (size
> reserved_va
) {
261 return (abi_ulong
)-1;
264 /* Note that start and size have already been aligned by mmap_find_vma. */
266 end_addr
= start
+ size
;
267 if (start
> reserved_va
- size
) {
268 /* Start at the top of the address space. */
269 end_addr
= ((reserved_va
- size
) & -align
) + size
;
273 /* Search downward from END_ADDR, checking to see if a page is in use. */
277 if (addr
> end_addr
) {
279 /* Failure. The entire address space has been searched. */
280 return (abi_ulong
)-1;
282 /* Re-start at the top of the address space. */
283 addr
= end_addr
= ((reserved_va
- size
) & -align
) + size
;
286 prot
= page_get_flags(addr
);
288 /* Page in use. Restart below this page. */
289 addr
= end_addr
= ((addr
- size
) & -align
) + size
;
290 } else if (addr
&& addr
+ size
== end_addr
) {
291 /* Success! All pages between ADDR and END_ADDR are free. */
292 if (start
== mmap_next_start
) {
293 mmap_next_start
= addr
;
302 * Find and reserve a free memory area of size 'size'. The search
304 * It must be called with mmap_lock() held.
305 * Return -1 if error.
307 abi_ulong
mmap_find_vma(abi_ulong start
, abi_ulong size
, abi_ulong align
)
313 align
= MAX(align
, qemu_host_page_size
);
315 /* If 'start' == 0, then a default start address is used. */
317 start
= mmap_next_start
;
319 start
&= qemu_host_page_mask
;
321 start
= ROUND_UP(start
, align
);
323 size
= HOST_PAGE_ALIGN(size
);
326 return mmap_find_vma_reserved(start
, size
, align
);
330 wrapped
= repeat
= 0;
333 for (;; prev
= ptr
) {
335 * Reserve needed memory area to avoid a race.
336 * It should be discarded using:
337 * - mmap() with MAP_FIXED flag
338 * - mremap() with MREMAP_FIXED flag
339 * - shmat() with SHM_REMAP flag
341 ptr
= mmap(g2h(addr
), size
, PROT_NONE
,
342 MAP_ANONYMOUS
|MAP_PRIVATE
|MAP_NORESERVE
, -1, 0);
344 /* ENOMEM, if host address space has no memory */
345 if (ptr
== MAP_FAILED
) {
346 return (abi_ulong
)-1;
349 /* Count the number of sequential returns of the same address.
350 This is used to modify the search algorithm below. */
351 repeat
= (ptr
== prev
? repeat
+ 1 : 0);
353 if (h2g_valid(ptr
+ size
- 1)) {
356 if ((addr
& (align
- 1)) == 0) {
358 if (start
== mmap_next_start
&& addr
>= TASK_UNMAPPED_BASE
) {
359 mmap_next_start
= addr
+ size
;
364 /* The address is not properly aligned for the target. */
367 /* Assume the result that the kernel gave us is the
368 first with enough free space, so start again at the
369 next higher target page. */
370 addr
= ROUND_UP(addr
, align
);
373 /* Sometimes the kernel decides to perform the allocation
374 at the top end of memory instead. */
378 /* Start over at low memory. */
382 /* Fail. This unaligned block must the last. */
387 /* Since the result the kernel gave didn't fit, start
388 again at low memory. If any repetition, fail. */
389 addr
= (repeat
? -1 : 0);
392 /* Unmap and try again. */
395 /* ENOMEM if we checked the whole of the target address space. */
396 if (addr
== (abi_ulong
)-1) {
397 return (abi_ulong
)-1;
398 } else if (addr
== 0) {
400 return (abi_ulong
)-1;
403 /* Don't actually use 0 when wrapping, instead indicate
404 that we'd truly like an allocation in low memory. */
405 addr
= (mmap_min_addr
> TARGET_PAGE_SIZE
406 ? TARGET_PAGE_ALIGN(mmap_min_addr
)
408 } else if (wrapped
&& addr
>= start
) {
409 return (abi_ulong
)-1;
414 /* NOTE: all the constants are the HOST ones */
415 abi_long
target_mmap(abi_ulong start
, abi_ulong len
, int target_prot
,
416 int flags
, int fd
, abi_ulong offset
)
418 abi_ulong ret
, end
, real_start
, real_end
, retaddr
, host_offset
, host_len
;
419 int page_flags
, host_prot
;
422 trace_target_mmap(start
, len
, target_prot
, flags
, fd
, offset
);
429 page_flags
= validate_prot_to_pageflags(&host_prot
, target_prot
);
435 /* Also check for overflows... */
436 len
= TARGET_PAGE_ALIGN(len
);
442 if (offset
& ~TARGET_PAGE_MASK
) {
447 real_start
= start
& qemu_host_page_mask
;
448 host_offset
= offset
& qemu_host_page_mask
;
450 /* If the user is asking for the kernel to find a location, do that
451 before we truncate the length for mapping files below. */
452 if (!(flags
& MAP_FIXED
)) {
453 host_len
= len
+ offset
- host_offset
;
454 host_len
= HOST_PAGE_ALIGN(host_len
);
455 start
= mmap_find_vma(real_start
, host_len
, TARGET_PAGE_SIZE
);
456 if (start
== (abi_ulong
)-1) {
462 /* When mapping files into a memory area larger than the file, accesses
463 to pages beyond the file size will cause a SIGBUS.
465 For example, if mmaping a file of 100 bytes on a host with 4K pages
466 emulating a target with 8K pages, the target expects to be able to
467 access the first 8K. But the host will trap us on any access beyond
470 When emulating a target with a larger page-size than the hosts, we
471 may need to truncate file maps at EOF and add extra anonymous pages
472 up to the targets page boundary. */
474 if ((qemu_real_host_page_size
< qemu_host_page_size
) &&
475 !(flags
& MAP_ANONYMOUS
)) {
478 if (fstat (fd
, &sb
) == -1)
481 /* Are we trying to create a map beyond EOF?. */
482 if (offset
+ len
> sb
.st_size
) {
483 /* If so, truncate the file map at eof aligned with
484 the hosts real pagesize. Additional anonymous maps
485 will be created beyond EOF. */
486 len
= REAL_HOST_PAGE_ALIGN(sb
.st_size
- offset
);
490 if (!(flags
& MAP_FIXED
)) {
491 unsigned long host_start
;
494 host_len
= len
+ offset
- host_offset
;
495 host_len
= HOST_PAGE_ALIGN(host_len
);
497 /* Note: we prefer to control the mapping address. It is
498 especially important if qemu_host_page_size >
499 qemu_real_host_page_size */
500 p
= mmap(g2h(start
), host_len
, host_prot
,
501 flags
| MAP_FIXED
| MAP_ANONYMOUS
, -1, 0);
502 if (p
== MAP_FAILED
) {
505 /* update start so that it points to the file position at 'offset' */
506 host_start
= (unsigned long)p
;
507 if (!(flags
& MAP_ANONYMOUS
)) {
508 p
= mmap(g2h(start
), len
, host_prot
,
509 flags
| MAP_FIXED
, fd
, host_offset
);
510 if (p
== MAP_FAILED
) {
511 munmap(g2h(start
), host_len
);
514 host_start
+= offset
- host_offset
;
516 start
= h2g(host_start
);
518 if (start
& ~TARGET_PAGE_MASK
) {
523 real_end
= HOST_PAGE_ALIGN(end
);
526 * Test if requested memory area fits target address space
527 * It can fail only on 64-bit host with 32-bit target.
528 * On any other target/host host mmap() handles this error correctly.
530 if (end
< start
|| !guest_range_valid(start
, len
)) {
535 /* worst case: we cannot map the file because the offset is not
536 aligned, so we read it */
537 if (!(flags
& MAP_ANONYMOUS
) &&
538 (offset
& ~qemu_host_page_mask
) != (start
& ~qemu_host_page_mask
)) {
539 /* msync() won't work here, so we return an error if write is
540 possible while it is a shared mapping */
541 if ((flags
& MAP_TYPE
) == MAP_SHARED
&&
542 (host_prot
& PROT_WRITE
)) {
546 retaddr
= target_mmap(start
, len
, target_prot
| PROT_WRITE
,
547 MAP_FIXED
| MAP_PRIVATE
| MAP_ANONYMOUS
,
551 if (pread(fd
, g2h(start
), len
, offset
) == -1)
553 if (!(host_prot
& PROT_WRITE
)) {
554 ret
= target_mprotect(start
, len
, target_prot
);
560 /* handle the start of the mapping */
561 if (start
> real_start
) {
562 if (real_end
== real_start
+ qemu_host_page_size
) {
563 /* one single host page */
564 ret
= mmap_frag(real_start
, start
, end
,
565 host_prot
, flags
, fd
, offset
);
570 ret
= mmap_frag(real_start
, start
, real_start
+ qemu_host_page_size
,
571 host_prot
, flags
, fd
, offset
);
574 real_start
+= qemu_host_page_size
;
576 /* handle the end of the mapping */
577 if (end
< real_end
) {
578 ret
= mmap_frag(real_end
- qemu_host_page_size
,
579 real_end
- qemu_host_page_size
, end
,
580 host_prot
, flags
, fd
,
581 offset
+ real_end
- qemu_host_page_size
- start
);
584 real_end
-= qemu_host_page_size
;
587 /* map the middle (easier) */
588 if (real_start
< real_end
) {
590 unsigned long offset1
;
591 if (flags
& MAP_ANONYMOUS
)
594 offset1
= offset
+ real_start
- start
;
595 p
= mmap(g2h(real_start
), real_end
- real_start
,
596 host_prot
, flags
, fd
, offset1
);
602 page_set_flags(start
, start
+ len
, page_flags
);
604 trace_target_mmap_complete(start
);
605 if (qemu_loglevel_mask(CPU_LOG_PAGE
)) {
606 log_page_dump(__func__
);
608 tb_invalidate_phys_range(start
, start
+ len
);
616 static void mmap_reserve(abi_ulong start
, abi_ulong size
)
618 abi_ulong real_start
;
624 real_start
= start
& qemu_host_page_mask
;
625 real_end
= HOST_PAGE_ALIGN(start
+ size
);
627 if (start
> real_start
) {
628 /* handle host page containing start */
630 for (addr
= real_start
; addr
< start
; addr
+= TARGET_PAGE_SIZE
) {
631 prot
|= page_get_flags(addr
);
633 if (real_end
== real_start
+ qemu_host_page_size
) {
634 for (addr
= end
; addr
< real_end
; addr
+= TARGET_PAGE_SIZE
) {
635 prot
|= page_get_flags(addr
);
640 real_start
+= qemu_host_page_size
;
642 if (end
< real_end
) {
644 for (addr
= end
; addr
< real_end
; addr
+= TARGET_PAGE_SIZE
) {
645 prot
|= page_get_flags(addr
);
648 real_end
-= qemu_host_page_size
;
650 if (real_start
!= real_end
) {
651 mmap(g2h(real_start
), real_end
- real_start
, PROT_NONE
,
652 MAP_FIXED
| MAP_ANONYMOUS
| MAP_PRIVATE
| MAP_NORESERVE
,
657 int target_munmap(abi_ulong start
, abi_ulong len
)
659 abi_ulong end
, real_start
, real_end
, addr
;
662 trace_target_munmap(start
, len
);
664 if (start
& ~TARGET_PAGE_MASK
)
665 return -TARGET_EINVAL
;
666 len
= TARGET_PAGE_ALIGN(len
);
667 if (len
== 0 || !guest_range_valid(start
, len
)) {
668 return -TARGET_EINVAL
;
673 real_start
= start
& qemu_host_page_mask
;
674 real_end
= HOST_PAGE_ALIGN(end
);
676 if (start
> real_start
) {
677 /* handle host page containing start */
679 for(addr
= real_start
; addr
< start
; addr
+= TARGET_PAGE_SIZE
) {
680 prot
|= page_get_flags(addr
);
682 if (real_end
== real_start
+ qemu_host_page_size
) {
683 for(addr
= end
; addr
< real_end
; addr
+= TARGET_PAGE_SIZE
) {
684 prot
|= page_get_flags(addr
);
689 real_start
+= qemu_host_page_size
;
691 if (end
< real_end
) {
693 for(addr
= end
; addr
< real_end
; addr
+= TARGET_PAGE_SIZE
) {
694 prot
|= page_get_flags(addr
);
697 real_end
-= qemu_host_page_size
;
701 /* unmap what we can */
702 if (real_start
< real_end
) {
704 mmap_reserve(real_start
, real_end
- real_start
);
706 ret
= munmap(g2h(real_start
), real_end
- real_start
);
711 page_set_flags(start
, start
+ len
, 0);
712 tb_invalidate_phys_range(start
, start
+ len
);
718 abi_long
target_mremap(abi_ulong old_addr
, abi_ulong old_size
,
719 abi_ulong new_size
, unsigned long flags
,
725 if (!guest_range_valid(old_addr
, old_size
) ||
726 ((flags
& MREMAP_FIXED
) &&
727 !guest_range_valid(new_addr
, new_size
))) {
734 if (flags
& MREMAP_FIXED
) {
735 host_addr
= mremap(g2h(old_addr
), old_size
, new_size
,
736 flags
, g2h(new_addr
));
738 if (reserved_va
&& host_addr
!= MAP_FAILED
) {
739 /* If new and old addresses overlap then the above mremap will
740 already have failed with EINVAL. */
741 mmap_reserve(old_addr
, old_size
);
743 } else if (flags
& MREMAP_MAYMOVE
) {
744 abi_ulong mmap_start
;
746 mmap_start
= mmap_find_vma(0, new_size
, TARGET_PAGE_SIZE
);
748 if (mmap_start
== -1) {
750 host_addr
= MAP_FAILED
;
752 host_addr
= mremap(g2h(old_addr
), old_size
, new_size
,
753 flags
| MREMAP_FIXED
, g2h(mmap_start
));
755 mmap_reserve(old_addr
, old_size
);
760 if (reserved_va
&& old_size
< new_size
) {
762 for (addr
= old_addr
+ old_size
;
763 addr
< old_addr
+ new_size
;
765 prot
|= page_get_flags(addr
);
769 host_addr
= mremap(g2h(old_addr
), old_size
, new_size
, flags
);
771 if (host_addr
!= MAP_FAILED
) {
772 /* Check if address fits target address space */
773 if (!guest_range_valid(h2g(host_addr
), new_size
)) {
774 /* Revert mremap() changes */
775 host_addr
= mremap(g2h(old_addr
), new_size
, old_size
,
778 host_addr
= MAP_FAILED
;
779 } else if (reserved_va
&& old_size
> new_size
) {
780 mmap_reserve(old_addr
+ old_size
, old_size
- new_size
);
785 host_addr
= MAP_FAILED
;
789 if (host_addr
== MAP_FAILED
) {
792 new_addr
= h2g(host_addr
);
793 prot
= page_get_flags(old_addr
);
794 page_set_flags(old_addr
, old_addr
+ old_size
, 0);
795 page_set_flags(new_addr
, new_addr
+ new_size
, prot
| PAGE_VALID
);
797 tb_invalidate_phys_range(new_addr
, new_addr
+ new_size
);