2 ** Machine code management.
3 ** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
15 #include "lj_dispatch.h"
17 #if LJ_HASJIT || LJ_HASFFI
21 /* -- OS-specific functions ----------------------------------------------- */
23 #if LJ_HASJIT || LJ_HASFFI
25 /* Define this if you want to run LuaJIT with Valgrind. */
26 #ifdef LUAJIT_USE_VALGRIND
27 #include <valgrind/valgrind.h>
31 void sys_icache_invalidate(void *start
, size_t len
);
34 /* Synchronize data/instruction cache. */
35 void lj_mcode_sync(void *start
, void *end
)
37 #ifdef LUAJIT_USE_VALGRIND
38 VALGRIND_DISCARD_TRANSLATIONS(start
, (char *)end
-(char *)start
);
40 #if LJ_TARGET_X86ORX64
41 UNUSED(start
); UNUSED(end
);
43 sys_icache_invalidate(start
, (char *)end
-(char *)start
);
45 lj_vm_cachesync(start
, end
);
46 #elif defined(__GNUC__)
47 __clear_cache(start
, end
);
49 #error "Missing builtin to flush instruction cache"
59 #define WIN32_LEAN_AND_MEAN
62 #define MCPROT_RW PAGE_READWRITE
63 #define MCPROT_RX PAGE_EXECUTE_READ
64 #define MCPROT_RWX PAGE_EXECUTE_READWRITE
66 static void *mcode_alloc_at(jit_State
*J
, uintptr_t hint
, size_t sz
, DWORD prot
)
68 void *p
= VirtualAlloc((void *)hint
, sz
,
69 MEM_RESERVE
|MEM_COMMIT
|MEM_TOP_DOWN
, prot
);
71 lj_trace_err(J
, LJ_TRERR_MCODEAL
);
75 static void mcode_free(jit_State
*J
, void *p
, size_t sz
)
77 UNUSED(J
); UNUSED(sz
);
78 VirtualFree(p
, 0, MEM_RELEASE
);
81 static void mcode_setprot(void *p
, size_t sz
, DWORD prot
)
84 VirtualProtect(p
, sz
, prot
, &oprot
);
92 #define MAP_ANONYMOUS MAP_ANON
95 #define MCPROT_RW (PROT_READ|PROT_WRITE)
96 #define MCPROT_RX (PROT_READ|PROT_EXEC)
97 #define MCPROT_RWX (PROT_READ|PROT_WRITE|PROT_EXEC)
99 static void *mcode_alloc_at(jit_State
*J
, uintptr_t hint
, size_t sz
, int prot
)
101 void *p
= mmap((void *)hint
, sz
, prot
, MAP_PRIVATE
|MAP_ANONYMOUS
, -1, 0);
102 if (p
== MAP_FAILED
) {
103 if (!hint
) lj_trace_err(J
, LJ_TRERR_MCODEAL
);
109 static void mcode_free(jit_State
*J
, void *p
, size_t sz
)
115 static void mcode_setprot(void *p
, size_t sz
, int prot
)
117 mprotect(p
, sz
, prot
);
122 #error "Missing OS support for explicit placement of executable memory"
126 /* Fallback allocator. This will fail if memory is not executable by default. */
127 #define LUAJIT_UNPROTECT_MCODE
132 static void *mcode_alloc_at(jit_State
*J
, uintptr_t hint
, size_t sz
, int prot
)
134 UNUSED(hint
); UNUSED(prot
);
135 return lj_mem_new(J
->L
, sz
);
138 static void mcode_free(jit_State
*J
, void *p
, size_t sz
)
140 lj_mem_free(J2G(J
), p
, sz
);
143 #define mcode_setprot(p, sz, prot) UNUSED(p)
147 /* -- MCode area protection ----------------------------------------------- */
149 /* Define this ONLY if the page protection twiddling becomes a bottleneck. */
150 #ifdef LUAJIT_UNPROTECT_MCODE
152 /* It's generally considered to be a potential security risk to have
153 ** pages with simultaneous write *and* execute access in a process.
155 ** Do not even think about using this mode for server processes or
156 ** apps handling untrusted external data (such as a browser).
158 ** The security risk is not in LuaJIT itself -- but if an adversary finds
159 ** any *other* flaw in your C application logic, then any RWX memory page
160 ** simplifies writing an exploit considerably.
162 #define MCPROT_GEN MCPROT_RWX
163 #define MCPROT_RUN MCPROT_RWX
165 static void mcode_protect(jit_State
*J
, int prot
)
167 UNUSED(J
); UNUSED(prot
);
172 /* This is the default behaviour and much safer:
174 ** Most of the time the memory pages holding machine code are executable,
175 ** but NONE of them is writable.
177 ** The current memory area is marked read-write (but NOT executable) only
178 ** during the short time window while the assembler generates machine code.
180 #define MCPROT_GEN MCPROT_RW
181 #define MCPROT_RUN MCPROT_RX
183 /* Change protection of MCode area. */
184 static void mcode_protect(jit_State
*J
, int prot
)
186 if (J
->mcprot
!= prot
) {
187 mcode_setprot(J
->mcarea
, J
->szmcarea
, prot
);
194 /* -- MCode area allocation ----------------------------------------------- */
197 #define mcode_validptr(p) ((p) && (uintptr_t)(p) < (uintptr_t)1<<47)
199 #define mcode_validptr(p) ((p) && (uintptr_t)(p) < 0xffff0000)
202 #ifdef LJ_TARGET_JUMPRANGE
204 /* Get memory within relative jump distance of our code in 64 bit mode. */
205 static void *mcode_alloc(jit_State
*J
, size_t sz
)
207 /* Target an address in the static assembler code (64K aligned).
208 ** Try addresses within a distance of target-range/2+1MB..target+range/2-1MB.
211 /* Use the middle of the 256MB-aligned region. */
212 uintptr_t target
= ((uintptr_t)(void *)lj_vm_exit_handler
& 0xf0000000u
) +
215 uintptr_t target
= (uintptr_t)(void *)lj_vm_exit_handler
& ~(uintptr_t)0xffff;
217 const uintptr_t range
= (1u << LJ_TARGET_JUMPRANGE
) - (1u << 21);
218 /* First try a contiguous area below the last one. */
219 uintptr_t hint
= J
->mcarea
? (uintptr_t)J
->mcarea
- sz
: 0;
221 for (i
= 0; i
< 32; i
++) { /* 32 attempts ought to be enough ... */
222 if (mcode_validptr(hint
)) {
223 void *p
= mcode_alloc_at(J
, hint
, sz
, MCPROT_GEN
);
225 if (mcode_validptr(p
) &&
226 ((uintptr_t)p
+ sz
- target
< range
|| target
- (uintptr_t)p
< range
))
228 if (p
) mcode_free(J
, p
, sz
); /* Free badly placed area. */
230 /* Next try probing pseudo-random addresses. */
232 hint
= (0x78fb ^ LJ_PRNG_BITS(J
, 15)) << 16; /* 64K aligned. */
233 } while (!(hint
+ sz
< range
));
234 hint
= target
+ hint
- (range
>>1);
236 lj_trace_err(J
, LJ_TRERR_MCODEAL
); /* Give up. OS probably ignores hints? */
242 /* All memory addresses are reachable by relative jumps. */
243 #define mcode_alloc(J, sz) mcode_alloc_at((J), 0, (sz), MCPROT_GEN)
247 /* -- MCode area management ----------------------------------------------- */
249 /* Linked list of MCode areas. */
250 typedef struct MCLink
{
251 MCode
*next
; /* Next area. */
252 size_t size
; /* Size of current area. */
255 /* Allocate a new MCode area. */
256 static void mcode_allocarea(jit_State
*J
)
258 MCode
*oldarea
= J
->mcarea
;
259 size_t sz
= (size_t)J
->param
[JIT_P_sizemcode
] << 10;
260 sz
= (sz
+ LJ_PAGESIZE
-1) & ~(size_t)(LJ_PAGESIZE
- 1);
261 J
->mcarea
= (MCode
*)mcode_alloc(J
, sz
);
263 J
->mcprot
= MCPROT_GEN
;
264 J
->mctop
= (MCode
*)((char *)J
->mcarea
+ J
->szmcarea
);
265 J
->mcbot
= (MCode
*)((char *)J
->mcarea
+ sizeof(MCLink
));
266 ((MCLink
*)J
->mcarea
)->next
= oldarea
;
267 ((MCLink
*)J
->mcarea
)->size
= sz
;
268 J
->szallmcarea
+= sz
;
271 /* Free all MCode areas. */
272 void lj_mcode_free(jit_State
*J
)
274 MCode
*mc
= J
->mcarea
;
278 MCode
*next
= ((MCLink
*)mc
)->next
;
279 mcode_free(J
, mc
, ((MCLink
*)mc
)->size
);
284 /* -- MCode transactions -------------------------------------------------- */
286 /* Reserve the remainder of the current MCode area. */
287 MCode
*lj_mcode_reserve(jit_State
*J
, MCode
**lim
)
292 mcode_protect(J
, MCPROT_GEN
);
297 /* Commit the top part of the current MCode area. */
298 void lj_mcode_commit(jit_State
*J
, MCode
*top
)
301 mcode_protect(J
, MCPROT_RUN
);
304 /* Abort the reservation. */
305 void lj_mcode_abort(jit_State
*J
)
307 mcode_protect(J
, MCPROT_RUN
);
310 /* Set/reset protection to allow patching of MCode areas. */
311 MCode
*lj_mcode_patch(jit_State
*J
, MCode
*ptr
, int finish
)
313 #ifdef LUAJIT_UNPROTECT_MCODE
314 UNUSED(J
); UNUSED(ptr
); UNUSED(finish
);
318 if (J
->mcarea
== ptr
)
319 mcode_protect(J
, MCPROT_RUN
);
321 mcode_setprot(ptr
, ((MCLink
*)ptr
)->size
, MCPROT_RUN
);
324 MCode
*mc
= J
->mcarea
;
325 /* Try current area first to use the protection cache. */
326 if (ptr
>= mc
&& ptr
< (MCode
*)((char *)mc
+ J
->szmcarea
)) {
327 mcode_protect(J
, MCPROT_GEN
);
330 /* Otherwise search through the list of MCode areas. */
332 mc
= ((MCLink
*)mc
)->next
;
333 lua_assert(mc
!= NULL
);
334 if (ptr
>= mc
&& ptr
< (MCode
*)((char *)mc
+ ((MCLink
*)mc
)->size
)) {
335 mcode_setprot(mc
, ((MCLink
*)mc
)->size
, MCPROT_GEN
);
343 /* Limit of MCode reservation reached. */
344 void lj_mcode_limiterr(jit_State
*J
, size_t need
)
346 size_t sizemcode
, maxmcode
;
348 sizemcode
= (size_t)J
->param
[JIT_P_sizemcode
] << 10;
349 sizemcode
= (sizemcode
+ LJ_PAGESIZE
-1) & ~(size_t)(LJ_PAGESIZE
- 1);
350 maxmcode
= (size_t)J
->param
[JIT_P_maxmcode
] << 10;
351 if ((size_t)need
> sizemcode
)
352 lj_trace_err(J
, LJ_TRERR_MCODEOV
); /* Too long for any area. */
353 if (J
->szallmcarea
+ sizemcode
> maxmcode
)
354 lj_trace_err(J
, LJ_TRERR_MCODEAL
);
356 lj_trace_err(J
, LJ_TRERR_MCODELM
); /* Retry with new area. */