2 ** Machine code management.
3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
15 #include "lj_dispatch.h"
17 #if LJ_HASJIT || LJ_HASFFI
21 /* -- OS-specific functions ----------------------------------------------- */
23 #if LJ_HASJIT || LJ_HASFFI
25 /* Define this if you want to run LuaJIT with Valgrind. */
26 #ifdef LUAJIT_USE_VALGRIND
27 #include <valgrind/valgrind.h>
31 void sys_icache_invalidate(void *start
, size_t len
);
34 /* Synchronize data/instruction cache. */
35 void lj_mcode_sync(void *start
, void *end
)
37 #ifdef LUAJIT_USE_VALGRIND
38 VALGRIND_DISCARD_TRANSLATIONS(start
, (char *)end
-(char *)start
);
40 #if LJ_TARGET_X86ORX64
41 UNUSED(start
); UNUSED(end
);
43 sys_icache_invalidate(start
, (char *)end
-(char *)start
);
45 lj_vm_cachesync(start
, end
);
46 #elif defined(__GNUC__)
47 __clear_cache(start
, end
);
49 #error "Missing builtin to flush instruction cache"
59 #define WIN32_LEAN_AND_MEAN
62 #define MCPROT_RW PAGE_READWRITE
63 #define MCPROT_RX PAGE_EXECUTE_READ
64 #define MCPROT_RWX PAGE_EXECUTE_READWRITE
66 static void *mcode_alloc_at(jit_State
*J
, uintptr_t hint
, size_t sz
, DWORD prot
)
68 void *p
= VirtualAlloc((void *)hint
, sz
,
69 MEM_RESERVE
|MEM_COMMIT
|MEM_TOP_DOWN
, prot
);
71 lj_trace_err(J
, LJ_TRERR_MCODEAL
);
75 static void mcode_free(jit_State
*J
, void *p
, size_t sz
)
77 UNUSED(J
); UNUSED(sz
);
78 VirtualFree(p
, 0, MEM_RELEASE
);
81 static void mcode_setprot(void *p
, size_t sz
, DWORD prot
)
84 VirtualProtect(p
, sz
, prot
, &oprot
);
92 #define MAP_ANONYMOUS MAP_ANON
95 #define MCPROT_RW (PROT_READ|PROT_WRITE)
96 #define MCPROT_RX (PROT_READ|PROT_EXEC)
97 #define MCPROT_RWX (PROT_READ|PROT_WRITE|PROT_EXEC)
99 static void *mcode_alloc_at(jit_State
*J
, uintptr_t hint
, size_t sz
, int prot
)
101 void *p
= mmap((void *)hint
, sz
, prot
, MAP_PRIVATE
|MAP_ANONYMOUS
, -1, 0);
102 if (p
== MAP_FAILED
&& !hint
)
103 lj_trace_err(J
, LJ_TRERR_MCODEAL
);
107 static void mcode_free(jit_State
*J
, void *p
, size_t sz
)
113 static void mcode_setprot(void *p
, size_t sz
, int prot
)
115 mprotect(p
, sz
, prot
);
120 #error "Missing OS support for explicit placement of executable memory"
124 /* Fallback allocator. This will fail if memory is not executable by default. */
125 #define LUAJIT_UNPROTECT_MCODE
130 static void *mcode_alloc_at(jit_State
*J
, uintptr_t hint
, size_t sz
, int prot
)
132 UNUSED(hint
); UNUSED(prot
);
133 return lj_mem_new(J
->L
, sz
);
136 static void mcode_free(jit_State
*J
, void *p
, size_t sz
)
138 lj_mem_free(J2G(J
), p
, sz
);
141 #define mcode_setprot(p, sz, prot) UNUSED(p)
145 /* -- MCode area protection ----------------------------------------------- */
147 /* Define this ONLY if the page protection twiddling becomes a bottleneck. */
148 #ifdef LUAJIT_UNPROTECT_MCODE
150 /* It's generally considered to be a potential security risk to have
151 ** pages with simultaneous write *and* execute access in a process.
153 ** Do not even think about using this mode for server processes or
154 ** apps handling untrusted external data (such as a browser).
156 ** The security risk is not in LuaJIT itself -- but if an adversary finds
157 ** any *other* flaw in your C application logic, then any RWX memory page
158 ** simplifies writing an exploit considerably.
160 #define MCPROT_GEN MCPROT_RWX
161 #define MCPROT_RUN MCPROT_RWX
163 static void mcode_protect(jit_State
*J
, int prot
)
165 UNUSED(J
); UNUSED(prot
);
170 /* This is the default behaviour and much safer:
172 ** Most of the time the memory pages holding machine code are executable,
173 ** but NONE of them is writable.
175 ** The current memory area is marked read-write (but NOT executable) only
176 ** during the short time window while the assembler generates machine code.
178 #define MCPROT_GEN MCPROT_RW
179 #define MCPROT_RUN MCPROT_RX
181 /* Change protection of MCode area. */
182 static void mcode_protect(jit_State
*J
, int prot
)
184 if (J
->mcprot
!= prot
) {
185 mcode_setprot(J
->mcarea
, J
->szmcarea
, prot
);
192 /* -- MCode area allocation ----------------------------------------------- */
195 #define mcode_validptr(p) ((p) && (uintptr_t)(p) < (uintptr_t)1<<47)
197 #define mcode_validptr(p) ((p) && (uintptr_t)(p) < 0xffff0000)
200 #ifdef LJ_TARGET_JUMPRANGE
202 /* Get memory within relative jump distance of our code in 64 bit mode. */
203 static void *mcode_alloc(jit_State
*J
, size_t sz
)
205 /* Target an address in the static assembler code (64K aligned).
206 ** Try addresses within a distance of target-range/2+1MB..target+range/2-1MB.
209 /* Use the middle of the 256MB-aligned region. */
210 uintptr_t target
= ((uintptr_t)(void *)lj_vm_exit_handler
& 0xf0000000u
) +
213 uintptr_t target
= (uintptr_t)(void *)lj_vm_exit_handler
& ~(uintptr_t)0xffff;
215 const uintptr_t range
= (1u << LJ_TARGET_JUMPRANGE
) - (1u << 21);
216 /* First try a contiguous area below the last one. */
217 uintptr_t hint
= J
->mcarea
? (uintptr_t)J
->mcarea
- sz
: 0;
219 for (i
= 0; i
< 32; i
++) { /* 32 attempts ought to be enough ... */
220 if (mcode_validptr(hint
)) {
221 void *p
= mcode_alloc_at(J
, hint
, sz
, MCPROT_GEN
);
223 if (mcode_validptr(p
)) {
224 if ((uintptr_t)p
+ sz
- target
< range
|| target
- (uintptr_t)p
< range
)
226 mcode_free(J
, p
, sz
); /* Free badly placed area. */
229 /* Next try probing pseudo-random addresses. */
231 hint
= (0x78fb ^ LJ_PRNG_BITS(J
, 15)) << 16; /* 64K aligned. */
232 } while (!(hint
+ sz
< range
));
233 hint
= target
+ hint
- (range
>>1);
235 lj_trace_err(J
, LJ_TRERR_MCODEAL
); /* Give up. OS probably ignores hints? */
241 /* All memory addresses are reachable by relative jumps. */
242 #define mcode_alloc(J, sz) mcode_alloc_at((J), 0, (sz), MCPROT_GEN)
246 /* -- MCode area management ----------------------------------------------- */
248 /* Linked list of MCode areas. */
249 typedef struct MCLink
{
250 MCode
*next
; /* Next area. */
251 size_t size
; /* Size of current area. */
254 /* Allocate a new MCode area. */
255 static void mcode_allocarea(jit_State
*J
)
257 MCode
*oldarea
= J
->mcarea
;
258 size_t sz
= (size_t)J
->param
[JIT_P_sizemcode
] << 10;
259 sz
= (sz
+ LJ_PAGESIZE
-1) & ~(size_t)(LJ_PAGESIZE
- 1);
260 J
->mcarea
= (MCode
*)mcode_alloc(J
, sz
);
262 J
->mcprot
= MCPROT_GEN
;
263 J
->mctop
= (MCode
*)((char *)J
->mcarea
+ J
->szmcarea
);
264 J
->mcbot
= (MCode
*)((char *)J
->mcarea
+ sizeof(MCLink
));
265 ((MCLink
*)J
->mcarea
)->next
= oldarea
;
266 ((MCLink
*)J
->mcarea
)->size
= sz
;
267 J
->szallmcarea
+= sz
;
270 /* Free all MCode areas. */
271 void lj_mcode_free(jit_State
*J
)
273 MCode
*mc
= J
->mcarea
;
277 MCode
*next
= ((MCLink
*)mc
)->next
;
278 mcode_free(J
, mc
, ((MCLink
*)mc
)->size
);
283 /* -- MCode transactions -------------------------------------------------- */
285 /* Reserve the remainder of the current MCode area. */
286 MCode
*lj_mcode_reserve(jit_State
*J
, MCode
**lim
)
291 mcode_protect(J
, MCPROT_GEN
);
296 /* Commit the top part of the current MCode area. */
297 void lj_mcode_commit(jit_State
*J
, MCode
*top
)
300 mcode_protect(J
, MCPROT_RUN
);
303 /* Abort the reservation. */
304 void lj_mcode_abort(jit_State
*J
)
306 mcode_protect(J
, MCPROT_RUN
);
309 /* Set/reset protection to allow patching of MCode areas. */
310 MCode
*lj_mcode_patch(jit_State
*J
, MCode
*ptr
, int finish
)
312 #ifdef LUAJIT_UNPROTECT_MCODE
313 UNUSED(J
); UNUSED(ptr
); UNUSED(finish
);
317 if (J
->mcarea
== ptr
)
318 mcode_protect(J
, MCPROT_RUN
);
320 mcode_setprot(ptr
, ((MCLink
*)ptr
)->size
, MCPROT_RUN
);
323 MCode
*mc
= J
->mcarea
;
324 /* Try current area first to use the protection cache. */
325 if (ptr
>= mc
&& ptr
< (MCode
*)((char *)mc
+ J
->szmcarea
)) {
326 mcode_protect(J
, MCPROT_GEN
);
329 /* Otherwise search through the list of MCode areas. */
331 mc
= ((MCLink
*)mc
)->next
;
332 lua_assert(mc
!= NULL
);
333 if (ptr
>= mc
&& ptr
< (MCode
*)((char *)mc
+ ((MCLink
*)mc
)->size
)) {
334 mcode_setprot(mc
, ((MCLink
*)mc
)->size
, MCPROT_GEN
);
342 /* Limit of MCode reservation reached. */
343 void lj_mcode_limiterr(jit_State
*J
, size_t need
)
345 size_t sizemcode
, maxmcode
;
347 sizemcode
= (size_t)J
->param
[JIT_P_sizemcode
] << 10;
348 sizemcode
= (sizemcode
+ LJ_PAGESIZE
-1) & ~(size_t)(LJ_PAGESIZE
- 1);
349 maxmcode
= (size_t)J
->param
[JIT_P_maxmcode
] << 10;
350 if ((size_t)need
> sizemcode
)
351 lj_trace_err(J
, LJ_TRERR_MCODEOV
); /* Too long for any area. */
352 if (J
->szallmcarea
+ sizemcode
> maxmcode
)
353 lj_trace_err(J
, LJ_TRERR_MCODEAL
);
355 lj_trace_err(J
, LJ_TRERR_MCODELM
); /* Retry with new area. */