| blob | f1251f2c48acff274fda5122f7a07a7d0b9b21e8 |
1 |// Low-level VM code for ARM64 CPUs.
2 |// Bytecode interpreter, fast functions and helper functions.
3 |// Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
4 |
5 |.arch arm64
6 |.section code_op, code_sub
7 |
8 |.actionlist build_actionlist
9 |.globals GLOB_
10 |.globalnames globnames
11 |.externnames extnames
12 |
13 |// Note: The ragged indentation of the instructions is intentional.
14 |// The starting columns indicate data dependencies.
15 |
16 |//-----------------------------------------------------------------------
17 |
18 |// ARM64 registers and the AAPCS64 ABI 1.0 at a glance:
19 |//
20 |// x0-x17 temp, x19-x28 callee-saved, x29 fp, x30 lr
21 |// x18 is reserved on most platforms. Don't use it, save it or restore it.
22 |// x31 doesn't exist. Register number 31 either means xzr/wzr (zero) or sp,
23 |// depending on the instruction.
24 |// v0-v7 temp, v8-v15 callee-saved (only d8-d15 preserved), v16-v31 temp
25 |//
26 |// x0-x7/v0-v7 hold parameters and results.
27 |
28 |// Fixed register assignments for the interpreter.
29 |
30 |// The following must be C callee-save.
31 |.define BASE, x19 // Base of current Lua stack frame.
32 |.define KBASE, x20 // Constants of current Lua function.
33 |.define PC, x21 // Next PC.
34 |.define GLREG, x22 // Global state.
35 |.define LREG, x23 // Register holding lua_State (also in SAVE_L).
36 |.define TISNUM, x24 // Constant LJ_TISNUM << 47.
37 |.define TISNUMhi, x25 // Constant LJ_TISNUM << 15.
38 |.define TISNIL, x26 // Constant -1LL.
39 |.define fp, x29 // Yes, we have to maintain a frame pointer.
40 |
41 |.define ST_INTERP, w26 // Constant -1.
42 |
43 |// The following temporaries are not saved across C calls, except for RA/RC.
44 |.define RA, x27
45 |.define RC, x28
46 |.define RB, x17
47 |.define RAw, w27
48 |.define RCw, w28
49 |.define RBw, w17
50 |.define INS, x16
51 |.define INSw, w16
52 |.define ITYPE, x15
53 |.define TMP0, x8
54 |.define TMP1, x9
55 |.define TMP2, x10
56 |.define TMP3, x11
57 |.define TMP0w, w8
58 |.define TMP1w, w9
59 |.define TMP2w, w10
60 |.define TMP3w, w11
61 |
62 |// Calling conventions. Also used as temporaries.
63 |.define CARG1, x0
64 |.define CARG2, x1
65 |.define CARG3, x2
66 |.define CARG4, x3
67 |.define CARG5, x4
68 |.define CARG1w, w0
69 |.define CARG2w, w1
70 |.define CARG3w, w2
71 |.define CARG4w, w3
72 |.define CARG5w, w4
73 |
74 |.define FARG1, d0
75 |.define FARG2, d1
76 |
77 |.define CRET1, x0
78 |.define CRET1w, w0
79 |
80 |// Stack layout while in interpreter. Must match with lj_frame.h.
81 |
82 |.define CFRAME_SPACE, 208
83 |//----- 16 byte aligned, <-- sp entering interpreter
84 |// Unused [sp, #204] // 32 bit values
85 |.define SAVE_NRES, [sp, #200]
86 |.define SAVE_ERRF, [sp, #196]
87 |.define SAVE_MULTRES, [sp, #192]
88 |.define TMPD, [sp, #184] // 64 bit values
89 |.define SAVE_L, [sp, #176]
90 |.define SAVE_PC, [sp, #168]
91 |.define SAVE_CFRAME, [sp, #160]
92 |.define SAVE_FPR_, 96 // 96+8*8: 64 bit FPR saves
93 |.define SAVE_GPR_, 16 // 16+10*8: 64 bit GPR saves
94 |.define SAVE_LR, [sp, #8]
95 |.define SAVE_FP, [sp]
96 |//----- 16 byte aligned, <-- sp while in interpreter.
97 |
98 |.define TMPDofs, #184
99 |
100 |.macro save_, gpr1, gpr2, fpr1, fpr2
101 | stp d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(fpr1-8)*8]
102 | stp x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(gpr1-19)*8]
103 |.endmacro
104 |.macro rest_, gpr1, gpr2, fpr1, fpr2
105 | ldp d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(fpr1-8)*8]
106 | ldp x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(gpr1-19)*8]
107 |.endmacro
108 |
109 |.macro saveregs
110 | stp fp, lr, [sp, #-CFRAME_SPACE]!
111 | add fp, sp, #0
112 | stp x19, x20, [sp, # SAVE_GPR_]
113 | save_ 21, 22, 8, 9
114 | save_ 23, 24, 10, 11
115 | save_ 25, 26, 12, 13
116 | save_ 27, 28, 14, 15
117 |.endmacro
118 |.macro restoreregs
119 | ldp x19, x20, [sp, # SAVE_GPR_]
120 | rest_ 21, 22, 8, 9
121 | rest_ 23, 24, 10, 11
122 | rest_ 25, 26, 12, 13
123 | rest_ 27, 28, 14, 15
124 | ldp fp, lr, [sp], # CFRAME_SPACE
125 |.endmacro
126 |
127 |// Type definitions. Some of these are only used for documentation.
128 |.type L, lua_State, LREG
129 |.type GL, global_State, GLREG
130 |.type TVALUE, TValue
131 |.type GCOBJ, GCobj
132 |.type STR, GCstr
133 |.type TAB, GCtab
134 |.type LFUNC, GCfuncL
135 |.type CFUNC, GCfuncC
136 |.type PROTO, GCproto
137 |.type UPVAL, GCupval
138 |.type NODE, Node
139 |.type NARGS8, int
140 |.type TRACE, GCtrace
141 |.type SBUF, SBuf
142 |
143 |//-----------------------------------------------------------------------
144 |
145 |// Trap for not-yet-implemented parts.
146 |.macro NYI; brk; .endmacro
147 |
148 |//-----------------------------------------------------------------------
149 |
150 |// Access to frame relative to BASE.
151 |.define FRAME_FUNC, #-16
152 |.define FRAME_PC, #-8
153 |
154 |.macro decode_RA, dst, ins; ubfx dst, ins, #8, #8; .endmacro
155 |.macro decode_RB, dst, ins; ubfx dst, ins, #24, #8; .endmacro
156 |.macro decode_RC, dst, ins; ubfx dst, ins, #16, #8; .endmacro
157 |.macro decode_RD, dst, ins; ubfx dst, ins, #16, #16; .endmacro
158 |.macro decode_RC8RD, dst, src; ubfiz dst, src, #3, #8; .endmacro
159 |
160 |// Instruction decode+dispatch.
161 |.macro ins_NEXT
162 | ldr INSw, [PC], #4
163 | add TMP1, GL, INS, uxtb #3
164 | decode_RA RA, INS
165 | ldr TMP0, [TMP1, #GG_G2DISP]
166 | decode_RD RC, INS
167 | br TMP0
168 |.endmacro
169 |
170 |// Instruction footer.
171 |.if 1
172 | // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
173 | .define ins_next, ins_NEXT
174 | .define ins_next_, ins_NEXT
175 |.else
176 | // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
177 | // Affects only certain kinds of benchmarks (and only with -j off).
178 | .macro ins_next
179 | b ->ins_next
180 | .endmacro
181 | .macro ins_next_
182 | ->ins_next:
183 | ins_NEXT
184 | .endmacro
185 |.endif
186 |
187 |// Call decode and dispatch.
188 |.macro ins_callt
189 | // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
190 | ldr PC, LFUNC:CARG3->pc
191 | ldr INSw, [PC], #4
192 | add TMP1, GL, INS, uxtb #3
193 | decode_RA RA, INS
194 | ldr TMP0, [TMP1, #GG_G2DISP]
195 | add RA, BASE, RA, lsl #3
196 | br TMP0
197 |.endmacro
198 |
199 |.macro ins_call
200 | // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
201 | str PC, [BASE, FRAME_PC]
202 | ins_callt
203 |.endmacro
204 |
205 |//-----------------------------------------------------------------------
206 |
207 |// Macros to check the TValue type and extract the GCobj. Branch on failure.
208 |.macro checktp, reg, tp, target
209 | asr ITYPE, reg, #47
210 | cmn ITYPE, #-tp
211 | and reg, reg, #LJ_GCVMASK
212 | bne target
213 |.endmacro
214 |.macro checktp, dst, reg, tp, target
215 | asr ITYPE, reg, #47
216 | cmn ITYPE, #-tp
217 | and dst, reg, #LJ_GCVMASK
218 | bne target
219 |.endmacro
220 |.macro checkstr, reg, target; checktp reg, LJ_TSTR, target; .endmacro
221 |.macro checktab, reg, target; checktp reg, LJ_TTAB, target; .endmacro
222 |.macro checkfunc, reg, target; checktp reg, LJ_TFUNC, target; .endmacro
223 |.macro checkint, reg, target
224 | cmp TISNUMhi, reg, lsr #32
225 | bne target
226 |.endmacro
227 |.macro checknum, reg, target
228 | cmp TISNUMhi, reg, lsr #32
229 | bls target
230 |.endmacro
231 |.macro checknumber, reg, target
232 | cmp TISNUMhi, reg, lsr #32
233 | blo target
234 |.endmacro
235 |
236 |.macro mov_false, reg; movn reg, #0x8000, lsl #32; .endmacro
237 |.macro mov_true, reg; movn reg, #0x0001, lsl #48; .endmacro
238 |
239 #define GL_J(field) (GG_OFS(J) + (int)offsetof(jit_State, field))
240 |
241 #define PC2PROTO(field) ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
242 |
243 |.macro hotcheck, delta
244 | NYI
245 |.endmacro
246 |
247 |.macro hotloop
248 | hotcheck HOTCOUNT_LOOP
249 | blo ->vm_hotloop
250 |.endmacro
251 |
252 |.macro hotcall
253 | hotcheck HOTCOUNT_CALL
254 | blo ->vm_hotcall
255 |.endmacro
256 |
257 |// Set current VM state.
258 |.macro mv_vmstate, reg, st; movn reg, #LJ_VMST_..st; .endmacro
259 |.macro st_vmstate, reg; str reg, GL->vmstate; .endmacro
260 |
261 |// Move table write barrier back. Overwrites mark and tmp.
262 |.macro barrierback, tab, mark, tmp
263 | ldr tmp, GL->gc.grayagain
264 | and mark, mark, #~LJ_GC_BLACK // black2gray(tab)
265 | str tab, GL->gc.grayagain
266 | strb mark, tab->marked
267 | str tmp, tab->gclist
268 |.endmacro
269 |
270 |//-----------------------------------------------------------------------
272 #if !LJ_DUALNUM
273 #error "Only dual-number mode supported for ARM64 target"
274 #endif
276 /* Generate subroutines used by opcodes and other parts of the VM. */
277 /* The .code_sub section should be last to help static branch prediction. */
278 static void build_subroutines(BuildCtx *ctx)
279 {
280 |.code_sub
281 |
282 |//-----------------------------------------------------------------------
283 |//-- Return handling ----------------------------------------------------
284 |//-----------------------------------------------------------------------
285 |
286 |->vm_returnp:
287 | // See vm_return. Also: RB = previous base.
288 | tbz PC, #2, ->cont_dispatch // (PC & FRAME_P) == 0?
289 |
290 | // Return from pcall or xpcall fast func.
291 | ldr PC, [RB, FRAME_PC] // Fetch PC of previous frame.
292 | mov_true TMP0
293 | mov BASE, RB
294 | // Prepending may overwrite the pcall frame, so do it at the end.
295 | str TMP0, [RA, #-8]! // Prepend true to results.
296 |
297 |->vm_returnc:
298 | adds RC, RC, #8 // RC = (nresults+1)*8.
299 | mov CRET1, #LUA_YIELD
300 | beq ->vm_unwind_c_eh
301 | str RCw, SAVE_MULTRES
302 | ands CARG1, PC, #FRAME_TYPE
303 | beq ->BC_RET_Z // Handle regular return to Lua.
304 |
305 |->vm_return:
306 | // BASE = base, RA = resultptr, RC/MULTRES = (nresults+1)*8, PC = return
307 | // CARG1 = PC & FRAME_TYPE
308 | and RB, PC, #~FRAME_TYPEP
309 | cmp CARG1, #FRAME_C
310 | sub RB, BASE, RB // RB = previous base.
311 | bne ->vm_returnp
312 |
313 | str RB, L->base
314 | ldrsw CARG2, SAVE_NRES // CARG2 = nresults+1.
315 | mv_vmstate TMP0w, C
316 | sub BASE, BASE, #16
317 | subs TMP2, RC, #8
318 | st_vmstate TMP0w
319 | beq >2
320 |1:
321 | subs TMP2, TMP2, #8
322 | ldr TMP0, [RA], #8
323 | str TMP0, [BASE], #8
324 | bne <1
325 |2:
326 | cmp RC, CARG2, lsl #3 // More/less results wanted?
327 | bne >6
328 |3:
329 | str BASE, L->top // Store new top.
330 |
331 |->vm_leave_cp:
332 | ldr RC, SAVE_CFRAME // Restore previous C frame.
333 | mov CRET1, #0 // Ok return status for vm_pcall.
334 | str RC, L->cframe
335 |
336 |->vm_leave_unw:
337 | restoreregs
338 | ret
339 |
340 |6:
341 | bgt >7 // Less results wanted?
342 | // More results wanted. Check stack size and fill up results with nil.
343 | ldr CARG3, L->maxstack
344 | cmp BASE, CARG3
345 | bhs >8
346 | str TISNIL, [BASE], #8
347 | add RC, RC, #8
348 | b <2
349 |
350 |7: // Less results wanted.
351 | cbz CARG2, <3 // LUA_MULTRET+1 case?
352 | sub CARG1, RC, CARG2, lsl #3
353 | sub BASE, BASE, CARG1 // Shrink top.
354 | b <3
355 |
356 |8: // Corner case: need to grow stack for filling up results.
357 | // This can happen if:
358 | // - A C function grows the stack (a lot).
359 | // - The GC shrinks the stack in between.
360 | // - A return back from a lua_call() with (high) nresults adjustment.
361 | str BASE, L->top // Save current top held in BASE (yes).
362 | mov CARG1, L
363 | bl extern lj_state_growstack // (lua_State *L, int n)
364 | ldr BASE, L->top // Need the (realloced) L->top in BASE.
365 | ldrsw CARG2, SAVE_NRES
366 | b <2
367 |
368 |->vm_unwind_c: // Unwind C stack, return from vm_pcall.
369 | // (void *cframe, int errcode)
370 | mov sp, CARG1
371 | mov CRET1, CARG2
372 |->vm_unwind_c_eh: // Landing pad for external unwinder.
373 | ldr L, SAVE_L
374 | mv_vmstate TMP0w, C
375 | ldr GL, L->glref
376 | st_vmstate TMP0w
377 | b ->vm_leave_unw
378 |
379 |->vm_unwind_ff: // Unwind C stack, return from ff pcall.
380 | // (void *cframe)
381 | and sp, CARG1, #CFRAME_RAWMASK
382 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
383 | ldr L, SAVE_L
384 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
385 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
386 | movn TISNIL, #0
387 | mov RC, #16 // 2 results: false + error message.
388 | ldr BASE, L->base
389 | ldr GL, L->glref // Setup pointer to global state.
390 | mov_false TMP0
391 | sub RA, BASE, #8 // Results start at BASE-8.
392 | ldr PC, [BASE, FRAME_PC] // Fetch PC of previous frame.
393 | str TMP0, [BASE, #-8] // Prepend false to error message.
394 | st_vmstate ST_INTERP
395 | b ->vm_returnc
396 |
397 |//-----------------------------------------------------------------------
398 |//-- Grow stack for calls -----------------------------------------------
399 |//-----------------------------------------------------------------------
400 |
401 |->vm_growstack_c: // Grow stack for C function.
402 | // CARG1 = L
403 | mov CARG2, #LUA_MINSTACK
404 | b >2
405 |
406 |->vm_growstack_l: // Grow stack for Lua function.
407 | // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
408 | add RC, BASE, RC
409 | sub RA, RA, BASE
410 | mov CARG1, L
411 | stp BASE, RC, L->base
412 | add PC, PC, #4 // Must point after first instruction.
413 | lsr CARG2, RA, #3
414 |2:
415 | // L->base = new base, L->top = top
416 | str PC, SAVE_PC
417 | bl extern lj_state_growstack // (lua_State *L, int n)
418 | ldp BASE, RC, L->base
419 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
420 | sub NARGS8:RC, RC, BASE
421 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
422 | // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
423 | ins_callt // Just retry the call.
424 |
425 |//-----------------------------------------------------------------------
426 |//-- Entry points into the assembler VM ---------------------------------
427 |//-----------------------------------------------------------------------
428 |
429 |->vm_resume: // Setup C frame and resume thread.
430 | // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
431 | saveregs
432 | mov L, CARG1
433 | ldr GL, L->glref // Setup pointer to global state.
434 | mov BASE, CARG2
435 | str L, SAVE_L
436 | mov PC, #FRAME_CP
437 | str wzr, SAVE_NRES
438 | add TMP0, sp, #CFRAME_RESUME
439 | ldrb TMP1w, L->status
440 | str wzr, SAVE_ERRF
441 | str L, SAVE_PC // Any value outside of bytecode is ok.
442 | str xzr, SAVE_CFRAME
443 | str TMP0, L->cframe
444 | cbz TMP1w, >3
445 |
446 | // Resume after yield (like a return).
447 | str L, GL->cur_L
448 | mov RA, BASE
449 | ldp BASE, CARG1, L->base
450 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
451 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
452 | ldr PC, [BASE, FRAME_PC]
453 | strb wzr, L->status
454 | movn TISNIL, #0
455 | sub RC, CARG1, BASE
456 | ands CARG1, PC, #FRAME_TYPE
457 | add RC, RC, #8
458 | st_vmstate ST_INTERP
459 | str RCw, SAVE_MULTRES
460 | beq ->BC_RET_Z
461 | b ->vm_return
462 |
463 |->vm_pcall: // Setup protected C frame and enter VM.
464 | // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
465 | saveregs
466 | mov PC, #FRAME_CP
467 | str CARG4w, SAVE_ERRF
468 | b >1
469 |
470 |->vm_call: // Setup C frame and enter VM.
471 | // (lua_State *L, TValue *base, int nres1)
472 | saveregs
473 | mov PC, #FRAME_C
474 |
475 |1: // Entry point for vm_pcall above (PC = ftype).
476 | ldr RC, L:CARG1->cframe
477 | str CARG3w, SAVE_NRES
478 | mov L, CARG1
479 | str CARG1, SAVE_L
480 | ldr GL, L->glref // Setup pointer to global state.
481 | mov BASE, CARG2
482 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
483 | str RC, SAVE_CFRAME
484 | str fp, L->cframe // Add our C frame to cframe chain.
485 |
486 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
487 | str L, GL->cur_L
488 | ldp RB, CARG1, L->base // RB = old base (for vmeta_call).
489 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
490 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
491 | add PC, PC, BASE
492 | movn TISNIL, #0
493 | sub PC, PC, RB // PC = frame delta + frame type
494 | sub NARGS8:RC, CARG1, BASE
495 | st_vmstate ST_INTERP
496 |
497 |->vm_call_dispatch:
498 | // RB = old base, BASE = new base, RC = nargs*8, PC = caller PC
499 | ldr CARG3, [BASE, FRAME_FUNC]
500 | checkfunc CARG3, ->vmeta_call
501 |
502 |->vm_call_dispatch_f:
503 | ins_call
504 | // BASE = new base, CARG3 = func, RC = nargs*8, PC = caller PC
505 |
506 |->vm_cpcall: // Setup protected C frame, call C.
507 | // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
508 | saveregs
509 | mov L, CARG1
510 | ldr RA, L:CARG1->stack
511 | str CARG1, SAVE_L
512 | ldr GL, L->glref // Setup pointer to global state.
513 | ldr RB, L->top
514 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
515 | ldr RC, L->cframe
516 | sub RA, RA, RB // Compute -savestack(L, L->top).
517 | str RAw, SAVE_NRES // Neg. delta means cframe w/o frame.
518 | str wzr, SAVE_ERRF // No error function.
519 | str RC, SAVE_CFRAME
520 | str fp, L->cframe // Add our C frame to cframe chain.
521 | str L, GL->cur_L
522 | blr CARG4 // (lua_State *L, lua_CFunction func, void *ud)
523 | mov BASE, CRET1
524 | mov PC, #FRAME_CP
525 | cbnz BASE, <3 // Else continue with the call.
526 | b ->vm_leave_cp // No base? Just remove C frame.
527 |
528 |//-----------------------------------------------------------------------
529 |//-- Metamethod handling ------------------------------------------------
530 |//-----------------------------------------------------------------------
531 |
532 |//-- Continuation dispatch ----------------------------------------------
533 |
534 |->cont_dispatch:
535 | // BASE = meta base, RA = resultptr, RC = (nresults+1)*8
536 | ldr LFUNC:CARG3, [RB, FRAME_FUNC]
537 | ldr CARG1, [BASE, #-32] // Get continuation.
538 | mov CARG4, BASE
539 | mov BASE, RB // Restore caller BASE.
540 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
541 |.if FFI
542 | cmp CARG1, #1
543 |.endif
544 | ldr PC, [CARG4, #-24] // Restore PC from [cont|PC].
545 | ldr CARG3, LFUNC:CARG3->pc
546 | add TMP0, RA, RC
547 | str TISNIL, [TMP0, #-8] // Ensure one valid arg.
548 |.if FFI
549 | bls >1
550 |.endif
551 | ldr KBASE, [CARG3, #PC2PROTO(k)]
552 | // BASE = base, RA = resultptr, CARG4 = meta base
553 | br CARG1
554 |
555 |.if FFI
556 |1:
557 | beq ->cont_ffi_callback // cont = 1: return from FFI callback.
558 | // cont = 0: tailcall from C function.
559 | sub CARG4, CARG4, #32
560 | sub RC, CARG4, BASE
561 | b ->vm_call_tail
562 |.endif
563 |
564 |->cont_cat: // RA = resultptr, CARG4 = meta base
565 | ldr INSw, [PC, #-4]
566 | sub CARG2, CARG4, #32
567 | ldr TMP0, [RA]
568 | str BASE, L->base
569 | decode_RB RB, INS
570 | decode_RA RA, INS
571 | add TMP1, BASE, RB, lsl #3
572 | subs TMP1, CARG2, TMP1
573 | beq >1
574 | str TMP0, [CARG2]
575 | lsr CARG3, TMP1, #3
576 | b ->BC_CAT_Z
577 |
578 |1:
579 | str TMP0, [BASE, RA, lsl #3]
580 | b ->cont_nop
581 |
582 |//-- Table indexing metamethods -----------------------------------------
583 |
584 |->vmeta_tgets1:
585 | movn CARG4, #~LJ_TSTR
586 | add CARG2, BASE, RB, lsl #3
587 | add CARG4, STR:RC, CARG4, lsl #47
588 | b >2
589 |
590 |->vmeta_tgets:
591 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
592 | str CARG2, GL->tmptv
593 | add CARG2, GL, #offsetof(global_State, tmptv)
594 |2:
595 | add CARG3, sp, TMPDofs
596 | str CARG4, TMPD
597 | b >1
598 |
599 |->vmeta_tgetb: // RB = table, RC = index
600 | add RC, RC, TISNUM
601 | add CARG2, BASE, RB, lsl #3
602 | add CARG3, sp, TMPDofs
603 | str RC, TMPD
604 | b >1
605 |
606 |->vmeta_tgetv: // RB = table, RC = key
607 | add CARG2, BASE, RB, lsl #3
608 | add CARG3, BASE, RC, lsl #3
609 |1:
610 | str BASE, L->base
611 | mov CARG1, L
612 | str PC, SAVE_PC
613 | bl extern lj_meta_tget // (lua_State *L, TValue *o, TValue *k)
614 | // Returns TValue * (finished) or NULL (metamethod).
615 | cbz CRET1, >3
616 | ldr TMP0, [CRET1]
617 | str TMP0, [BASE, RA, lsl #3]
618 | ins_next
619 |
620 |3: // Call __index metamethod.
621 | // BASE = base, L->top = new base, stack = cont/func/t/k
622 | sub TMP1, BASE, #FRAME_CONT
623 | ldr BASE, L->top
624 | mov NARGS8:RC, #16 // 2 args for func(t, k).
625 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
626 | str PC, [BASE, #-24] // [cont|PC]
627 | sub PC, BASE, TMP1
628 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
629 | b ->vm_call_dispatch_f
630 |
631 |->vmeta_tgetr:
632 | sxtw CARG2, TMP1w
633 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
634 | // Returns cTValue * or NULL.
635 | mov TMP0, TISNIL
636 | cbz CRET1, ->BC_TGETR_Z
637 | ldr TMP0, [CRET1]
638 | b ->BC_TGETR_Z
639 |
640 |//-----------------------------------------------------------------------
641 |
642 |->vmeta_tsets1:
643 | movn CARG4, #~LJ_TSTR
644 | add CARG2, BASE, RB, lsl #3
645 | add CARG4, STR:RC, CARG4, lsl #47
646 | b >2
647 |
648 |->vmeta_tsets:
649 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
650 | str CARG2, GL->tmptv
651 | add CARG2, GL, #offsetof(global_State, tmptv)
652 |2:
653 | add CARG3, sp, TMPDofs
654 | str CARG4, TMPD
655 | b >1
656 |
657 |->vmeta_tsetb: // RB = table, RC = index
658 | add RC, RC, TISNUM
659 | add CARG2, BASE, RB, lsl #3
660 | add CARG3, sp, TMPDofs
661 | str RC, TMPD
662 | b >1
663 |
664 |->vmeta_tsetv:
665 | add CARG2, BASE, RB, lsl #3
666 | add CARG3, BASE, RC, lsl #3
667 |1:
668 | str BASE, L->base
669 | mov CARG1, L
670 | str PC, SAVE_PC
671 | bl extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
672 | // Returns TValue * (finished) or NULL (metamethod).
673 | ldr TMP0, [BASE, RA, lsl #3]
674 | cbz CRET1, >3
675 | // NOBARRIER: lj_meta_tset ensures the table is not black.
676 | str TMP0, [CRET1]
677 | ins_next
678 |
679 |3: // Call __newindex metamethod.
680 | // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
681 | sub TMP1, BASE, #FRAME_CONT
682 | ldr BASE, L->top
683 | mov NARGS8:RC, #24 // 3 args for func(t, k, v).
684 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
685 | str TMP0, [BASE, #16] // Copy value to third argument.
686 | str PC, [BASE, #-24] // [cont|PC]
687 | sub PC, BASE, TMP1
688 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
689 | b ->vm_call_dispatch_f
690 |
691 |->vmeta_tsetr:
692 | sxtw CARG3, TMP1w
693 | str BASE, L->base
694 | str PC, SAVE_PC
695 | bl extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
696 | // Returns TValue *.
697 | b ->BC_TSETR_Z
698 |
699 |//-- Comparison metamethods ---------------------------------------------
700 |
701 |->vmeta_comp:
702 | add CARG2, BASE, RA, lsl #3
703 | sub PC, PC, #4
704 | add CARG3, BASE, RC, lsl #3
705 | str BASE, L->base
706 | mov CARG1, L
707 | str PC, SAVE_PC
708 | uxtb CARG4w, INSw
709 | bl extern lj_meta_comp // (lua_State *L, TValue *o1, *o2, int op)
710 | // Returns 0/1 or TValue * (metamethod).
711 |3:
712 | cmp CRET1, #1
713 | bhi ->vmeta_binop
714 |4:
715 | ldrh RBw, [PC, #2]
716 | add PC, PC, #4
717 | add RB, PC, RB, lsl #2
718 | sub RB, RB, #0x20000
719 | csel PC, PC, RB, lo
720 |->cont_nop:
721 | ins_next
722 |
723 |->cont_ra: // RA = resultptr
724 | ldr INSw, [PC, #-4]
725 | ldr TMP0, [RA]
726 | decode_RA TMP1, INS
727 | str TMP0, [BASE, TMP1, lsl #3]
728 | b ->cont_nop
729 |
730 |->cont_condt: // RA = resultptr
731 | ldr TMP0, [RA]
732 | mov_true TMP1
733 | cmp TMP1, TMP0 // Branch if result is true.
734 | b <4
735 |
736 |->cont_condf: // RA = resultptr
737 | ldr TMP0, [RA]
738 | mov_false TMP1
739 | cmp TMP0, TMP1 // Branch if result is false.
740 | b <4
741 |
742 |->vmeta_equal:
743 | // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
744 | and TAB:CARG3, CARG3, #LJ_GCVMASK
745 | sub PC, PC, #4
746 | str BASE, L->base
747 | mov CARG1, L
748 | str PC, SAVE_PC
749 | bl extern lj_meta_equal // (lua_State *L, GCobj *o1, *o2, int ne)
750 | // Returns 0/1 or TValue * (metamethod).
751 | b <3
752 |
753 |->vmeta_equal_cd:
754 |.if FFI
755 | sub PC, PC, #4
756 | str BASE, L->base
757 | mov CARG1, L
758 | mov CARG2, INS
759 | str PC, SAVE_PC
760 | bl extern lj_meta_equal_cd // (lua_State *L, BCIns op)
761 | // Returns 0/1 or TValue * (metamethod).
762 | b <3
763 |.endif
764 |
765 |->vmeta_istype:
766 | sub PC, PC, #4
767 | str BASE, L->base
768 | mov CARG1, L
769 | mov CARG2, RA
770 | mov CARG3, RC
771 | str PC, SAVE_PC
772 | bl extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
773 | b ->cont_nop
774 |
775 |//-- Arithmetic metamethods ---------------------------------------------
776 |
777 |->vmeta_arith_vn:
778 | add CARG3, BASE, RB, lsl #3
779 | add CARG4, KBASE, RC, lsl #3
780 | b >1
781 |
782 |->vmeta_arith_nv:
783 | add CARG4, BASE, RB, lsl #3
784 | add CARG3, KBASE, RC, lsl #3
785 | b >1
786 |
787 |->vmeta_unm:
788 | add CARG3, BASE, RC, lsl #3
789 | mov CARG4, CARG3
790 | b >1
791 |
792 |->vmeta_arith_vv:
793 | add CARG3, BASE, RB, lsl #3
794 | add CARG4, BASE, RC, lsl #3
795 |1:
796 | uxtb CARG5w, INSw
797 | add CARG2, BASE, RA, lsl #3
798 | str BASE, L->base
799 | mov CARG1, L
800 | str PC, SAVE_PC
801 | bl extern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
802 | // Returns NULL (finished) or TValue * (metamethod).
803 | cbz CRET1, ->cont_nop
804 |
805 | // Call metamethod for binary op.
806 |->vmeta_binop:
807 | // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
808 | sub TMP1, CRET1, BASE
809 | str PC, [CRET1, #-24] // [cont|PC]
810 | add PC, TMP1, #FRAME_CONT
811 | mov BASE, CRET1
812 | mov NARGS8:RC, #16 // 2 args for func(o1, o2).
813 | b ->vm_call_dispatch
814 |
815 |->vmeta_len:
816 | add CARG2, BASE, RC, lsl #3
817 #if LJ_52
818 | mov TAB:RC, TAB:CARG1 // Save table (ignored for other types).
819 #endif
820 | str BASE, L->base
821 | mov CARG1, L
822 | str PC, SAVE_PC
823 | bl extern lj_meta_len // (lua_State *L, TValue *o)
824 | // Returns NULL (retry) or TValue * (metamethod base).
825 #if LJ_52
826 | cbnz CRET1, ->vmeta_binop // Binop call for compatibility.
827 | mov TAB:CARG1, TAB:RC
828 | b ->BC_LEN_Z
829 #else
830 | b ->vmeta_binop // Binop call for compatibility.
831 #endif
832 |
833 |//-- Call metamethod ----------------------------------------------------
834 |
835 |->vmeta_call: // Resolve and call __call metamethod.
836 | // RB = old base, BASE = new base, RC = nargs*8
837 | mov CARG1, L
838 | str RB, L->base // This is the callers base!
839 | sub CARG2, BASE, #16
840 | str PC, SAVE_PC
841 | add CARG3, BASE, NARGS8:RC
842 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
843 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
844 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
845 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
846 | ins_call
847 |
848 |->vmeta_callt: // Resolve __call for BC_CALLT.
849 | // BASE = old base, RA = new base, RC = nargs*8
850 | mov CARG1, L
851 | str BASE, L->base
852 | sub CARG2, RA, #16
853 | str PC, SAVE_PC
854 | add CARG3, RA, NARGS8:RC
855 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
856 | ldr TMP1, [RA, FRAME_FUNC] // Guaranteed to be a function here.
857 | ldr PC, [BASE, FRAME_PC]
858 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
859 | and LFUNC:CARG3, TMP1, #LJ_GCVMASK
860 | b ->BC_CALLT2_Z
861 |
862 |//-- Argument coercion for 'for' statement ------------------------------
863 |
864 |->vmeta_for:
865 | mov CARG1, L
866 | str BASE, L->base
867 | mov CARG2, RA
868 | str PC, SAVE_PC
869 | bl extern lj_meta_for // (lua_State *L, TValue *base)
870 | ldr INSw, [PC, #-4]
871 |.if JIT
872 | uxtb TMP0, INS
873 |.endif
874 | decode_RA RA, INS
875 | decode_RD RC, INS
876 |.if JIT
877 | cmp TMP0, #BC_JFORI
878 | beq =>BC_JFORI
879 |.endif
880 | b =>BC_FORI
881 |
882 |//-----------------------------------------------------------------------
883 |//-- Fast functions -----------------------------------------------------
884 |//-----------------------------------------------------------------------
885 |
886 |.macro .ffunc, name
887 |->ff_ .. name:
888 |.endmacro
889 |
890 |.macro .ffunc_1, name
891 |->ff_ .. name:
892 | ldr CARG1, [BASE]
893 | cmp NARGS8:RC, #8
894 | blo ->fff_fallback
895 |.endmacro
896 |
897 |.macro .ffunc_2, name
898 |->ff_ .. name:
899 | ldp CARG1, CARG2, [BASE]
900 | cmp NARGS8:RC, #16
901 | blo ->fff_fallback
902 |.endmacro
903 |
904 |.macro .ffunc_n, name
905 | .ffunc name
906 | ldr CARG1, [BASE]
907 | cmp NARGS8:RC, #8
908 | ldr FARG1, [BASE]
909 | blo ->fff_fallback
910 | checknum CARG1, ->fff_fallback
911 |.endmacro
912 |
913 |.macro .ffunc_nn, name
914 | .ffunc name
915 | ldp CARG1, CARG2, [BASE]
916 | cmp NARGS8:RC, #16
917 | ldp FARG1, FARG2, [BASE]
918 | blo ->fff_fallback
919 | checknum CARG1, ->fff_fallback
920 | checknum CARG2, ->fff_fallback
921 |.endmacro
922 |
923 |// Inlined GC threshold check. Caveat: uses CARG1 and CARG2.
924 |.macro ffgccheck
925 | ldp CARG1, CARG2, GL->gc.total // Assumes threshold follows total.
926 | cmp CARG1, CARG2
927 | blt >1
928 | bl ->fff_gcstep
929 |1:
930 |.endmacro
931 |
932 |//-- Base library: checks -----------------------------------------------
933 |
934 |.ffunc_1 assert
935 | ldr PC, [BASE, FRAME_PC]
936 | mov_false TMP1
937 | cmp CARG1, TMP1
938 | bhs ->fff_fallback
939 | str CARG1, [BASE, #-16]
940 | sub RB, BASE, #8
941 | subs RA, NARGS8:RC, #8
942 | add RC, NARGS8:RC, #8 // Compute (nresults+1)*8.
943 | cbz RA, ->fff_res // Done if exactly 1 argument.
944 |1:
945 | ldr CARG1, [RB, #16]
946 | sub RA, RA, #8
947 | str CARG1, [RB], #8
948 | cbnz RA, <1
949 | b ->fff_res
950 |
951 |.ffunc_1 type
952 | mov TMP0, #~LJ_TISNUM
953 | asr ITYPE, CARG1, #47
954 | cmn ITYPE, #~LJ_TISNUM
955 | csinv TMP1, TMP0, ITYPE, lo
956 | add TMP1, TMP1, #offsetof(GCfuncC, upvalue)/8
957 | ldr CARG1, [CFUNC:CARG3, TMP1, lsl #3]
958 | b ->fff_restv
959 |
960 |//-- Base library: getters and setters ---------------------------------
961 |
962 |.ffunc_1 getmetatable
963 | asr ITYPE, CARG1, #47
964 | cmn ITYPE, #-LJ_TTAB
965 | ccmn ITYPE, #-LJ_TUDATA, #4, ne
966 | and TAB:CARG1, CARG1, #LJ_GCVMASK
967 | bne >6
968 |1: // Field metatable must be at same offset for GCtab and GCudata!
969 | ldr TAB:RB, TAB:CARG1->metatable
970 |2:
971 | mov CARG1, TISNIL
972 | ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable]
973 | cbz TAB:RB, ->fff_restv
974 | ldr TMP1w, TAB:RB->hmask
975 | ldr TMP2w, STR:RC->hash
976 | ldr NODE:CARG3, TAB:RB->node
977 | and TMP1w, TMP1w, TMP2w // idx = str->hash & tab->hmask
978 | add TMP1, TMP1, TMP1, lsl #1
979 | movn CARG4, #~LJ_TSTR
980 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
981 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
982 |3: // Rearranged logic, because we expect _not_ to find the key.
983 | ldp CARG1, TMP0, NODE:CARG3->val
984 | ldr NODE:CARG3, NODE:CARG3->next
985 | cmp TMP0, CARG4
986 | beq >5
987 | cbnz NODE:CARG3, <3
988 |4:
989 | mov CARG1, RB // Use metatable as default result.
990 | movk CARG1, #(LJ_TTAB>>1)&0xffff, lsl #48
991 | b ->fff_restv
992 |5:
993 | cmp TMP0, TISNIL
994 | bne ->fff_restv
995 | b <4
996 |
997 |6:
998 | movn TMP0, #~LJ_TISNUM
999 | cmp ITYPE, TMP0
1000 | csel ITYPE, ITYPE, TMP0, hs
1001 | sub TMP1, GL, ITYPE, lsl #3
1002 | ldr TAB:RB, [TMP1, #offsetof(global_State, gcroot[GCROOT_BASEMT])-8]
1003 | b <2
1004 |
1005 |.ffunc_2 setmetatable
1006 | // Fast path: no mt for table yet and not clearing the mt.
1007 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1008 | ldr TAB:TMP0, TAB:TMP1->metatable
1009 | asr ITYPE, CARG2, #47
1010 | ldrb TMP2w, TAB:TMP1->marked
1011 | cmn ITYPE, #-LJ_TTAB
1012 | and TAB:CARG2, CARG2, #LJ_GCVMASK
1013 | ccmp TAB:TMP0, #0, #0, eq
1014 | bne ->fff_fallback
1015 | str TAB:CARG2, TAB:TMP1->metatable
1016 | tbz TMP2w, #2, ->fff_restv // isblack(table)
1017 | barrierback TAB:TMP1, TMP2w, TMP0
1018 | b ->fff_restv
1019 |
1020 |.ffunc rawget
1021 | ldr CARG2, [BASE]
1022 | cmp NARGS8:RC, #16
1023 | blo ->fff_fallback
1024 | checktab CARG2, ->fff_fallback
1025 | mov CARG1, L
1026 | add CARG3, BASE, #8
1027 | bl extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1028 | // Returns cTValue *.
1029 | ldr CARG1, [CRET1]
1030 | b ->fff_restv
1031 |
1032 |//-- Base library: conversions ------------------------------------------
1033 |
1034 |.ffunc tonumber
1035 | // Only handles the number case inline (without a base argument).
1036 | ldr CARG1, [BASE]
1037 | cmp NARGS8:RC, #8
1038 | bne ->fff_fallback
1039 | checknumber CARG1, ->fff_fallback
1040 | b ->fff_restv
1041 |
1042 |.ffunc_1 tostring
1043 | // Only handles the string or number case inline.
1044 | asr ITYPE, CARG1, #47
1045 | cmn ITYPE, #-LJ_TSTR
1046 | // A __tostring method in the string base metatable is ignored.
1047 | beq ->fff_restv
1048 | // Handle numbers inline, unless a number base metatable is present.
1049 | ldr TMP1, GL->gcroot[GCROOT_BASEMT_NUM]
1050 | str BASE, L->base
1051 | cmn ITYPE, #-LJ_TISNUM
1052 | ccmp TMP1, #0, #0, ls
1053 | str PC, SAVE_PC // Redundant (but a defined value).
1054 | bne ->fff_fallback
1055 | ffgccheck
1056 | mov CARG1, L
1057 | mov CARG2, BASE
1058 | bl extern lj_strfmt_number // (lua_State *L, cTValue *o)
1059 | // Returns GCstr *.
1060 | movn TMP1, #~LJ_TSTR
1061 | ldr BASE, L->base
1062 | add CARG1, CARG1, TMP1, lsl #47
1063 | b ->fff_restv
1064 |
1065 |//-- Base library: iterators -------------------------------------------
1066 |
1067 |.ffunc_1 next
1068 | checktp CARG2, CARG1, LJ_TTAB, ->fff_fallback
1069 | str TISNIL, [BASE, NARGS8:RC] // Set missing 2nd arg to nil.
1070 | ldr PC, [BASE, FRAME_PC]
1071 | stp BASE, BASE, L->base // Add frame since C call can throw.
1072 | mov CARG1, L
1073 | add CARG3, BASE, #8
1074 | str PC, SAVE_PC
1075 | bl extern lj_tab_next // (lua_State *L, GCtab *t, TValue *key)
1076 | // Returns 0 at end of traversal.
1077 | str TISNIL, [BASE, #-16]
1078 | cbz CRET1, ->fff_res1 // End of traversal: return nil.
1079 | ldp CARG1, CARG2, [BASE, #8] // Copy key and value to results.
1080 | mov RC, #(2+1)*8
1081 | stp CARG1, CARG2, [BASE, #-16]
1082 | b ->fff_res
1083 |
1084 |.ffunc_1 pairs
1085 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1086 #if LJ_52
1087 | ldr TAB:CARG2, TAB:TMP1->metatable
1088 #endif
1089 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1090 | ldr PC, [BASE, FRAME_PC]
1091 #if LJ_52
1092 | cbnz TAB:CARG2, ->fff_fallback
1093 #endif
1094 | mov RC, #(3+1)*8
1095 | stp CARG1, TISNIL, [BASE, #-8]
1096 | str CFUNC:CARG4, [BASE, #-16]
1097 | b ->fff_res
1098 |
1099 |.ffunc_2 ipairs_aux
1100 | checktab CARG1, ->fff_fallback
1101 | checkint CARG2, ->fff_fallback
1102 | ldr TMP1w, TAB:CARG1->asize
1103 | ldr CARG3, TAB:CARG1->array
1104 | ldr TMP0w, TAB:CARG1->hmask
1105 | add CARG2w, CARG2w, #1
1106 | cmp CARG2w, TMP1w
1107 | ldr PC, [BASE, FRAME_PC]
1108 | add TMP2, CARG2, TISNUM
1109 | mov RC, #(0+1)*8
1110 | str TMP2, [BASE, #-16]
1111 | bhs >2 // Not in array part?
1112 | ldr TMP0, [CARG3, CARG2, lsl #3]
1113 |1:
1114 | mov TMP1, #(2+1)*8
1115 | cmp TMP0, TISNIL
1116 | str TMP0, [BASE, #-8]
1117 | csel RC, RC, TMP1, eq
1118 | b ->fff_res
1119 |2: // Check for empty hash part first. Otherwise call C function.
1120 | cbz TMP0w, ->fff_res
1121 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
1122 | // Returns cTValue * or NULL.
1123 | cbz CRET1, ->fff_res
1124 | ldr TMP0, [CRET1]
1125 | b <1
1126 |
1127 |.ffunc_1 ipairs
1128 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1129 #if LJ_52
1130 | ldr TAB:CARG2, TAB:TMP1->metatable
1131 #endif
1132 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1133 | ldr PC, [BASE, FRAME_PC]
1134 #if LJ_52
1135 | cbnz TAB:CARG2, ->fff_fallback
1136 #endif
1137 | mov RC, #(3+1)*8
1138 | stp CARG1, TISNUM, [BASE, #-8]
1139 | str CFUNC:CARG4, [BASE, #-16]
1140 | b ->fff_res
1141 |
1142 |//-- Base library: catch errors ----------------------------------------
1143 |
1144 |.ffunc pcall
1145 | ldrb TMP0w, GL->hookmask
1146 | subs NARGS8:RC, NARGS8:RC, #8
1147 | blo ->fff_fallback
1148 | mov RB, BASE
1149 | add BASE, BASE, #16
1150 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1151 | add PC, TMP0, #16+FRAME_PCALL
1152 | beq ->vm_call_dispatch
1153 |1:
1154 | add TMP2, BASE, NARGS8:RC
1155 |2:
1156 | ldr TMP0, [TMP2, #-16]
1157 | str TMP0, [TMP2, #-8]!
1158 | cmp TMP2, BASE
1159 | bne <2
1160 | b ->vm_call_dispatch
1161 |
1162 |.ffunc xpcall
1163 | ldp CARG1, CARG2, [BASE]
1164 | ldrb TMP0w, GL->hookmask
1165 | subs NARGS8:RC, NARGS8:RC, #16
1166 | blo ->fff_fallback
1167 | mov RB, BASE
1168 | add BASE, BASE, #24
1169 | asr ITYPE, CARG2, #47
1170 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1171 | cmn ITYPE, #-LJ_TFUNC
1172 | add PC, TMP0, #24+FRAME_PCALL
1173 | bne ->fff_fallback // Traceback must be a function.
1174 | stp CARG2, CARG1, [RB] // Swap function and traceback.
1175 | cbz NARGS8:RC, ->vm_call_dispatch
1176 | b <1
1177 |
1178 |//-- Coroutine library --------------------------------------------------
1179 |
1180 |.macro coroutine_resume_wrap, resume
1181 |.if resume
1182 |.ffunc_1 coroutine_resume
1183 | checktp CARG1, LJ_TTHREAD, ->fff_fallback
1184 |.else
1185 |.ffunc coroutine_wrap_aux
1186 | ldr L:CARG1, CFUNC:CARG3->upvalue[0].gcr
1187 | and L:CARG1, CARG1, #LJ_GCVMASK
1188 |.endif
1189 | ldr PC, [BASE, FRAME_PC]
1190 | str BASE, L->base
1191 | ldp RB, CARG2, L:CARG1->base
1192 | ldrb TMP1w, L:CARG1->status
1193 | add TMP0, CARG2, TMP1
1194 | str PC, SAVE_PC
1195 | cmp TMP0, RB
1196 | beq ->fff_fallback
1197 | cmp TMP1, #LUA_YIELD
1198 | add TMP0, CARG2, #8
1199 | csel CARG2, CARG2, TMP0, hs
1200 | ldr CARG4, L:CARG1->maxstack
1201 | add CARG3, CARG2, NARGS8:RC
1202 | ldr RB, L:CARG1->cframe
1203 | ccmp CARG3, CARG4, #2, ls
1204 | ccmp RB, #0, #2, ls
1205 | bhi ->fff_fallback
1206 |.if resume
1207 | sub CARG3, CARG3, #8 // Keep resumed thread in stack for GC.
1208 | add BASE, BASE, #8
1209 | sub NARGS8:RC, NARGS8:RC, #8
1210 |.endif
1211 | str CARG3, L:CARG1->top
1212 | str BASE, L->top
1213 | cbz NARGS8:RC, >3
1214 |2: // Move args to coroutine.
1215 | ldr TMP0, [BASE, RB]
1216 | cmp RB, NARGS8:RC
1217 | str TMP0, [CARG2, RB]
1218 | add RB, RB, #8
1219 | bne <2
1220 |3:
1221 | mov CARG3, #0
1222 | mov L:RA, L:CARG1
1223 | mov CARG4, #0
1224 | bl ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1225 | // Returns thread status.
1226 |4:
1227 | ldp CARG3, CARG4, L:RA->base
1228 | cmp CRET1, #LUA_YIELD
1229 | ldr BASE, L->base
1230 | str L, GL->cur_L
1231 | st_vmstate ST_INTERP
1232 | bhi >8
1233 | sub RC, CARG4, CARG3
1234 | ldr CARG1, L->maxstack
1235 | add CARG2, BASE, RC
1236 | cbz RC, >6 // No results?
1237 | cmp CARG2, CARG1
1238 | mov RB, #0
1239 | bhi >9 // Need to grow stack?
1240 |
1241 | sub CARG4, RC, #8
1242 | str CARG3, L:RA->top // Clear coroutine stack.
1243 |5: // Move results from coroutine.
1244 | ldr TMP0, [CARG3, RB]
1245 | cmp RB, CARG4
1246 | str TMP0, [BASE, RB]
1247 | add RB, RB, #8
1248 | bne <5
1249 |6:
1250 |.if resume
1251 | mov_true TMP1
1252 | add RC, RC, #16
1253 |7:
1254 | str TMP1, [BASE, #-8] // Prepend true/false to results.
1255 | sub RA, BASE, #8
1256 |.else
1257 | mov RA, BASE
1258 | add RC, RC, #8
1259 |.endif
1260 | ands CARG1, PC, #FRAME_TYPE
1261 | str PC, SAVE_PC
1262 | str RCw, SAVE_MULTRES
1263 | beq ->BC_RET_Z
1264 | b ->vm_return
1265 |
1266 |8: // Coroutine returned with error (at co->top-1).
1267 |.if resume
1268 | ldr TMP0, [CARG4, #-8]!
1269 | mov_false TMP1
1270 | mov RC, #(2+1)*8
1271 | str CARG4, L:RA->top // Remove error from coroutine stack.
1272 | str TMP0, [BASE] // Copy error message.
1273 | b <7
1274 |.else
1275 | mov CARG1, L
1276 | mov CARG2, L:RA
1277 | bl extern lj_ffh_coroutine_wrap_err // (lua_State *L, lua_State *co)
1278 | // Never returns.
1279 |.endif
1280 |
1281 |9: // Handle stack expansion on return from yield.
1282 | mov CARG1, L
1283 | lsr CARG2, RC, #3
1284 | bl extern lj_state_growstack // (lua_State *L, int n)
1285 | mov CRET1, #0
1286 | b <4
1287 |.endmacro
1288 |
1289 | coroutine_resume_wrap 1 // coroutine.resume
1290 | coroutine_resume_wrap 0 // coroutine.wrap
1291 |
1292 |.ffunc coroutine_yield
1293 | ldr TMP0, L->cframe
1294 | add TMP1, BASE, NARGS8:RC
1295 | mov CRET1, #LUA_YIELD
1296 | stp BASE, TMP1, L->base
1297 | tbz TMP0, #0, ->fff_fallback
1298 | str xzr, L->cframe
1299 | strb CRET1w, L->status
1300 | b ->vm_leave_unw
1301 |
1302 |//-- Math library -------------------------------------------------------
1303 |
1304 |.macro math_round, func, round
1305 | .ffunc math_ .. func
1306 | ldr CARG1, [BASE]
1307 | cmp NARGS8:RC, #8
1308 | ldr d0, [BASE]
1309 | blo ->fff_fallback
1310 | cmp TISNUMhi, CARG1, lsr #32
1311 | beq ->fff_restv
1312 | blo ->fff_fallback
1313 | round d0, d0
1314 | b ->fff_resn
1315 |.endmacro
1316 |
1317 | math_round floor, frintm
1318 | math_round ceil, frintp
1319 |
1320 |.ffunc_1 math_abs
1321 | checknumber CARG1, ->fff_fallback
1322 | and CARG1, CARG1, #U64x(7fffffff,ffffffff)
1323 | bne ->fff_restv
1324 | eor CARG2w, CARG1w, CARG1w, asr #31
1325 | movz CARG3, #0x41e0, lsl #48 // 2^31.
1326 | subs CARG1w, CARG2w, CARG1w, asr #31
1327 | add CARG1, CARG1, TISNUM
1328 | csel CARG1, CARG1, CARG3, pl
1329 | // Fallthrough.
1330 |
1331 |->fff_restv:
1332 | // CARG1 = TValue result.
1333 | ldr PC, [BASE, FRAME_PC]
1334 | str CARG1, [BASE, #-16]
1335 |->fff_res1:
1336 | // PC = return.
1337 | mov RC, #(1+1)*8
1338 |->fff_res:
1339 | // RC = (nresults+1)*8, PC = return.
1340 | ands CARG1, PC, #FRAME_TYPE
1341 | str RCw, SAVE_MULTRES
1342 | sub RA, BASE, #16
1343 | bne ->vm_return
1344 | ldr INSw, [PC, #-4]
1345 | decode_RB RB, INS
1346 |5:
1347 | cmp RC, RB, lsl #3 // More results expected?
1348 | blo >6
1349 | decode_RA TMP1, INS
1350 | // Adjust BASE. KBASE is assumed to be set for the calling frame.
1351 | sub BASE, RA, TMP1, lsl #3
1352 | ins_next
1353 |
1354 |6: // Fill up results with nil.
1355 | add TMP1, RA, RC
1356 | add RC, RC, #8
1357 | str TISNIL, [TMP1, #-8]
1358 | b <5
1359 |
1360 |.macro math_extern, func
1361 | .ffunc_n math_ .. func
1362 | bl extern func
1363 | b ->fff_resn
1364 |.endmacro
1365 |
1366 |.macro math_extern2, func
1367 | .ffunc_nn math_ .. func
1368 | bl extern func
1369 | b ->fff_resn
1370 |.endmacro
1371 |
1372 |.ffunc_n math_sqrt
1373 | fsqrt d0, d0
1374 |->fff_resn:
1375 | ldr PC, [BASE, FRAME_PC]
1376 | str d0, [BASE, #-16]
1377 | b ->fff_res1
1378 |
1379 |.ffunc math_log
1380 | ldr CARG1, [BASE]
1381 | cmp NARGS8:RC, #8
1382 | ldr FARG1, [BASE]
1383 | bne ->fff_fallback // Need exactly 1 argument.
1384 | checknum CARG1, ->fff_fallback
1385 | bl extern log
1386 | b ->fff_resn
1387 |
1388 | math_extern log10
1389 | math_extern exp
1390 | math_extern sin
1391 | math_extern cos
1392 | math_extern tan
1393 | math_extern asin
1394 | math_extern acos
1395 | math_extern atan
1396 | math_extern sinh
1397 | math_extern cosh
1398 | math_extern tanh
1399 | math_extern2 pow
1400 | math_extern2 atan2
1401 | math_extern2 fmod
1402 |
1403 |.ffunc_2 math_ldexp
1404 | ldr FARG1, [BASE]
1405 | checknum CARG1, ->fff_fallback
1406 | checkint CARG2, ->fff_fallback
1407 | sxtw CARG1, CARG2w
1408 | bl extern ldexp // (double x, int exp)
1409 | b ->fff_resn
1410 |
1411 |.ffunc_n math_frexp
1412 | add CARG1, sp, TMPDofs
1413 | bl extern frexp
1414 | ldr CARG2w, TMPD
1415 | ldr PC, [BASE, FRAME_PC]
1416 | str d0, [BASE, #-16]
1417 | mov RC, #(2+1)*8
1418 | add CARG2, CARG2, TISNUM
1419 | str CARG2, [BASE, #-8]
1420 | b ->fff_res
1421 |
1422 |.ffunc_n math_modf
1423 | sub CARG1, BASE, #16
1424 | ldr PC, [BASE, FRAME_PC]
1425 | bl extern modf
1426 | mov RC, #(2+1)*8
1427 | str d0, [BASE, #-8]
1428 | b ->fff_res
1429 |
1430 |.macro math_minmax, name, cond, fcond
1431 | .ffunc_1 name
1432 | add RB, BASE, RC
1433 | add RA, BASE, #8
1434 | checkint CARG1, >4
1435 |1: // Handle integers.
1436 | ldr CARG2, [RA]
1437 | cmp RA, RB
1438 | bhs ->fff_restv
1439 | checkint CARG2, >3
1440 | cmp CARG1w, CARG2w
1441 | add RA, RA, #8
1442 | csel CARG1, CARG2, CARG1, cond
1443 | b <1
1444 |3: // Convert intermediate result to number and continue below.
1445 | scvtf d0, CARG1w
1446 | blo ->fff_fallback
1447 | ldr d1, [RA]
1448 | b >6
1449 |
1450 |4:
1451 | ldr d0, [BASE]
1452 | blo ->fff_fallback
1453 |5: // Handle numbers.
1454 | ldr CARG2, [RA]
1455 | ldr d1, [RA]
1456 | cmp RA, RB
1457 | bhs ->fff_resn
1458 | checknum CARG2, >7
1459 |6:
1460 | fcmp d0, d1
1461 | add RA, RA, #8
1462 | fcsel d0, d1, d0, fcond
1463 | b <5
1464 |7: // Convert integer to number and continue above.
1465 | scvtf d1, CARG2w
1466 | blo ->fff_fallback
1467 | b <6
1468 |.endmacro
1469 |
1470 | math_minmax math_min, gt, hi
1471 | math_minmax math_max, lt, lo
1472 |
1473 |//-- String library -----------------------------------------------------
1474 |
1475 |.ffunc string_byte // Only handle the 1-arg case here.
1476 | ldp PC, CARG1, [BASE, FRAME_PC]
1477 | cmp NARGS8:RC, #8
1478 | asr ITYPE, CARG1, #47
1479 | ccmn ITYPE, #-LJ_TSTR, #0, eq
1480 | and STR:CARG1, CARG1, #LJ_GCVMASK
1481 | bne ->fff_fallback
1482 | ldrb TMP0w, STR:CARG1[1] // Access is always ok (NUL at end).
1483 | ldr CARG3w, STR:CARG1->len
1484 | add TMP0, TMP0, TISNUM
1485 | str TMP0, [BASE, #-16]
1486 | mov RC, #(0+1)*8
1487 | cbz CARG3, ->fff_res
1488 | b ->fff_res1
1489 |
1490 |.ffunc string_char // Only handle the 1-arg case here.
1491 | ffgccheck
1492 | ldp PC, CARG1, [BASE, FRAME_PC]
1493 | cmp CARG1w, #255
1494 | ccmp NARGS8:RC, #8, #0, ls // Need exactly 1 argument.
1495 | bne ->fff_fallback
1496 | checkint CARG1, ->fff_fallback
1497 | mov CARG3, #1
1498 | mov CARG2, BASE // Points to stack. Little-endian.
1499 |->fff_newstr:
1500 | // CARG2 = str, CARG3 = len.
1501 | str BASE, L->base
1502 | mov CARG1, L
1503 | str PC, SAVE_PC
1504 | bl extern lj_str_new // (lua_State *L, char *str, size_t l)
1505 |->fff_resstr:
1506 | // Returns GCstr *.
1507 | ldr BASE, L->base
1508 | movn TMP1, #~LJ_TSTR
1509 | add CARG1, CARG1, TMP1, lsl #47
1510 | b ->fff_restv
1511 |
1512 |.ffunc string_sub
1513 | ffgccheck
1514 | ldr CARG1, [BASE]
1515 | ldr CARG3, [BASE, #16]
1516 | cmp NARGS8:RC, #16
1517 | movn RB, #0
1518 | beq >1
1519 | blo ->fff_fallback
1520 | checkint CARG3, ->fff_fallback
1521 | sxtw RB, CARG3w
1522 |1:
1523 | ldr CARG2, [BASE, #8]
1524 | checkstr CARG1, ->fff_fallback
1525 | ldr TMP1w, STR:CARG1->len
1526 | checkint CARG2, ->fff_fallback
1527 | sxtw CARG2, CARG2w
1528 | // CARG1 = str, TMP1 = str->len, CARG2 = start, RB = end
1529 | add TMP2, RB, TMP1
1530 | cmp RB, #0
1531 | add TMP0, CARG2, TMP1
1532 | csinc RB, RB, TMP2, ge // if (end < 0) end += len+1
1533 | cmp CARG2, #0
1534 | csinc CARG2, CARG2, TMP0, ge // if (start < 0) start += len+1
1535 | cmp RB, #0
1536 | csel RB, RB, xzr, ge // if (end < 0) end = 0
1537 | cmp CARG2, #1
1538 | csinc CARG2, CARG2, xzr, ge // if (start < 1) start = 1
1539 | cmp RB, TMP1
1540 | csel RB, RB, TMP1, le // if (end > len) end = len
1541 | add CARG1, STR:CARG1, #sizeof(GCstr)-1
1542 | subs CARG3, RB, CARG2 // len = end - start
1543 | add CARG2, CARG1, CARG2
1544 | add CARG3, CARG3, #1 // len += 1
1545 | bge ->fff_newstr
1546 | add STR:CARG1, GL, #offsetof(global_State, strempty)
1547 | movn TMP1, #~LJ_TSTR
1548 | add CARG1, CARG1, TMP1, lsl #47
1549 | b ->fff_restv
1550 |
1551 |.macro ffstring_op, name
1552 | .ffunc string_ .. name
1553 | ffgccheck
1554 | ldr CARG2, [BASE]
1555 | cmp NARGS8:RC, #8
1556 | asr ITYPE, CARG2, #47
1557 | ccmn ITYPE, #-LJ_TSTR, #0, hs
1558 | and STR:CARG2, CARG2, #LJ_GCVMASK
1559 | bne ->fff_fallback
1560 | ldr TMP0, GL->tmpbuf.b
1561 | add SBUF:CARG1, GL, #offsetof(global_State, tmpbuf)
1562 | str BASE, L->base
1563 | str PC, SAVE_PC
1564 | str L, GL->tmpbuf.L
1565 | str TMP0, GL->tmpbuf.p
1566 | bl extern lj_buf_putstr_ .. name
1567 | bl extern lj_buf_tostr
1568 | b ->fff_resstr
1569 |.endmacro
1570 |
1571 |ffstring_op reverse
1572 |ffstring_op lower
1573 |ffstring_op upper
1574 |
1575 |//-- Bit library --------------------------------------------------------
1576 |
1577 |// FP number to bit conversion for soft-float. Clobbers CARG1-CARG3
1578 |->vm_tobit_fb:
1579 | bls ->fff_fallback
1580 | add CARG2, CARG1, CARG1
1581 | mov CARG3, #1076
1582 | sub CARG3, CARG3, CARG2, lsr #53
1583 | cmp CARG3, #53
1584 | bhi >1
1585 | and CARG2, CARG2, #U64x(001fffff,ffffffff)
1586 | orr CARG2, CARG2, #U64x(00200000,00000000)
1587 | cmp CARG1, #0
1588 | lsr CARG2, CARG2, CARG3
1589 | cneg CARG1w, CARG2w, mi
1590 | br lr
1591 |1:
1592 | mov CARG1w, #0
1593 | br lr
1594 |
1595 |.macro .ffunc_bit, name
1596 | .ffunc_1 bit_..name
1597 | adr lr, >1
1598 | checkint CARG1, ->vm_tobit_fb
1599 |1:
1600 |.endmacro
1601 |
1602 |.macro .ffunc_bit_op, name, ins
1603 | .ffunc_bit name
1604 | mov RA, #8
1605 | mov TMP0w, CARG1w
1606 | adr lr, >2
1607 |1:
1608 | ldr CARG1, [BASE, RA]
1609 | cmp RA, NARGS8:RC
1610 | add RA, RA, #8
1611 | bge >9
1612 | checkint CARG1, ->vm_tobit_fb
1613 |2:
1614 | ins TMP0w, TMP0w, CARG1w
1615 | b <1
1616 |.endmacro
1617 |
1618 |.ffunc_bit_op band, and
1619 |.ffunc_bit_op bor, orr
1620 |.ffunc_bit_op bxor, eor
1621 |
1622 |.ffunc_bit tobit
1623 | mov TMP0w, CARG1w
1624 |9: // Label reused by .ffunc_bit_op users.
1625 | add CARG1, TMP0, TISNUM
1626 | b ->fff_restv
1627 |
1628 |.ffunc_bit bswap
1629 | rev TMP0w, CARG1w
1630 | add CARG1, TMP0, TISNUM
1631 | b ->fff_restv
1632 |
1633 |.ffunc_bit bnot
1634 | mvn TMP0w, CARG1w
1635 | add CARG1, TMP0, TISNUM
1636 | b ->fff_restv
1637 |
1638 |.macro .ffunc_bit_sh, name, ins, shmod
1639 | .ffunc bit_..name
1640 | ldp TMP0, CARG1, [BASE]
1641 | cmp NARGS8:RC, #16
1642 | blo ->fff_fallback
1643 | adr lr, >1
1644 | checkint CARG1, ->vm_tobit_fb
1645 |1:
1646 |.if shmod == 0
1647 | mov TMP1, CARG1
1648 |.else
1649 | neg TMP1, CARG1
1650 |.endif
1651 | mov CARG1, TMP0
1652 | adr lr, >2
1653 | checkint CARG1, ->vm_tobit_fb
1654 |2:
1655 | ins TMP0w, CARG1w, TMP1w
1656 | add CARG1, TMP0, TISNUM
1657 | b ->fff_restv
1658 |.endmacro
1659 |
1660 |.ffunc_bit_sh lshift, lsl, 0
1661 |.ffunc_bit_sh rshift, lsr, 0
1662 |.ffunc_bit_sh arshift, asr, 0
1663 |.ffunc_bit_sh rol, ror, 1
1664 |.ffunc_bit_sh ror, ror, 0
1665 |
1666 |//-----------------------------------------------------------------------
1667 |
1668 |->fff_fallback: // Call fast function fallback handler.
1669 | // BASE = new base, RC = nargs*8
1670 | ldp CFUNC:CARG3, PC, [BASE, FRAME_FUNC] // Fallback may overwrite PC.
1671 | ldr TMP2, L->maxstack
1672 | add TMP1, BASE, NARGS8:RC
1673 | stp BASE, TMP1, L->base
1674 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1675 | add TMP1, TMP1, #8*LUA_MINSTACK
1676 | ldr CARG3, CFUNC:CARG3->f
1677 | str PC, SAVE_PC // Redundant (but a defined value).
1678 | cmp TMP1, TMP2
1679 | mov CARG1, L
1680 | bhi >5 // Need to grow stack.
1681 | blr CARG3 // (lua_State *L)
1682 | // Either throws an error, or recovers and returns -1, 0 or nresults+1.
1683 | ldr BASE, L->base
1684 | cmp CRET1w, #0
1685 | lsl RC, CRET1, #3
1686 | sub RA, BASE, #16
1687 | bgt ->fff_res // Returned nresults+1?
1688 |1: // Returned 0 or -1: retry fast path.
1689 | ldr CARG1, L->top
1690 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1691 | sub NARGS8:RC, CARG1, BASE
1692 | bne ->vm_call_tail // Returned -1?
1693 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1694 | ins_callt // Returned 0: retry fast path.
1695 |
1696 |// Reconstruct previous base for vmeta_call during tailcall.
1697 |->vm_call_tail:
1698 | ands TMP0, PC, #FRAME_TYPE
1699 | and TMP1, PC, #~FRAME_TYPEP
1700 | bne >3
1701 | ldrb RAw, [PC, #-3]
1702 | lsl RA, RA, #3
1703 | add TMP1, RA, #16
1704 |3:
1705 | sub RB, BASE, TMP1
1706 | b ->vm_call_dispatch // Resolve again for tailcall.
1707 |
1708 |5: // Grow stack for fallback handler.
1709 | mov CARG2, #LUA_MINSTACK
1710 | bl extern lj_state_growstack // (lua_State *L, int n)
1711 | ldr BASE, L->base
1712 | cmp CARG1, CARG1 // Set zero-flag to force retry.
1713 | b <1
1714 |
1715 |->fff_gcstep: // Call GC step function.
1716 | // BASE = new base, RC = nargs*8
1717 | add CARG2, BASE, NARGS8:RC // Calculate L->top.
1718 | mov RA, lr
1719 | stp BASE, CARG2, L->base
1720 | str PC, SAVE_PC // Redundant (but a defined value).
1721 | mov CARG1, L
1722 | bl extern lj_gc_step // (lua_State *L)
1723 | ldp BASE, CARG2, L->base
1724 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1725 | mov lr, RA // Help return address predictor.
1726 | sub NARGS8:RC, CARG2, BASE // Calculate nargs*8.
1727 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1728 | ret
1729 |
1730 |//-----------------------------------------------------------------------
1731 |//-- Special dispatch targets -------------------------------------------
1732 |//-----------------------------------------------------------------------
1733 |
1734 |->vm_record: // Dispatch target for recording phase.
1735 | NYI
1736 |
1737 |->vm_rethook: // Dispatch target for return hooks.
1738 | ldrb TMP2w, GL->hookmask
1739 | tbz TMP2w, #HOOK_ACTIVE_SHIFT, >1 // Hook already active?
1740 |5: // Re-dispatch to static ins.
1741 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1742 | br TMP0
1743 |
1744 |->vm_inshook: // Dispatch target for instr/line hooks.
1745 | ldrb TMP2w, GL->hookmask
1746 | ldr TMP3w, GL->hookcount
1747 | tbnz TMP2w, #HOOK_ACTIVE_SHIFT, <5 // Hook already active?
1748 | tst TMP2w, #LUA_MASKLINE|LUA_MASKCOUNT
1749 | beq <5
1750 | sub TMP3w, TMP3w, #1
1751 | str TMP3w, GL->hookcount
1752 | cbz TMP3w, >1
1753 | tbz TMP2w, #LUA_HOOKLINE, <5
1754 |1:
1755 | mov CARG1, L
1756 | str BASE, L->base
1757 | mov CARG2, PC
1758 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
1759 | bl extern lj_dispatch_ins // (lua_State *L, const BCIns *pc)
1760 |3:
1761 | ldr BASE, L->base
1762 |4: // Re-dispatch to static ins.
1763 | ldr INSw, [PC, #-4]
1764 | add TMP1, GL, INS, uxtb #3
1765 | decode_RA RA, INS
1766 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1767 | decode_RD RC, INS
1768 | br TMP0
1769 |
1770 |->cont_hook: // Continue from hook yield.
1771 | ldr CARG1, [CARG4, #-40]
1772 | add PC, PC, #4
1773 | str CARG1w, SAVE_MULTRES // Restore MULTRES for *M ins.
1774 | b <4
1775 |
1776 |->vm_hotloop: // Hot loop counter underflow.
1777 | NYI
1778 |
1779 |->vm_callhook: // Dispatch target for call hooks.
1780 | mov CARG2, PC
1781 |.if JIT
1782 | b >1
1783 |.endif
1784 |
1785 |->vm_hotcall: // Hot call counter underflow.
1786 |.if JIT
1787 | orr CARG2, PC, #1
1788 |1:
1789 |.endif
1790 | add TMP1, BASE, NARGS8:RC
1791 | str PC, SAVE_PC
1792 | mov CARG1, L
1793 | sub RA, RA, BASE
1794 | stp BASE, TMP1, L->base
1795 | bl extern lj_dispatch_call // (lua_State *L, const BCIns *pc)
1796 | // Returns ASMFunction.
1797 | ldp BASE, TMP1, L->base
1798 | str xzr, SAVE_PC // Invalidate for subsequent line hook.
1799 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
1800 | add RA, BASE, RA
1801 | sub NARGS8:RC, TMP1, BASE
1802 | ldr INSw, [PC, #-4]
1803 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1804 | br CRET1
1805 |
1806 |->cont_stitch: // Trace stitching.
1807 | NYI
1808 |
1809 |->vm_profhook: // Dispatch target for profiler hook.
1810 #if LJ_HASPROFILE
1811 | mov CARG1, L
1812 | str BASE, L->base
1813 | mov CARG2, PC
1814 | bl extern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
1815 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
1816 | ldr BASE, L->base
1817 | sub PC, PC, #4
1818 | b ->cont_nop
1819 #endif
1820 |
1821 |//-----------------------------------------------------------------------
1822 |//-- Trace exit handler -------------------------------------------------
1823 |//-----------------------------------------------------------------------
1824 |
1825 |->vm_exit_handler:
1826 | NYI
1827 |->vm_exit_interp:
1828 | NYI
1829 |
1830 |//-----------------------------------------------------------------------
1831 |//-- Math helper functions ----------------------------------------------
1832 |//-----------------------------------------------------------------------
1833 |
1834 | // int lj_vm_modi(int dividend, int divisor);
1835 |->vm_modi:
1836 | eor CARG4w, CARG1w, CARG2w
1837 | cmp CARG4w, #0
1838 | eor CARG3w, CARG1w, CARG1w, asr #31
1839 | eor CARG4w, CARG2w, CARG2w, asr #31
1840 | sub CARG3w, CARG3w, CARG1w, asr #31
1841 | sub CARG4w, CARG4w, CARG2w, asr #31
1842 | udiv CARG1w, CARG3w, CARG4w
1843 | msub CARG1w, CARG1w, CARG4w, CARG3w
1844 | ccmp CARG1w, #0, #4, mi
1845 | sub CARG3w, CARG1w, CARG4w
1846 | csel CARG1w, CARG1w, CARG3w, eq
1847 | eor CARG3w, CARG1w, CARG2w
1848 | cmp CARG3w, #0
1849 | cneg CARG1w, CARG1w, mi
1850 | ret
1851 |
1852 |//-----------------------------------------------------------------------
1853 |//-- Miscellaneous functions --------------------------------------------
1854 |//-----------------------------------------------------------------------
1855 |
1856 |//-----------------------------------------------------------------------
1857 |//-- FFI helper functions -----------------------------------------------
1858 |//-----------------------------------------------------------------------
1859 |
1860 |// Handler for callback functions.
1861 |// Saveregs already performed. Callback slot number in [sp], g in r12.
1862 |->vm_ffi_callback:
1863 |.if FFI
1864 |.type CTSTATE, CTState, PC
1865 | saveregs
1866 | ldr CTSTATE, GL:x10->ctype_state
1867 | mov GL, x10
1868 | add x10, sp, # CFRAME_SPACE
1869 | str w9, CTSTATE->cb.slot
1870 | stp x0, x1, CTSTATE->cb.gpr[0]
1871 | stp d0, d1, CTSTATE->cb.fpr[0]
1872 | stp x2, x3, CTSTATE->cb.gpr[2]
1873 | stp d2, d3, CTSTATE->cb.fpr[2]
1874 | stp x4, x5, CTSTATE->cb.gpr[4]
1875 | stp d4, d5, CTSTATE->cb.fpr[4]
1876 | stp x6, x7, CTSTATE->cb.gpr[6]
1877 | stp d6, d7, CTSTATE->cb.fpr[6]
1878 | str x10, CTSTATE->cb.stack
1879 | mov CARG1, CTSTATE
1880 | str CTSTATE, SAVE_PC // Any value outside of bytecode is ok.
1881 | mov CARG2, sp
1882 | bl extern lj_ccallback_enter // (CTState *cts, void *cf)
1883 | // Returns lua_State *.
1884 | ldp BASE, RC, L:CRET1->base
1885 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
1886 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
1887 | movn TISNIL, #0
1888 | mov L, CRET1
1889 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
1890 | sub RC, RC, BASE
1891 | st_vmstate ST_INTERP
1892 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1893 | ins_callt
1894 |.endif
1895 |
1896 |->cont_ffi_callback: // Return from FFI callback.
1897 |.if FFI
1898 | ldr CTSTATE, GL->ctype_state
1899 | stp BASE, CARG4, L->base
1900 | str L, CTSTATE->L
1901 | mov CARG1, CTSTATE
1902 | mov CARG2, RA
1903 | bl extern lj_ccallback_leave // (CTState *cts, TValue *o)
1904 | ldp x0, x1, CTSTATE->cb.gpr[0]
1905 | ldp d0, d1, CTSTATE->cb.fpr[0]
1906 | b ->vm_leave_unw
1907 |.endif
1908 |
1909 |->vm_ffi_call: // Call C function via FFI.
1910 | // Caveat: needs special frame unwinding, see below.
1911 |.if FFI
1912 | .type CCSTATE, CCallState, x19
1913 | stp fp, lr, [sp, #-32]!
1914 | add fp, sp, #0
1915 | str CCSTATE, [sp, #16]
1916 | mov CCSTATE, x0
1917 | ldr TMP0w, CCSTATE:x0->spadj
1918 | ldrb TMP1w, CCSTATE->nsp
1919 | add TMP2, CCSTATE, #offsetof(CCallState, stack)
1920 | subs TMP1, TMP1, #1
1921 | ldr TMP3, CCSTATE->func
1922 | sub sp, fp, TMP0
1923 | bmi >2
1924 |1: // Copy stack slots
1925 | ldr TMP0, [TMP2, TMP1, lsl #3]
1926 | str TMP0, [sp, TMP1, lsl #3]
1927 | subs TMP1, TMP1, #1
1928 | bpl <1
1929 |2:
1930 | ldp x0, x1, CCSTATE->gpr[0]
1931 | ldp d0, d1, CCSTATE->fpr[0]
1932 | ldp x2, x3, CCSTATE->gpr[2]
1933 | ldp d2, d3, CCSTATE->fpr[2]
1934 | ldp x4, x5, CCSTATE->gpr[4]
1935 | ldp d4, d5, CCSTATE->fpr[4]
1936 | ldp x6, x7, CCSTATE->gpr[6]
1937 | ldp d6, d7, CCSTATE->fpr[6]
1938 | ldr x8, CCSTATE->retp
1939 | blr TMP3
1940 | mov sp, fp
1941 | stp x0, x1, CCSTATE->gpr[0]
1942 | stp d0, d1, CCSTATE->fpr[0]
1943 | stp d2, d3, CCSTATE->fpr[2]
1944 | ldr CCSTATE, [sp, #16]
1945 | ldp fp, lr, [sp], #32
1946 | ret
1947 |.endif
1948 |// Note: vm_ffi_call must be the last function in this object file!
1949 |
1950 |//-----------------------------------------------------------------------
1951 }
1953 /* Generate the code for a single instruction. */
1954 static void build_ins(BuildCtx *ctx, BCOp op, int defop)
1955 {
1956 int vk = 0;
1957 |=>defop:
1959 switch (op) {
1961 /* -- Comparison ops ---------------------------------------------------- */
1963 /* Remember: all ops branch for a true comparison, fall through otherwise. */
1965 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
1966 | // RA = src1, RC = src2, JMP with RC = target
1967 | ldr CARG1, [BASE, RA, lsl #3]
1968 | ldrh RBw, [PC, #2]
1969 | ldr CARG2, [BASE, RC, lsl #3]
1970 | add PC, PC, #4
1971 | add RB, PC, RB, lsl #2
1972 | sub RB, RB, #0x20000
1973 | checkint CARG1, >3
1974 | checkint CARG2, >4
1975 | cmp CARG1w, CARG2w
1976 if (op == BC_ISLT) {
1977 | csel PC, RB, PC, lt
1978 } else if (op == BC_ISGE) {
1979 | csel PC, RB, PC, ge
1980 } else if (op == BC_ISLE) {
1981 | csel PC, RB, PC, le
1982 } else {
1983 | csel PC, RB, PC, gt
1984 }
1985 |1:
1986 | ins_next
1987 |
1988 |3: // RA not int.
1989 | ldr FARG1, [BASE, RA, lsl #3]
1990 | blo ->vmeta_comp
1991 | ldr FARG2, [BASE, RC, lsl #3]
1992 | cmp TISNUMhi, CARG2, lsr #32
1993 | bhi >5
1994 | bne ->vmeta_comp
1995 | // RA number, RC int.
1996 | scvtf FARG2, CARG2w
1997 | b >5
1998 |
1999 |4: // RA int, RC not int
2000 | ldr FARG2, [BASE, RC, lsl #3]
2001 | blo ->vmeta_comp
2002 | // RA int, RC number.
2003 | scvtf FARG1, CARG1w
2004 |
2005 |5: // RA number, RC number
2006 | fcmp FARG1, FARG2
2007 | // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2008 if (op == BC_ISLT) {
2009 | csel PC, RB, PC, lo
2010 } else if (op == BC_ISGE) {
2011 | csel PC, RB, PC, hs
2012 } else if (op == BC_ISLE) {
2013 | csel PC, RB, PC, ls
2014 } else {
2015 | csel PC, RB, PC, hi
2016 }
2017 | b <1
2018 break;
2020 case BC_ISEQV: case BC_ISNEV:
2021 vk = op == BC_ISEQV;
2022 | // RA = src1, RC = src2, JMP with RC = target
2023 | ldr CARG1, [BASE, RA, lsl #3]
2024 | add RC, BASE, RC, lsl #3
2025 | ldrh RBw, [PC, #2]
2026 | ldr CARG3, [RC]
2027 | add PC, PC, #4
2028 | add RB, PC, RB, lsl #2
2029 | sub RB, RB, #0x20000
2030 | asr ITYPE, CARG3, #47
2031 | cmn ITYPE, #-LJ_TISNUM
2032 if (vk) {
2033 | bls ->BC_ISEQN_Z
2034 } else {
2035 | bls ->BC_ISNEN_Z
2036 }
2037 | // RC is not a number.
2038 | asr TMP0, CARG1, #47
2039 |.if FFI
2040 | // Check if RC or RA is a cdata.
2041 | cmn ITYPE, #-LJ_TCDATA
2042 | ccmn TMP0, #-LJ_TCDATA, #4, ne
2043 | beq ->vmeta_equal_cd
2044 |.endif
2045 | cmp CARG1, CARG3
2046 | bne >2
2047 | // Tag and value are equal.
2048 if (vk) {
2049 |->BC_ISEQV_Z:
2050 | mov PC, RB // Perform branch.
2051 }
2052 |1:
2053 | ins_next
2054 |
2055 |2: // Check if the tags are the same and it's a table or userdata.
2056 | cmp ITYPE, TMP0
2057 | ccmn ITYPE, #-LJ_TISTABUD, #2, eq
2058 if (vk) {
2059 | bhi <1
2060 } else {
2061 | bhi ->BC_ISEQV_Z // Reuse code from opposite instruction.
2062 }
2063 | // Different tables or userdatas. Need to check __eq metamethod.
2064 | // Field metatable must be at same offset for GCtab and GCudata!
2065 | and TAB:CARG2, CARG1, #LJ_GCVMASK
2066 | ldr TAB:TMP2, TAB:CARG2->metatable
2067 if (vk) {
2068 | cbz TAB:TMP2, <1 // No metatable?
2069 | ldrb TMP1w, TAB:TMP2->nomm
2070 | mov CARG4, #0 // ne = 0
2071 | tbnz TMP1w, #MM_eq, <1 // 'no __eq' flag set: done.
2072 } else {
2073 | cbz TAB:TMP2, ->BC_ISEQV_Z // No metatable?
2074 | ldrb TMP1w, TAB:TMP2->nomm
2075 | mov CARG4, #1 // ne = 1.
2076 | tbnz TMP1w, #MM_eq, ->BC_ISEQV_Z // 'no __eq' flag set: done.
2077 }
2078 | b ->vmeta_equal
2079 break;
2081 case BC_ISEQS: case BC_ISNES:
2082 vk = op == BC_ISEQS;
2083 | // RA = src, RC = str_const (~), JMP with RC = target
2084 | ldr CARG1, [BASE, RA, lsl #3]
2085 | mvn RC, RC
2086 | ldrh RBw, [PC, #2]
2087 | ldr CARG2, [KBASE, RC, lsl #3]
2088 | add PC, PC, #4
2089 | movn TMP0, #~LJ_TSTR
2090 |.if FFI
2091 | asr ITYPE, CARG1, #47
2092 |.endif
2093 | add RB, PC, RB, lsl #2
2094 | add CARG2, CARG2, TMP0, lsl #47
2095 | sub RB, RB, #0x20000
2096 |.if FFI
2097 | cmn ITYPE, #-LJ_TCDATA
2098 | beq ->vmeta_equal_cd
2099 |.endif
2100 | cmp CARG1, CARG2
2101 if (vk) {
2102 | csel PC, RB, PC, eq
2103 } else {
2104 | csel PC, RB, PC, ne
2105 }
2106 | ins_next
2107 break;
2109 case BC_ISEQN: case BC_ISNEN:
2110 vk = op == BC_ISEQN;
2111 | // RA = src, RC = num_const (~), JMP with RC = target
2112 | ldr CARG1, [BASE, RA, lsl #3]
2113 | add RC, KBASE, RC, lsl #3
2114 | ldrh RBw, [PC, #2]
2115 | ldr CARG3, [RC]
2116 | add PC, PC, #4
2117 | add RB, PC, RB, lsl #2
2118 | sub RB, RB, #0x20000
2119 if (vk) {
2120 |->BC_ISEQN_Z:
2121 } else {
2122 |->BC_ISNEN_Z:
2123 }
2124 | checkint CARG1, >4
2125 | checkint CARG3, >6
2126 | cmp CARG1w, CARG3w
2127 |1:
2128 if (vk) {
2129 | csel PC, RB, PC, eq
2130 |2:
2131 } else {
2132 |2:
2133 | csel PC, RB, PC, ne
2134 }
2135 |3:
2136 | ins_next
2137 |
2138 |4: // RA not int.
2139 |.if FFI
2140 | blo >7
2141 |.else
2142 | blo <2
2143 |.endif
2144 | ldr FARG1, [BASE, RA, lsl #3]
2145 | ldr FARG2, [RC]
2146 | cmp TISNUMhi, CARG3, lsr #32
2147 | bne >5
2148 | // RA number, RC int.
2149 | scvtf FARG2, CARG3w
2150 |5:
2151 | // RA number, RC number.
2152 | fcmp FARG1, FARG2
2153 | b <1
2154 |
2155 |6: // RA int, RC number
2156 | ldr FARG2, [RC]
2157 | scvtf FARG1, CARG1w
2158 | fcmp FARG1, FARG2
2159 | b <1
2160 |
2161 |.if FFI
2162 |7:
2163 | asr ITYPE, CARG1, #47
2164 | cmn ITYPE, #-LJ_TCDATA
2165 | bne <2
2166 | b ->vmeta_equal_cd
2167 |.endif
2168 break;
2170 case BC_ISEQP: case BC_ISNEP:
2171 vk = op == BC_ISEQP;
2172 | // RA = src, RC = primitive_type (~), JMP with RC = target
2173 | ldr TMP0, [BASE, RA, lsl #3]
2174 | ldrh RBw, [PC, #2]
2175 | add PC, PC, #4
2176 | add RC, RC, #1
2177 | add RB, PC, RB, lsl #2
2178 |.if FFI
2179 | asr ITYPE, TMP0, #47
2180 | cmn ITYPE, #-LJ_TCDATA
2181 | beq ->vmeta_equal_cd
2182 | cmn RC, ITYPE
2183 |.else
2184 | cmn RC, TMP0, asr #47
2185 |.endif
2186 | sub RB, RB, #0x20000
2187 if (vk) {
2188 | csel PC, RB, PC, eq
2189 } else {
2190 | csel PC, RB, PC, ne
2191 }
2192 | ins_next
2193 break;
2195 /* -- Unary test and copy ops ------------------------------------------- */
2197 case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
2198 | // RA = dst or unused, RC = src, JMP with RC = target
2199 | ldrh RBw, [PC, #2]
2200 | ldr TMP0, [BASE, RC, lsl #3]
2201 | add PC, PC, #4
2202 | mov_false TMP1
2203 | add RB, PC, RB, lsl #2
2204 | cmp TMP0, TMP1
2205 | sub RB, RB, #0x20000
2206 if (op == BC_ISTC || op == BC_IST) {
2207 if (op == BC_ISTC) {
2208 | csel RA, RA, RC, lo
2209 }
2210 | csel PC, RB, PC, lo
2211 } else {
2212 if (op == BC_ISFC) {
2213 | csel RA, RA, RC, hs
2214 }
2215 | csel PC, RB, PC, hs
2216 }
2217 if (op == BC_ISTC || op == BC_ISFC) {
2218 | str TMP0, [BASE, RA, lsl #3]
2219 }
2220 | ins_next
2221 break;
2223 case BC_ISTYPE:
2224 | // RA = src, RC = -type
2225 | ldr TMP0, [BASE, RA, lsl #3]
2226 | cmn RC, TMP0, asr #47
2227 | bne ->vmeta_istype
2228 | ins_next
2229 break;
2230 case BC_ISNUM:
2231 | // RA = src, RC = -(TISNUM-1)
2232 | ldr TMP0, [BASE, RA]
2233 | checknum TMP0, ->vmeta_istype
2234 | ins_next
2235 break;
2237 /* -- Unary ops --------------------------------------------------------- */
2239 case BC_MOV:
2240 | // RA = dst, RC = src
2241 | ldr TMP0, [BASE, RC, lsl #3]
2242 | str TMP0, [BASE, RA, lsl #3]
2243 | ins_next
2244 break;
2245 case BC_NOT:
2246 | // RA = dst, RC = src
2247 | ldr TMP0, [BASE, RC, lsl #3]
2248 | mov_false TMP1
2249 | mov_true TMP2
2250 | cmp TMP0, TMP1
2251 | csel TMP0, TMP1, TMP2, lo
2252 | str TMP0, [BASE, RA, lsl #3]
2253 | ins_next
2254 break;
2255 case BC_UNM:
2256 | // RA = dst, RC = src
2257 | ldr TMP0, [BASE, RC, lsl #3]
2258 | asr ITYPE, TMP0, #47
2259 | cmn ITYPE, #-LJ_TISNUM
2260 | bhi ->vmeta_unm
2261 | eor TMP0, TMP0, #U64x(80000000,00000000)
2262 | bne >5
2263 | negs TMP0w, TMP0w
2264 | movz CARG3, #0x41e0, lsl #48 // 2^31.
2265 | add TMP0, TMP0, TISNUM
2266 | csel TMP0, TMP0, CARG3, vc
2267 |5:
2268 | str TMP0, [BASE, RA, lsl #3]
2269 | ins_next
2270 break;
2271 case BC_LEN:
2272 | // RA = dst, RC = src
2273 | ldr CARG1, [BASE, RC, lsl #3]
2274 | asr ITYPE, CARG1, #47
2275 | cmn ITYPE, #-LJ_TSTR
2276 | and CARG1, CARG1, #LJ_GCVMASK
2277 | bne >2
2278 | ldr CARG1w, STR:CARG1->len
2279 |1:
2280 | add CARG1, CARG1, TISNUM
2281 | str CARG1, [BASE, RA, lsl #3]
2282 | ins_next
2283 |
2284 |2:
2285 | cmn ITYPE, #-LJ_TTAB
2286 | bne ->vmeta_len
2287 #if LJ_52
2288 | ldr TAB:CARG2, TAB:CARG1->metatable
2289 | cbnz TAB:CARG2, >9
2290 |3:
2291 #endif
2292 |->BC_LEN_Z:
2293 | bl extern lj_tab_len // (GCtab *t)
2294 | // Returns uint32_t (but less than 2^31).
2295 | b <1
2296 |
2297 #if LJ_52
2298 |9:
2299 | ldrb TMP1w, TAB:CARG2->nomm
2300 | tbnz TMP1w, #MM_len, <3 // 'no __len' flag set: done.
2301 | b ->vmeta_len
2302 #endif
2303 break;
2305 /* -- Binary ops -------------------------------------------------------- */
2307 |.macro ins_arithcheck_int, target
2308 | checkint CARG1, target
2309 | checkint CARG2, target
2310 |.endmacro
2311 |
2312 |.macro ins_arithcheck_num, target
2313 | checknum CARG1, target
2314 | checknum CARG2, target
2315 |.endmacro
2316 |
2317 |.macro ins_arithcheck_nzdiv, target
2318 | cbz CARG2w, target
2319 |.endmacro
2320 |
2321 |.macro ins_arithhead
2322 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
2323 ||if (vk == 1) {
2324 | and RC, RC, #255
2325 | decode_RB RB, INS
2326 ||} else {
2327 | decode_RB RB, INS
2328 | and RC, RC, #255
2329 ||}
2330 |.endmacro
2331 |
2332 |.macro ins_arithload, reg1, reg2
2333 | // RA = dst, RB = src1, RC = src2 | num_const
2334 ||switch (vk) {
2335 ||case 0:
2336 | ldr reg1, [BASE, RB, lsl #3]
2337 | ldr reg2, [KBASE, RC, lsl #3]
2338 || break;
2339 ||case 1:
2340 | ldr reg1, [KBASE, RC, lsl #3]
2341 | ldr reg2, [BASE, RB, lsl #3]
2342 || break;
2343 ||default:
2344 | ldr reg1, [BASE, RB, lsl #3]
2345 | ldr reg2, [BASE, RC, lsl #3]
2346 || break;
2347 ||}
2348 |.endmacro
2349 |
2350 |.macro ins_arithfallback, ins
2351 ||switch (vk) {
2352 ||case 0:
2353 | ins ->vmeta_arith_vn
2354 || break;
2355 ||case 1:
2356 | ins ->vmeta_arith_nv
2357 || break;
2358 ||default:
2359 | ins ->vmeta_arith_vv
2360 || break;
2361 ||}
2362 |.endmacro
2363 |
2364 |.macro ins_arithmod, res, reg1, reg2
2365 | fdiv d2, reg1, reg2
2366 | frintm d2, d2
2367 | fmsub res, d2, reg2, reg1
2368 |.endmacro
2369 |
2370 |.macro ins_arithdn, intins, fpins
2371 | ins_arithhead
2372 | ins_arithload CARG1, CARG2
2373 | ins_arithcheck_int >5
2374 |.if "intins" == "smull"
2375 | smull CARG1, CARG1w, CARG2w
2376 | cmp CARG1, CARG1, sxtw
2377 | mov CARG1w, CARG1w
2378 | ins_arithfallback bne
2379 |.elif "intins" == "ins_arithmodi"
2380 | ins_arithfallback ins_arithcheck_nzdiv
2381 | bl ->vm_modi
2382 |.else
2383 | intins CARG1w, CARG1w, CARG2w
2384 | ins_arithfallback bvs
2385 |.endif
2386 | add CARG1, CARG1, TISNUM
2387 | str CARG1, [BASE, RA, lsl #3]
2388 |4:
2389 | ins_next
2390 |
2391 |5: // FP variant.
2392 | ins_arithload FARG1, FARG2
2393 | ins_arithfallback ins_arithcheck_num
2394 | fpins FARG1, FARG1, FARG2
2395 | str FARG1, [BASE, RA, lsl #3]
2396 | b <4
2397 |.endmacro
2398 |
2399 |.macro ins_arithfp, fpins
2400 | ins_arithhead
2401 | ins_arithload CARG1, CARG2
2402 | ins_arithload FARG1, FARG2
2403 | ins_arithfallback ins_arithcheck_num
2404 |.if "fpins" == "fpow"
2405 | bl extern pow
2406 |.else
2407 | fpins FARG1, FARG1, FARG2
2408 |.endif
2409 | str FARG1, [BASE, RA, lsl #3]
2410 | ins_next
2411 |.endmacro
2413 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
2414 | ins_arithdn adds, fadd
2415 break;
2416 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
2417 | ins_arithdn subs, fsub
2418 break;
2419 case BC_MULVN: case BC_MULNV: case BC_MULVV:
2420 | ins_arithdn smull, fmul
2421 break;
2422 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
2423 | ins_arithfp fdiv
2424 break;
2425 case BC_MODVN: case BC_MODNV: case BC_MODVV:
2426 | ins_arithdn ins_arithmodi, ins_arithmod
2427 break;
2428 case BC_POW:
2429 | // NYI: (partial) integer arithmetic.
2430 | ins_arithfp fpow
2431 break;
2433 case BC_CAT:
2434 | decode_RB RB, INS
2435 | and RC, RC, #255
2436 | // RA = dst, RB = src_start, RC = src_end
2437 | str BASE, L->base
2438 | sub CARG3, RC, RB
2439 | add CARG2, BASE, RC, lsl #3
2440 |->BC_CAT_Z:
2441 | // RA = dst, CARG2 = top-1, CARG3 = left
2442 | mov CARG1, L
2443 | str PC, SAVE_PC
2444 | bl extern lj_meta_cat // (lua_State *L, TValue *top, int left)
2445 | // Returns NULL (finished) or TValue * (metamethod).
2446 | ldrb RBw, [PC, #-1]
2447 | ldr BASE, L->base
2448 | cbnz CRET1, ->vmeta_binop
2449 | ldr TMP0, [BASE, RB, lsl #3]
2450 | str TMP0, [BASE, RA, lsl #3] // Copy result to RA.
2451 | ins_next
2452 break;
2454 /* -- Constant ops ------------------------------------------------------ */
2456 case BC_KSTR:
2457 | // RA = dst, RC = str_const (~)
2458 | mvn RC, RC
2459 | ldr TMP0, [KBASE, RC, lsl #3]
2460 | movn TMP1, #~LJ_TSTR
2461 | add TMP0, TMP0, TMP1, lsl #47
2462 | str TMP0, [BASE, RA, lsl #3]
2463 | ins_next
2464 break;
2465 case BC_KCDATA:
2466 |.if FFI
2467 | // RA = dst, RC = cdata_const (~)
2468 | mvn RC, RC
2469 | ldr TMP0, [KBASE, RC, lsl #3]
2470 | movn TMP1, #~LJ_TCDATA
2471 | add TMP0, TMP0, TMP1, lsl #47
2472 | str TMP0, [BASE, RA, lsl #3]
2473 | ins_next
2474 |.endif
2475 break;
2476 case BC_KSHORT:
2477 | // RA = dst, RC = int16_literal
2478 | sxth RCw, RCw
2479 | add TMP0, RC, TISNUM
2480 | str TMP0, [BASE, RA, lsl #3]
2481 | ins_next
2482 break;
2483 case BC_KNUM:
2484 | // RA = dst, RC = num_const
2485 | ldr TMP0, [KBASE, RC, lsl #3]
2486 | str TMP0, [BASE, RA, lsl #3]
2487 | ins_next
2488 break;
2489 case BC_KPRI:
2490 | // RA = dst, RC = primitive_type (~)
2491 | mvn TMP0, RC, lsl #47
2492 | str TMP0, [BASE, RA, lsl #3]
2493 | ins_next
2494 break;
2495 case BC_KNIL:
2496 | // RA = base, RC = end
2497 | add RA, BASE, RA, lsl #3
2498 | add RC, BASE, RC, lsl #3
2499 | str TISNIL, [RA], #8
2500 |1:
2501 | cmp RA, RC
2502 | str TISNIL, [RA], #8
2503 | blt <1
2504 | ins_next_
2505 break;
2507 /* -- Upvalue and function ops ------------------------------------------ */
2509 case BC_UGET:
2510 | // RA = dst, RC = uvnum
2511 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2512 | add RC, RC, #offsetof(GCfuncL, uvptr)/8
2513 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2514 | ldr UPVAL:CARG2, [LFUNC:CARG2, RC, lsl #3]
2515 | ldr CARG2, UPVAL:CARG2->v
2516 | ldr TMP0, [CARG2]
2517 | str TMP0, [BASE, RA, lsl #3]
2518 | ins_next
2519 break;
2520 case BC_USETV:
2521 | // RA = uvnum, RC = src
2522 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2523 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2524 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2525 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2526 | ldr CARG3, [BASE, RC, lsl #3]
2527 | ldr CARG2, UPVAL:CARG1->v
2528 | ldrb TMP2w, UPVAL:CARG1->marked
2529 | ldrb TMP0w, UPVAL:CARG1->closed
2530 | asr ITYPE, CARG3, #47
2531 | str CARG3, [CARG2]
2532 | add ITYPE, ITYPE, #-LJ_TISGCV
2533 | tst TMP2w, #LJ_GC_BLACK // isblack(uv)
2534 | ccmp TMP0w, #0, #4, ne // && uv->closed
2535 | ccmn ITYPE, #-(LJ_TNUMX - LJ_TISGCV), #0, ne // && tvisgcv(v)
2536 | bhi >2
2537 |1:
2538 | ins_next
2539 |
2540 |2: // Check if new value is white.
2541 | and GCOBJ:CARG3, CARG3, #LJ_GCVMASK
2542 | ldrb TMP1w, GCOBJ:CARG3->gch.marked
2543 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2544 | beq <1
2545 | // Crossed a write barrier. Move the barrier forward.
2546 | mov CARG1, GL
2547 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2548 | b <1
2549 break;
2550 case BC_USETS:
2551 | // RA = uvnum, RC = str_const (~)
2552 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2553 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2554 | mvn RC, RC
2555 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2556 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2557 | ldr STR:CARG3, [KBASE, RC, lsl #3]
2558 | movn TMP0, #~LJ_TSTR
2559 | ldr CARG2, UPVAL:CARG1->v
2560 | ldrb TMP2w, UPVAL:CARG1->marked
2561 | add TMP0, STR:CARG3, TMP0, lsl #47
2562 | ldrb TMP1w, STR:CARG3->marked
2563 | str TMP0, [CARG2]
2564 | tbnz TMP2w, #2, >2 // isblack(uv)
2565 |1:
2566 | ins_next
2567 |
2568 |2: // Check if string is white and ensure upvalue is closed.
2569 | ldrb TMP0w, UPVAL:CARG1->closed
2570 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2571 | ccmp TMP0w, #0, #0, ne
2572 | beq <1
2573 | // Crossed a write barrier. Move the barrier forward.
2574 | mov CARG1, GL
2575 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2576 | b <1
2577 break;
2578 case BC_USETN:
2579 | // RA = uvnum, RC = num_const
2580 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2581 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2582 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2583 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2584 | ldr TMP0, [KBASE, RC, lsl #3]
2585 | ldr CARG2, UPVAL:CARG2->v
2586 | str TMP0, [CARG2]
2587 | ins_next
2588 break;
2589 case BC_USETP:
2590 | // RA = uvnum, RC = primitive_type (~)
2591 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2592 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2593 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2594 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2595 | mvn TMP0, RC, lsl #47
2596 | ldr CARG2, UPVAL:CARG2->v
2597 | str TMP0, [CARG2]
2598 | ins_next
2599 break;
2601 case BC_UCLO:
2602 | // RA = level, RC = target
2603 | ldr CARG3, L->openupval
2604 | add RC, PC, RC, lsl #2
2605 | str BASE, L->base
2606 | sub PC, RC, #0x20000
2607 | cbz CARG3, >1
2608 | mov CARG1, L
2609 | add CARG2, BASE, RA, lsl #3
2610 | bl extern lj_func_closeuv // (lua_State *L, TValue *level)
2611 | ldr BASE, L->base
2612 |1:
2613 | ins_next
2614 break;
2616 case BC_FNEW:
2617 | // RA = dst, RC = proto_const (~) (holding function prototype)
2618 | mvn RC, RC
2619 | str BASE, L->base
2620 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2621 | str PC, SAVE_PC
2622 | ldr CARG2, [KBASE, RC, lsl #3]
2623 | mov CARG1, L
2624 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2625 | // (lua_State *L, GCproto *pt, GCfuncL *parent)
2626 | bl extern lj_func_newL_gc
2627 | // Returns GCfuncL *.
2628 | ldr BASE, L->base
2629 | movn TMP0, #~LJ_TFUNC
2630 | add CRET1, CRET1, TMP0, lsl #47
2631 | str CRET1, [BASE, RA, lsl #3]
2632 | ins_next
2633 break;
2635 /* -- Table ops --------------------------------------------------------- */
2637 case BC_TNEW:
2638 case BC_TDUP:
2639 | // RA = dst, RC = (hbits|asize) | tab_const (~)
2640 | ldp CARG3, CARG4, GL->gc.total // Assumes threshold follows total.
2641 | str BASE, L->base
2642 | str PC, SAVE_PC
2643 | mov CARG1, L
2644 | cmp CARG3, CARG4
2645 | bhs >5
2646 |1:
2647 if (op == BC_TNEW) {
2648 | and CARG2, RC, #0x7ff
2649 | lsr CARG3, RC, #11
2650 | cmp CARG2, #0x7ff
2651 | mov TMP0, #0x801
2652 | csel CARG2, CARG2, TMP0, ne
2653 | bl extern lj_tab_new // (lua_State *L, int32_t asize, uint32_t hbits)
2654 | // Returns GCtab *.
2655 } else {
2656 | mvn RC, RC
2657 | ldr CARG2, [KBASE, RC, lsl #3]
2658 | bl extern lj_tab_dup // (lua_State *L, Table *kt)
2659 | // Returns GCtab *.
2660 }
2661 | ldr BASE, L->base
2662 | movk CRET1, #(LJ_TTAB>>1)&0xffff, lsl #48
2663 | str CRET1, [BASE, RA, lsl #3]
2664 | ins_next
2665 |
2666 |5:
2667 | bl extern lj_gc_step_fixtop // (lua_State *L)
2668 | mov CARG1, L
2669 | b <1
2670 break;
2672 case BC_GGET:
2673 | // RA = dst, RC = str_const (~)
2674 case BC_GSET:
2675 | // RA = dst, RC = str_const (~)
2676 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
2677 | mvn RC, RC
2678 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
2679 | ldr TAB:CARG2, LFUNC:CARG1->env
2680 | ldr STR:RC, [KBASE, RC, lsl #3]
2681 if (op == BC_GGET) {
2682 | b ->BC_TGETS_Z
2683 } else {
2684 | b ->BC_TSETS_Z
2685 }
2686 break;
2688 case BC_TGETV:
2689 | decode_RB RB, INS
2690 | and RC, RC, #255
2691 | // RA = dst, RB = table, RC = key
2692 | ldr CARG2, [BASE, RB, lsl #3]
2693 | ldr TMP1, [BASE, RC, lsl #3]
2694 | checktab CARG2, ->vmeta_tgetv
2695 | checkint TMP1, >9 // Integer key?
2696 | ldr CARG3, TAB:CARG2->array
2697 | ldr CARG1w, TAB:CARG2->asize
2698 | add CARG3, CARG3, TMP1, uxtw #3
2699 | cmp TMP1w, CARG1w // In array part?
2700 | bhs ->vmeta_tgetv
2701 | ldr TMP0, [CARG3]
2702 | cmp TMP0, TISNIL
2703 | beq >5
2704 |1:
2705 | str TMP0, [BASE, RA, lsl #3]
2706 | ins_next
2707 |
2708 |5: // Check for __index if table value is nil.
2709 | ldr TAB:CARG1, TAB:CARG2->metatable
2710 | cbz TAB:CARG1, <1 // No metatable: done.
2711 | ldrb TMP1w, TAB:CARG1->nomm
2712 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
2713 | b ->vmeta_tgetv
2714 |
2715 |9:
2716 | asr ITYPE, TMP1, #47
2717 | cmn ITYPE, #-LJ_TSTR // String key?
2718 | bne ->vmeta_tgetv
2719 | and STR:RC, TMP1, #LJ_GCVMASK
2720 | b ->BC_TGETS_Z
2721 break;
2722 case BC_TGETS:
2723 | decode_RB RB, INS
2724 | and RC, RC, #255
2725 | // RA = dst, RB = table, RC = str_const (~)
2726 | ldr CARG2, [BASE, RB, lsl #3]
2727 | mvn RC, RC
2728 | ldr STR:RC, [KBASE, RC, lsl #3]
2729 | checktab CARG2, ->vmeta_tgets1
2730 |->BC_TGETS_Z:
2731 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst
2732 | ldr TMP1w, TAB:CARG2->hmask
2733 | ldr TMP2w, STR:RC->hash
2734 | ldr NODE:CARG3, TAB:CARG2->node
2735 | and TMP1w, TMP1w, TMP2w // idx = str->hash & tab->hmask
2736 | add TMP1, TMP1, TMP1, lsl #1
2737 | movn CARG4, #~LJ_TSTR
2738 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
2739 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
2740 |1:
2741 | ldp TMP0, CARG1, NODE:CARG3->val
2742 | ldr NODE:CARG3, NODE:CARG3->next
2743 | cmp CARG1, CARG4
2744 | bne >4
2745 | cmp TMP0, TISNIL
2746 | beq >5
2747 |3:
2748 | str TMP0, [BASE, RA, lsl #3]
2749 | ins_next
2750 |
2751 |4: // Follow hash chain.
2752 | cbnz NODE:CARG3, <1
2753 | // End of hash chain: key not found, nil result.
2754 | mov TMP0, TISNIL
2755 |
2756 |5: // Check for __index if table value is nil.
2757 | ldr TAB:CARG1, TAB:CARG2->metatable
2758 | cbz TAB:CARG1, <3 // No metatable: done.
2759 | ldrb TMP1w, TAB:CARG1->nomm
2760 | tbnz TMP1w, #MM_index, <3 // 'no __index' flag set: done.
2761 | b ->vmeta_tgets
2762 break;
2763 case BC_TGETB:
2764 | decode_RB RB, INS
2765 | and RC, RC, #255
2766 | // RA = dst, RB = table, RC = index
2767 | ldr CARG2, [BASE, RB, lsl #3]
2768 | checktab CARG2, ->vmeta_tgetb
2769 | ldr CARG3, TAB:CARG2->array
2770 | ldr CARG1w, TAB:CARG2->asize
2771 | add CARG3, CARG3, RC, lsl #3
2772 | cmp RCw, CARG1w // In array part?
2773 | bhs ->vmeta_tgetb
2774 | ldr TMP0, [CARG3]
2775 | cmp TMP0, TISNIL
2776 | beq >5
2777 |1:
2778 | str TMP0, [BASE, RA, lsl #3]
2779 | ins_next
2780 |
2781 |5: // Check for __index if table value is nil.
2782 | ldr TAB:CARG1, TAB:CARG2->metatable
2783 | cbz TAB:CARG1, <1 // No metatable: done.
2784 | ldrb TMP1w, TAB:CARG1->nomm
2785 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
2786 | b ->vmeta_tgetb
2787 break;
2788 case BC_TGETR:
2789 | decode_RB RB, INS
2790 | and RC, RC, #255
2791 | // RA = dst, RB = table, RC = key
2792 | ldr CARG1, [BASE, RB, lsl #3]
2793 | ldr TMP1, [BASE, RC, lsl #3]
2794 | and TAB:CARG1, CARG1, #LJ_GCVMASK
2795 | ldr CARG3, TAB:CARG1->array
2796 | ldr TMP2w, TAB:CARG1->asize
2797 | add CARG3, CARG3, TMP1w, uxtw #3
2798 | cmp TMP1w, TMP2w // In array part?
2799 | bhs ->vmeta_tgetr
2800 | ldr TMP0, [CARG3]
2801 |->BC_TGETR_Z:
2802 | str TMP0, [BASE, RA, lsl #3]
2803 | ins_next
2804 break;
2806 case BC_TSETV:
2807 | decode_RB RB, INS
2808 | and RC, RC, #255
2809 | // RA = src, RB = table, RC = key
2810 | ldr CARG2, [BASE, RB, lsl #3]
2811 | ldr TMP1, [BASE, RC, lsl #3]
2812 | checktab CARG2, ->vmeta_tsetv
2813 | checkint TMP1, >9 // Integer key?
2814 | ldr CARG3, TAB:CARG2->array
2815 | ldr CARG1w, TAB:CARG2->asize
2816 | add CARG3, CARG3, TMP1, uxtw #3
2817 | cmp TMP1w, CARG1w // In array part?
2818 | bhs ->vmeta_tsetv
2819 | ldr TMP1, [CARG3]
2820 | ldr TMP0, [BASE, RA, lsl #3]
2821 | ldrb TMP2w, TAB:CARG2->marked
2822 | cmp TMP1, TISNIL // Previous value is nil?
2823 | beq >5
2824 |1:
2825 | str TMP0, [CARG3]
2826 | tbnz TMP2w, #2, >7 // isblack(table)
2827 |2:
2828 | ins_next
2829 |
2830 |5: // Check for __newindex if previous value is nil.
2831 | ldr TAB:CARG1, TAB:CARG2->metatable
2832 | cbz TAB:CARG1, <1 // No metatable: done.
2833 | ldrb TMP1w, TAB:CARG1->nomm
2834 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
2835 | b ->vmeta_tsetv
2836 |
2837 |7: // Possible table write barrier for the value. Skip valiswhite check.
2838 | barrierback TAB:CARG2, TMP2w, TMP1
2839 | b <2
2840 |
2841 |9:
2842 | asr ITYPE, TMP1, #47
2843 | cmn ITYPE, #-LJ_TSTR // String key?
2844 | bne ->vmeta_tsetv
2845 | and STR:RC, TMP1, #LJ_GCVMASK
2846 | b ->BC_TSETS_Z
2847 break;
2848 case BC_TSETS:
2849 | decode_RB RB, INS
2850 | and RC, RC, #255
2851 | // RA = dst, RB = table, RC = str_const (~)
2852 | ldr CARG2, [BASE, RB, lsl #3]
2853 | mvn RC, RC
2854 | ldr STR:RC, [KBASE, RC, lsl #3]
2855 | checktab CARG2, ->vmeta_tsets1
2856 |->BC_TSETS_Z:
2857 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src
2858 | ldr TMP1w, TAB:CARG2->hmask
2859 | ldr TMP2w, STR:RC->hash
2860 | ldr NODE:CARG3, TAB:CARG2->node
2861 | and TMP1w, TMP1w, TMP2w // idx = str->hash & tab->hmask
2862 | add TMP1, TMP1, TMP1, lsl #1
2863 | movn CARG4, #~LJ_TSTR
2864 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
2865 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
2866 | strb wzr, TAB:CARG2->nomm // Clear metamethod cache.
2867 |1:
2868 | ldp TMP1, CARG1, NODE:CARG3->val
2869 | ldr NODE:TMP3, NODE:CARG3->next
2870 | ldrb TMP2w, TAB:CARG2->marked
2871 | cmp CARG1, CARG4
2872 | bne >5
2873 | ldr TMP0, [BASE, RA, lsl #3]
2874 | cmp TMP1, TISNIL // Previous value is nil?
2875 | beq >4
2876 |2:
2877 | str TMP0, NODE:CARG3->val
2878 | tbnz TMP2w, #2, >7 // isblack(table)
2879 |3:
2880 | ins_next
2881 |
2882 |4: // Check for __newindex if previous value is nil.
2883 | ldr TAB:CARG1, TAB:CARG2->metatable
2884 | cbz TAB:CARG1, <2 // No metatable: done.
2885 | ldrb TMP1w, TAB:CARG1->nomm
2886 | tbnz TMP1w, #MM_newindex, <2 // 'no __newindex' flag set: done.
2887 | b ->vmeta_tsets
2888 |
2889 |5: // Follow hash chain.
2890 | mov NODE:CARG3, NODE:TMP3
2891 | cbnz NODE:TMP3, <1
2892 | // End of hash chain: key not found, add a new one.
2893 |
2894 | // But check for __newindex first.
2895 | ldr TAB:CARG1, TAB:CARG2->metatable
2896 | cbz TAB:CARG1, >6 // No metatable: continue.
2897 | ldrb TMP1w, TAB:CARG1->nomm
2898 | // 'no __newindex' flag NOT set: check.
2899 | tbz TMP1w, #MM_newindex, ->vmeta_tsets
2900 |6:
2901 | movn TMP1, #~LJ_TSTR
2902 | str PC, SAVE_PC
2903 | add TMP0, STR:RC, TMP1, lsl #47
2904 | str BASE, L->base
2905 | mov CARG1, L
2906 | str TMP0, TMPD
2907 | add CARG3, sp, TMPDofs
2908 | bl extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k)
2909 | // Returns TValue *.
2910 | ldr BASE, L->base
2911 | ldr TMP0, [BASE, RA, lsl #3]
2912 | str TMP0, [CRET1]
2913 | b <3 // No 2nd write barrier needed.
2914 |
2915 |7: // Possible table write barrier for the value. Skip valiswhite check.
2916 | barrierback TAB:CARG2, TMP2w, TMP1
2917 | b <3
2918 break;
2919 case BC_TSETB:
2920 | decode_RB RB, INS
2921 | and RC, RC, #255
2922 | // RA = src, RB = table, RC = index
2923 | ldr CARG2, [BASE, RB, lsl #3]
2924 | checktab CARG2, ->vmeta_tsetb
2925 | ldr CARG3, TAB:CARG2->array
2926 | ldr CARG1w, TAB:CARG2->asize
2927 | add CARG3, CARG3, RC, lsl #3
2928 | cmp RCw, CARG1w // In array part?
2929 | bhs ->vmeta_tsetb
2930 | ldr TMP1, [CARG3]
2931 | ldr TMP0, [BASE, RA, lsl #3]
2932 | ldrb TMP2w, TAB:CARG2->marked
2933 | cmp TMP1, TISNIL // Previous value is nil?
2934 | beq >5
2935 |1:
2936 | str TMP0, [CARG3]
2937 | tbnz TMP2w, #2, >7 // isblack(table)
2938 |2:
2939 | ins_next
2940 |
2941 |5: // Check for __newindex if previous value is nil.
2942 | ldr TAB:CARG1, TAB:CARG2->metatable
2943 | cbz TAB:CARG1, <1 // No metatable: done.
2944 | ldrb TMP1w, TAB:CARG1->nomm
2945 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
2946 | b ->vmeta_tsetb
2947 |
2948 |7: // Possible table write barrier for the value. Skip valiswhite check.
2949 | barrierback TAB:CARG2, TMP2w, TMP1
2950 | b <2
2951 break;
2952 case BC_TSETR:
2953 | decode_RB RB, INS
2954 | and RC, RC, #255
2955 | // RA = src, RB = table, RC = key
2956 | ldr CARG2, [BASE, RB, lsl #3]
2957 | ldr TMP1, [BASE, RC, lsl #3]
2958 | and TAB:CARG2, CARG2, #LJ_GCVMASK
2959 | ldr CARG1, TAB:CARG2->array
2960 | ldrb TMP2w, TAB:CARG2->marked
2961 | ldr CARG4w, TAB:CARG2->asize
2962 | add CARG1, CARG1, TMP1, uxtw #3
2963 | tbnz TMP2w, #2, >7 // isblack(table)
2964 |2:
2965 | cmp TMP1w, CARG4w // In array part?
2966 | bhs ->vmeta_tsetr
2967 |->BC_TSETR_Z:
2968 | ldr TMP0, [BASE, RA, lsl #3]
2969 | str TMP0, [CARG1]
2970 | ins_next
2971 |
2972 |7: // Possible table write barrier for the value. Skip valiswhite check.
2973 | barrierback TAB:CARG2, TMP2w, TMP0
2974 | b <2
2975 break;
2977 case BC_TSETM:
2978 | // RA = base (table at base-1), RC = num_const (start index)
2979 | add RA, BASE, RA, lsl #3
2980 |1:
2981 | ldr RBw, SAVE_MULTRES
2982 | ldr TAB:CARG2, [RA, #-8] // Guaranteed to be a table.
2983 | ldr TMP1, [KBASE, RC, lsl #3] // Integer constant is in lo-word.
2984 | sub RB, RB, #8
2985 | cbz RB, >4 // Nothing to copy?
2986 | and TAB:CARG2, CARG2, #LJ_GCVMASK
2987 | ldr CARG1w, TAB:CARG2->asize
2988 | add CARG3w, TMP1w, RBw, lsr #3
2989 | ldr CARG4, TAB:CARG2->array
2990 | cmp CARG3, CARG1
2991 | add RB, RA, RB
2992 | bhi >5
2993 | add TMP1, CARG4, TMP1w, uxtw #3
2994 | ldrb TMP2w, TAB:CARG2->marked
2995 |3: // Copy result slots to table.
2996 | ldr TMP0, [RA], #8
2997 | str TMP0, [TMP1], #8
2998 | cmp RA, RB
2999 | blo <3
3000 | tbnz TMP2w, #2, >7 // isblack(table)
3001 |4:
3002 | ins_next
3003 |
3004 |5: // Need to resize array part.
3005 | str BASE, L->base
3006 | mov CARG1, L
3007 | str PC, SAVE_PC
3008 | bl extern lj_tab_reasize // (lua_State *L, GCtab *t, int nasize)
3009 | // Must not reallocate the stack.
3010 | b <1
3011 |
3012 |7: // Possible table write barrier for any value. Skip valiswhite check.
3013 | barrierback TAB:CARG2, TMP2w, TMP1
3014 | b <4
3015 break;
3017 /* -- Calls and vararg handling ----------------------------------------- */
3019 case BC_CALLM:
3020 | // RA = base, (RB = nresults+1,) RC = extra_nargs
3021 | ldr TMP0w, SAVE_MULTRES
3022 | decode_RC8RD NARGS8:RC, RC
3023 | add NARGS8:RC, NARGS8:RC, TMP0
3024 | b ->BC_CALL_Z
3025 break;
3026 case BC_CALL:
3027 | decode_RC8RD NARGS8:RC, RC
3028 | // RA = base, (RB = nresults+1,) RC = (nargs+1)*8
3029 |->BC_CALL_Z:
3030 | mov RB, BASE // Save old BASE for vmeta_call.
3031 | add BASE, BASE, RA, lsl #3
3032 | ldr CARG3, [BASE]
3033 | sub NARGS8:RC, NARGS8:RC, #8
3034 | add BASE, BASE, #16
3035 | checkfunc CARG3, ->vmeta_call
3036 | ins_call
3037 break;
3039 case BC_CALLMT:
3040 | // RA = base, (RB = 0,) RC = extra_nargs
3041 | ldr TMP0w, SAVE_MULTRES
3042 | add NARGS8:RC, TMP0, RC, lsl #3
3043 | b ->BC_CALLT1_Z
3044 break;
3045 case BC_CALLT:
3046 | lsl NARGS8:RC, RC, #3
3047 | // RA = base, (RB = 0,) RC = (nargs+1)*8
3048 |->BC_CALLT1_Z:
3049 | add RA, BASE, RA, lsl #3
3050 | ldr TMP1, [RA]
3051 | sub NARGS8:RC, NARGS8:RC, #8
3052 | add RA, RA, #16
3053 | checktp CARG3, TMP1, LJ_TFUNC, ->vmeta_callt
3054 | ldr PC, [BASE, FRAME_PC]
3055 |->BC_CALLT2_Z:
3056 | mov RB, #0
3057 | ldrb TMP2w, LFUNC:CARG3->ffid
3058 | tst PC, #FRAME_TYPE
3059 | bne >7
3060 |1:
3061 | str TMP1, [BASE, FRAME_FUNC] // Copy function down, but keep PC.
3062 | cbz NARGS8:RC, >3
3063 |2:
3064 | ldr TMP0, [RA, RB]
3065 | add TMP1, RB, #8
3066 | cmp TMP1, NARGS8:RC
3067 | str TMP0, [BASE, RB]
3068 | mov RB, TMP1
3069 | bne <2
3070 |3:
3071 | cmp TMP2, #1 // (> FF_C) Calling a fast function?
3072 | bhi >5
3073 |4:
3074 | ins_callt
3075 |
3076 |5: // Tailcall to a fast function with a Lua frame below.
3077 | ldrb RAw, [PC, #-3]
3078 | sub CARG1, BASE, RA, lsl #3
3079 | ldr LFUNC:CARG1, [CARG1, #-32]
3080 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3081 | ldr CARG1, LFUNC:CARG1->pc
3082 | ldr KBASE, [CARG1, #PC2PROTO(k)]
3083 | b <4
3084 |
3085 |7: // Tailcall from a vararg function.
3086 | eor PC, PC, #FRAME_VARG
3087 | tst PC, #FRAME_TYPEP // Vararg frame below?
3088 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3089 | bne <1
3090 | sub BASE, BASE, PC
3091 | ldr PC, [BASE, FRAME_PC]
3092 | tst PC, #FRAME_TYPE
3093 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3094 | b <1
3095 break;
3097 case BC_ITERC:
3098 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3099 | add RA, BASE, RA, lsl #3
3100 | ldr CARG3, [RA, #-24]
3101 | mov RB, BASE // Save old BASE for vmeta_call.
3102 | ldp CARG1, CARG2, [RA, #-16]
3103 | add BASE, RA, #16
3104 | mov NARGS8:RC, #16 // Iterators get 2 arguments.
3105 | str CARG3, [RA] // Copy callable.
3106 | stp CARG1, CARG2, [RA, #16] // Copy state and control var.
3107 | checkfunc CARG3, ->vmeta_call
3108 | ins_call
3109 break;
3111 case BC_ITERN:
3112 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3113 |.if JIT
3114 | // NYI: add hotloop, record BC_ITERN.
3115 |.endif
3116 | add RA, BASE, RA, lsl #3
3117 | ldr TAB:RB, [RA, #-16]
3118 | ldrh TMP3w, [PC, #2]
3119 | ldr CARG1w, [RA, #-8] // Get index from control var.
3120 | add PC, PC, #4
3121 | add TMP3, PC, TMP3, lsl #2
3122 | and TAB:RB, RB, #LJ_GCVMASK
3123 | sub TMP3, TMP3, #0x20000
3124 | ldr TMP1w, TAB:RB->asize
3125 | ldr CARG2, TAB:RB->array
3126 |1: // Traverse array part.
3127 | subs RC, CARG1, TMP1
3128 | add CARG3, CARG2, CARG1, lsl #3
3129 | bhs >5 // Index points after array part?
3130 | ldr TMP0, [CARG3]
3131 | cmp TMP0, TISNIL
3132 | cinc CARG1, CARG1, eq // Skip holes in array part.
3133 | beq <1
3134 | add CARG1, CARG1, TISNUM
3135 | stp CARG1, TMP0, [RA]
3136 | add CARG1, CARG1, #1
3137 |3:
3138 | str CARG1w, [RA, #-8] // Update control var.
3139 | mov PC, TMP3
3140 |4:
3141 | ins_next
3142 |
3143 |5: // Traverse hash part.
3144 | ldr TMP2w, TAB:RB->hmask
3145 | ldr NODE:RB, TAB:RB->node
3146 |6:
3147 | add CARG1, RC, RC, lsl #1
3148 | cmp RC, TMP2 // End of iteration? Branch to ITERN+1.
3149 | add NODE:CARG3, NODE:RB, CARG1, lsl #3 // node = tab->node + idx*3*8
3150 | bhi <4
3151 | ldp TMP0, CARG1, NODE:CARG3->val
3152 | cmp TMP0, TISNIL
3153 | add RC, RC, #1
3154 | beq <6 // Skip holes in hash part.
3155 | stp CARG1, TMP0, [RA]
3156 | add CARG1, RC, TMP1
3157 | b <3
3158 break;
3160 case BC_ISNEXT:
3161 | // RA = base, RC = target (points to ITERN)
3162 | add RA, BASE, RA, lsl #3
3163 | ldr CFUNC:CARG1, [RA, #-24]
3164 | add RC, PC, RC, lsl #2
3165 | ldp TAB:CARG3, CARG4, [RA, #-16]
3166 | sub RC, RC, #0x20000
3167 | checkfunc CFUNC:CARG1, >5
3168 | asr TMP0, TAB:CARG3, #47
3169 | ldrb TMP1w, CFUNC:CARG1->ffid
3170 | cmn TMP0, #-LJ_TTAB
3171 | ccmp CARG4, TISNIL, #0, eq
3172 | ccmp TMP1w, #FF_next_N, #0, eq
3173 | bne >5
3174 | mov TMP0w, #0xfffe7fff
3175 | lsl TMP0, TMP0, #32
3176 | str TMP0, [RA, #-8] // Initialize control var.
3177 |1:
3178 | mov PC, RC
3179 | ins_next
3180 |
3181 |5: // Despecialize bytecode if any of the checks fail.
3182 | mov TMP0, #BC_JMP
3183 | mov TMP1, #BC_ITERC
3184 | strb TMP0w, [PC, #-4]
3185 | strb TMP1w, [RC]
3186 | b <1
3187 break;
3189 case BC_VARG:
3190 | decode_RB RB, INS
3191 | and RC, RC, #255
3192 | // RA = base, RB = (nresults+1), RC = numparams
3193 | ldr TMP1, [BASE, FRAME_PC]
3194 | add RC, BASE, RC, lsl #3
3195 | add RA, BASE, RA, lsl #3
3196 | add RC, RC, #FRAME_VARG
3197 | add TMP2, RA, RB, lsl #3
3198 | sub RC, RC, TMP1 // RC = vbase
3199 | // Note: RC may now be even _above_ BASE if nargs was < numparams.
3200 | sub TMP3, BASE, #16 // TMP3 = vtop
3201 | cbz RB, >5
3202 | sub TMP2, TMP2, #16
3203 |1: // Copy vararg slots to destination slots.
3204 | cmp RC, TMP3
3205 | ldr TMP0, [RC], #8
3206 | csel TMP0, TMP0, TISNIL, lo
3207 | cmp RA, TMP2
3208 | str TMP0, [RA], #8
3209 | blo <1
3210 |2:
3211 | ins_next
3212 |
3213 |5: // Copy all varargs.
3214 | ldr TMP0, L->maxstack
3215 | subs TMP2, TMP3, RC
3216 | csel RB, xzr, TMP2, le // MULTRES = (max(vtop-vbase,0)+1)*8
3217 | add RB, RB, #8
3218 | add TMP1, RA, TMP2
3219 | str RBw, SAVE_MULTRES
3220 | ble <2 // Nothing to copy.
3221 | cmp TMP1, TMP0
3222 | bhi >7
3223 |6:
3224 | ldr TMP0, [RC], #8
3225 | str TMP0, [RA], #8
3226 | cmp RC, TMP3
3227 | blo <6
3228 | b <2
3229 |
3230 |7: // Grow stack for varargs.
3231 | lsr CARG2, TMP2, #3
3232 | stp BASE, RA, L->base
3233 | mov CARG1, L
3234 | sub RC, RC, BASE // Need delta, because BASE may change.
3235 | str PC, SAVE_PC
3236 | bl extern lj_state_growstack // (lua_State *L, int n)
3237 | ldp BASE, RA, L->base
3238 | add RC, BASE, RC
3239 | sub TMP3, BASE, #16
3240 | b <6
3241 break;
3243 /* -- Returns ----------------------------------------------------------- */
3245 case BC_RETM:
3246 | // RA = results, RC = extra results
3247 | ldr TMP0w, SAVE_MULTRES
3248 | ldr PC, [BASE, FRAME_PC]
3249 | add RA, BASE, RA, lsl #3
3250 | add RC, TMP0, RC, lsl #3
3251 | b ->BC_RETM_Z
3252 break;
3254 case BC_RET:
3255 | // RA = results, RC = nresults+1
3256 | ldr PC, [BASE, FRAME_PC]
3257 | lsl RC, RC, #3
3258 | add RA, BASE, RA, lsl #3
3259 |->BC_RETM_Z:
3260 | str RCw, SAVE_MULTRES
3261 |1:
3262 | ands CARG1, PC, #FRAME_TYPE
3263 | eor CARG2, PC, #FRAME_VARG
3264 | bne ->BC_RETV2_Z
3265 |
3266 |->BC_RET_Z:
3267 | // BASE = base, RA = resultptr, RC = (nresults+1)*8, PC = return
3268 | ldr INSw, [PC, #-4]
3269 | subs TMP1, RC, #8
3270 | sub CARG3, BASE, #16
3271 | beq >3
3272 |2:
3273 | ldr TMP0, [RA], #8
3274 | add BASE, BASE, #8
3275 | sub TMP1, TMP1, #8
3276 | str TMP0, [BASE, #-24]
3277 | cbnz TMP1, <2
3278 |3:
3279 | decode_RA RA, INS
3280 | sub CARG4, CARG3, RA, lsl #3
3281 | decode_RB RB, INS
3282 | ldr LFUNC:CARG1, [CARG4, FRAME_FUNC]
3283 |5:
3284 | cmp RC, RB, lsl #3 // More results expected?
3285 | blo >6
3286 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3287 | mov BASE, CARG4
3288 | ldr CARG2, LFUNC:CARG1->pc
3289 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3290 | ins_next
3291 |
3292 |6: // Fill up results with nil.
3293 | add BASE, BASE, #8
3294 | add RC, RC, #8
3295 | str TISNIL, [BASE, #-24]
3296 | b <5
3297 |
3298 |->BC_RETV1_Z: // Non-standard return case.
3299 | add RA, BASE, RA, lsl #3
3300 |->BC_RETV2_Z:
3301 | tst CARG2, #FRAME_TYPEP
3302 | bne ->vm_return
3303 | // Return from vararg function: relocate BASE down.
3304 | sub BASE, BASE, CARG2
3305 | ldr PC, [BASE, FRAME_PC]
3306 | b <1
3307 break;
3309 case BC_RET0: case BC_RET1:
3310 | // RA = results, RC = nresults+1
3311 | ldr PC, [BASE, FRAME_PC]
3312 | lsl RC, RC, #3
3313 | str RCw, SAVE_MULTRES
3314 | ands CARG1, PC, #FRAME_TYPE
3315 | eor CARG2, PC, #FRAME_VARG
3316 | bne ->BC_RETV1_Z
3317 | ldr INSw, [PC, #-4]
3318 if (op == BC_RET1) {
3319 | ldr TMP0, [BASE, RA, lsl #3]
3320 }
3321 | sub CARG4, BASE, #16
3322 | decode_RA RA, INS
3323 | sub BASE, CARG4, RA, lsl #3
3324 if (op == BC_RET1) {
3325 | str TMP0, [CARG4], #8
3326 }
3327 | decode_RB RB, INS
3328 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3329 |5:
3330 | cmp RC, RB, lsl #3
3331 | blo >6
3332 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3333 | ldr CARG2, LFUNC:CARG1->pc
3334 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3335 | ins_next
3336 |
3337 |6: // Fill up results with nil.
3338 | add RC, RC, #8
3339 | str TISNIL, [CARG4], #8
3340 | b <5
3341 break;
3343 /* -- Loops and branches ------------------------------------------------ */
3345 |.define FOR_IDX, [RA]; .define FOR_TIDX, [RA, #4]
3346 |.define FOR_STOP, [RA, #8]; .define FOR_TSTOP, [RA, #12]
3347 |.define FOR_STEP, [RA, #16]; .define FOR_TSTEP, [RA, #20]
3348 |.define FOR_EXT, [RA, #24]; .define FOR_TEXT, [RA, #28]
3350 case BC_FORL:
3351 |.if JIT
3352 | hotloop
3353 |.endif
3354 | // Fall through. Assumes BC_IFORL follows.
3355 break;
3357 case BC_JFORI:
3358 case BC_JFORL:
3359 #if !LJ_HASJIT
3360 break;
3361 #endif
3362 case BC_FORI:
3363 case BC_IFORL:
3364 | // RA = base, RC = target (after end of loop or start of loop)
3365 vk = (op == BC_IFORL || op == BC_JFORL);
3366 | add RA, BASE, RA, lsl #3
3367 | ldp CARG1, CARG2, FOR_IDX // CARG1 = IDX, CARG2 = STOP
3368 | ldr CARG3, FOR_STEP // CARG3 = STEP
3369 if (op != BC_JFORL) {
3370 | add RC, PC, RC, lsl #2
3371 | sub RC, RC, #0x20000
3372 }
3373 | checkint CARG1, >5
3374 if (!vk) {
3375 | checkint CARG2, ->vmeta_for
3376 | checkint CARG3, ->vmeta_for
3377 | tbnz CARG3w, #31, >4
3378 | cmp CARG1w, CARG2w
3379 } else {
3380 | adds CARG1w, CARG1w, CARG3w
3381 | bvs >2
3382 | add TMP0, CARG1, TISNUM
3383 | tbnz CARG3w, #31, >4
3384 | cmp CARG1w, CARG2w
3385 }
3386 |1:
3387 if (op == BC_FORI) {
3388 | csel PC, RC, PC, gt
3389 } else if (op == BC_JFORI) {
3390 | ldrh RCw, [RC, #-2]
3391 } else if (op == BC_IFORL) {
3392 | csel PC, RC, PC, le
3393 }
3394 if (vk) {
3395 | str TMP0, FOR_IDX
3396 | str TMP0, FOR_EXT
3397 } else {
3398 | str CARG1, FOR_EXT
3399 }
3400 if (op == BC_JFORI || op == BC_JFORL) {
3401 | ble =>BC_JLOOP
3402 }
3403 |2:
3404 | ins_next
3405 |
3406 |4: // Invert check for negative step.
3407 | cmp CARG2w, CARG1w
3408 | b <1
3409 |
3410 |5: // FP loop.
3411 | ldp d0, d1, FOR_IDX
3412 | blo ->vmeta_for
3413 if (!vk) {
3414 | checknum CARG2, ->vmeta_for
3415 | checknum CARG3, ->vmeta_for
3416 | str d0, FOR_EXT
3417 } else {
3418 | ldr d2, FOR_STEP
3419 | fadd d0, d0, d2
3420 }
3421 | tbnz CARG3, #63, >7
3422 | fcmp d0, d1
3423 |6:
3424 if (vk) {
3425 | str d0, FOR_IDX
3426 | str d0, FOR_EXT
3427 }
3428 if (op == BC_FORI) {
3429 | csel PC, RC, PC, hi
3430 } else if (op == BC_JFORI) {
3431 | ldrh RCw, [RC, #-2]
3432 | bls =>BC_JLOOP
3433 } else if (op == BC_IFORL) {
3434 | csel PC, RC, PC, ls
3435 } else {
3436 | bls =>BC_JLOOP
3437 }
3438 | b <2
3439 |
3440 |7: // Invert check for negative step.
3441 | fcmp d1, d0
3442 | b <6
3443 break;
3445 case BC_ITERL:
3446 |.if JIT
3447 | hotloop
3448 |.endif
3449 | // Fall through. Assumes BC_IITERL follows.
3450 break;
3452 case BC_JITERL:
3453 #if !LJ_HASJIT
3454 break;
3455 #endif
3456 case BC_IITERL:
3457 | // RA = base, RC = target
3458 | ldr CARG1, [BASE, RA, lsl #3]
3459 | add TMP1, BASE, RA, lsl #3
3460 | cmp CARG1, TISNIL
3461 | beq >1 // Stop if iterator returned nil.
3462 if (op == BC_JITERL) {
3463 | str CARG1, [TMP1, #-8]
3464 | b =>BC_JLOOP
3465 } else {
3466 | add TMP0, PC, RC, lsl #2 // Otherwise save control var + branch.
3467 | sub PC, TMP0, #0x20000
3468 | str CARG1, [TMP1, #-8]
3469 }
3470 |1:
3471 | ins_next
3472 break;
3474 case BC_LOOP:
3475 | // RA = base, RC = target (loop extent)
3476 | // Note: RA/RC is only used by trace recorder to determine scope/extent
3477 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
3478 |.if JIT
3479 | hotloop
3480 |.endif
3481 | // Fall through. Assumes BC_ILOOP follows.
3482 break;
3484 case BC_ILOOP:
3485 | // RA = base, RC = target (loop extent)
3486 | ins_next
3487 break;
3489 case BC_JLOOP:
3490 |.if JIT
3491 | NYI
3492 |.endif
3493 break;
3495 case BC_JMP:
3496 | // RA = base (only used by trace recorder), RC = target
3497 | add RC, PC, RC, lsl #2
3498 | sub PC, RC, #0x20000
3499 | ins_next
3500 break;
3502 /* -- Function headers -------------------------------------------------- */
3504 case BC_FUNCF:
3505 |.if JIT
3506 | hotcall
3507 |.endif
3508 case BC_FUNCV: /* NYI: compiled vararg functions. */
3509 | // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
3510 break;
3512 case BC_JFUNCF:
3513 #if !LJ_HASJIT
3514 break;
3515 #endif
3516 case BC_IFUNCF:
3517 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3518 | ldr CARG1, L->maxstack
3519 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3520 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3521 | cmp RA, CARG1
3522 | bhi ->vm_growstack_l
3523 |2:
3524 | cmp NARGS8:RC, TMP1, lsl #3 // Check for missing parameters.
3525 | blo >3
3526 if (op == BC_JFUNCF) {
3527 | decode_RD RC, INS
3528 | b =>BC_JLOOP
3529 } else {
3530 | ins_next
3531 }
3532 |
3533 |3: // Clear missing parameters.
3534 | str TISNIL, [BASE, NARGS8:RC]
3535 | add NARGS8:RC, NARGS8:RC, #8
3536 | b <2
3537 break;
3539 case BC_JFUNCV:
3540 #if !LJ_HASJIT
3541 break;
3542 #endif
3543 | NYI // NYI: compiled vararg functions
3544 break; /* NYI: compiled vararg functions. */
3546 case BC_IFUNCV:
3547 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3548 | ldr CARG1, L->maxstack
3549 | add TMP2, BASE, RC
3550 | add RA, RA, RC
3551 | add TMP0, RC, #16+FRAME_VARG
3552 | str LFUNC:CARG3, [TMP2], #8 // Store (untagged) copy of LFUNC.
3553 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3554 | cmp RA, CARG1
3555 | str TMP0, [TMP2], #8 // Store delta + FRAME_VARG.
3556 | bhs ->vm_growstack_l
3557 | sub RC, TMP2, #16
3558 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3559 | mov RA, BASE
3560 | mov BASE, TMP2
3561 | cbz TMP1, >2
3562 |1:
3563 | cmp RA, RC // Less args than parameters?
3564 | bhs >3
3565 | ldr TMP0, [RA]
3566 | sub TMP1, TMP1, #1
3567 | str TISNIL, [RA], #8 // Clear old fixarg slot (help the GC).
3568 | str TMP0, [TMP2], #8
3569 | cbnz TMP1, <1
3570 |2:
3571 | ins_next
3572 |
3573 |3:
3574 | sub TMP1, TMP1, #1
3575 | str TISNIL, [TMP2], #8
3576 | cbz TMP1, <2
3577 | b <3
3578 break;
3580 case BC_FUNCC:
3581 case BC_FUNCCW:
3582 | // BASE = new base, RA = BASE+framesize*8, CARG3 = CFUNC, RC = nargs*8
3583 if (op == BC_FUNCC) {
3584 | ldr CARG4, CFUNC:CARG3->f
3585 } else {
3586 | ldr CARG4, GL->wrapf
3587 }
3588 | add CARG2, RA, NARGS8:RC
3589 | ldr CARG1, L->maxstack
3590 | add RC, BASE, NARGS8:RC
3591 | cmp CARG2, CARG1
3592 | stp BASE, RC, L->base
3593 if (op == BC_FUNCCW) {
3594 | ldr CARG2, CFUNC:CARG3->f
3595 }
3596 | mv_vmstate TMP0w, C
3597 | mov CARG1, L
3598 | bhi ->vm_growstack_c // Need to grow stack.
3599 | st_vmstate TMP0w
3600 | blr CARG4 // (lua_State *L [, lua_CFunction f])
3601 | // Returns nresults.
3602 | ldp BASE, TMP1, L->base
3603 | str L, GL->cur_L
3604 | sbfiz RC, CRET1, #3, #32
3605 | st_vmstate ST_INTERP
3606 | ldr PC, [BASE, FRAME_PC]
3607 | sub RA, TMP1, RC // RA = L->top - nresults*8
3608 | b ->vm_returnc
3609 break;
3611 /* ---------------------------------------------------------------------- */
3613 default:
3614 fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
3615 exit(2);
3616 break;
3617 }
3618 }
3620 static int build_backend(BuildCtx *ctx)
3621 {
3622 int op;
3624 dasm_growpc(Dst, BC__MAX);
3626 build_subroutines(ctx);
3628 |.code_op
3629 for (op = 0; op < BC__MAX; op++)
3630 build_ins(ctx, (BCOp)op, op);
3632 return BC__MAX;
3633 }
3635 /* Emit pseudo frame-info for all assembler functions. */
3636 static void emit_asm_debug(BuildCtx *ctx)
3637 {
3638 int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
3639 int i, cf = CFRAME_SIZE >> 3;
3640 switch (ctx->mode) {
3641 case BUILD_elfasm:
3642 fprintf(ctx->fp, "\t.section .debug_frame,\"\",%%progbits\n");
3643 fprintf(ctx->fp,
3644 ".Lframe0:\n"
3645 "\t.long .LECIE0-.LSCIE0\n"
3646 ".LSCIE0:\n"
3647 "\t.long 0xffffffff\n"
3648 "\t.byte 0x1\n"
3649 "\t.string \"\"\n"
3650 "\t.uleb128 0x1\n"
3651 "\t.sleb128 -8\n"
3652 "\t.byte 30\n" /* Return address is in lr. */
3653 "\t.byte 0xc\n\t.uleb128 31\n\t.uleb128 0\n" /* def_cfa sp */
3654 "\t.align 3\n"
3655 ".LECIE0:\n\n");
3656 fprintf(ctx->fp,
3657 ".LSFDE0:\n"
3658 "\t.long .LEFDE0-.LASFDE0\n"
3659 ".LASFDE0:\n"
3660 "\t.long .Lframe0\n"
3661 "\t.quad .Lbegin\n"
3662 "\t.quad %d\n"
3663 "\t.byte 0xe\n\t.uleb128 %d\n" /* def_cfa_offset */
3664 "\t.byte 0x9d\n\t.uleb128 %d\n" /* offset fp */
3665 "\t.byte 0x9e\n\t.uleb128 %d\n", /* offset lr */
3666 fcofs, CFRAME_SIZE, cf, cf-1);
3667 for (i = 19; i <= 28; i++) /* offset x19-x28 */
3668 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, cf-i+17);
3669 for (i = 8; i <= 15; i++) /* offset d8-d15 */
3670 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
3671 64+i, cf-i-4);
3672 fprintf(ctx->fp,
3673 "\t.align 3\n"
3674 ".LEFDE0:\n\n");
3675 #if LJ_HASFFI
3676 fprintf(ctx->fp,
3677 ".LSFDE1:\n"
3678 "\t.long .LEFDE1-.LASFDE1\n"
3679 ".LASFDE1:\n"
3680 "\t.long .Lframe0\n"
3681 "\t.quad lj_vm_ffi_call\n"
3682 "\t.quad %d\n"
3683 "\t.byte 0xe\n\t.uleb128 32\n" /* def_cfa_offset */
3684 "\t.byte 0x9d\n\t.uleb128 4\n" /* offset fp */
3685 "\t.byte 0x9e\n\t.uleb128 3\n" /* offset lr */
3686 "\t.byte 0x93\n\t.uleb128 2\n" /* offset x19 */
3687 "\t.align 3\n"
3688 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
3689 #endif
3690 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",%%progbits\n");
3691 fprintf(ctx->fp,
3692 ".Lframe1:\n"
3693 "\t.long .LECIE1-.LSCIE1\n"
3694 ".LSCIE1:\n"
3695 "\t.long 0\n"
3696 "\t.byte 0x1\n"
3697 "\t.string \"zPR\"\n"
3698 "\t.uleb128 0x1\n"
3699 "\t.sleb128 -8\n"
3700 "\t.byte 30\n" /* Return address is in lr. */
3701 "\t.uleb128 6\n" /* augmentation length */
3702 "\t.byte 0x1b\n" /* pcrel|sdata4 */
3703 "\t.long lj_err_unwind_dwarf-.\n"
3704 "\t.byte 0x1b\n" /* pcrel|sdata4 */
3705 "\t.byte 0xc\n\t.uleb128 31\n\t.uleb128 0\n" /* def_cfa sp */
3706 "\t.align 3\n"
3707 ".LECIE1:\n\n");
3708 fprintf(ctx->fp,
3709 ".LSFDE2:\n"
3710 "\t.long .LEFDE2-.LASFDE2\n"
3711 ".LASFDE2:\n"
3712 "\t.long .LASFDE2-.Lframe1\n"
3713 "\t.long .Lbegin-.\n"
3714 "\t.long %d\n"
3715 "\t.uleb128 0\n" /* augmentation length */
3716 "\t.byte 0xe\n\t.uleb128 %d\n" /* def_cfa_offset */
3717 "\t.byte 0x9d\n\t.uleb128 %d\n" /* offset fp */
3718 "\t.byte 0x9e\n\t.uleb128 %d\n", /* offset lr */
3719 fcofs, CFRAME_SIZE, cf, cf-1);
3720 for (i = 19; i <= 28; i++) /* offset x19-x28 */
3721 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, cf-i+17);
3722 for (i = 8; i <= 15; i++) /* offset d8-d15 */
3723 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
3724 64+i, cf-i-4);
3725 fprintf(ctx->fp,
3726 "\t.align 3\n"
3727 ".LEFDE2:\n\n");
3728 #if LJ_HASFFI
3729 fprintf(ctx->fp,
3730 ".Lframe2:\n"
3731 "\t.long .LECIE2-.LSCIE2\n"
3732 ".LSCIE2:\n"
3733 "\t.long 0\n"
3734 "\t.byte 0x1\n"
3735 "\t.string \"zR\"\n"
3736 "\t.uleb128 0x1\n"
3737 "\t.sleb128 -8\n"
3738 "\t.byte 30\n" /* Return address is in lr. */
3739 "\t.uleb128 1\n" /* augmentation length */
3740 "\t.byte 0x1b\n" /* pcrel|sdata4 */
3741 "\t.byte 0xc\n\t.uleb128 31\n\t.uleb128 0\n" /* def_cfa sp */
3742 "\t.align 3\n"
3743 ".LECIE2:\n\n");
3744 fprintf(ctx->fp,
3745 ".LSFDE3:\n"
3746 "\t.long .LEFDE3-.LASFDE3\n"
3747 ".LASFDE3:\n"
3748 "\t.long .LASFDE3-.Lframe2\n"
3749 "\t.long lj_vm_ffi_call-.\n"
3750 "\t.long %d\n"
3751 "\t.uleb128 0\n" /* augmentation length */
3752 "\t.byte 0xe\n\t.uleb128 32\n" /* def_cfa_offset */
3753 "\t.byte 0x9d\n\t.uleb128 4\n" /* offset fp */
3754 "\t.byte 0x9e\n\t.uleb128 3\n" /* offset lr */
3755 "\t.byte 0x93\n\t.uleb128 2\n" /* offset x19 */
3756 "\t.align 3\n"
3757 ".LEFDE3:\n\n", (int)ctx->codesz - fcofs);
3758 #endif
3759 break;
3760 default:
3761 break;
3762 }
3763 }