1 |// Low-level VM code for ARM64 CPUs.
2 |// Bytecode interpreter, fast functions and helper functions.
3 |// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
6 |.section code_op, code_sub
8 |.actionlist build_actionlist
10 |.globalnames globnames
11 |.externnames extnames
13 |// Note: The ragged indentation of the instructions is intentional.
14 |// The starting columns indicate data dependencies.
16 |//-----------------------------------------------------------------------
18 |// ARM64 registers and the AAPCS64 ABI 1.0 at a glance:
20 |// x0-x17 temp, x19-x28 callee-saved, x29 fp, x30 lr
21 |// x18 is reserved on most platforms. Don't use it, save it or restore it.
22 |// x31 doesn't exist. Register number 31 either means xzr/wzr (zero) or sp,
23 |// depending on the instruction.
24 |// v0-v7 temp, v8-v15 callee-saved (only d8-d15 preserved), v16-v31 temp
26 |// x0-x7/v0-v7 hold parameters and results.
28 |// Fixed register assignments for the interpreter.
30 |// The following must be C callee-save.
31 |.define BASE, x19 // Base of current Lua stack frame.
32 |.define KBASE, x20 // Constants of current Lua function.
33 |.define PC, x21 // Next PC.
34 |.define GLREG, x22 // Global state.
35 |.define LREG, x23 // Register holding lua_State (also in SAVE_L).
36 |.define TISNUM, x24 // Constant LJ_TISNUM << 47.
37 |.define TISNUMhi, x25 // Constant LJ_TISNUM << 15.
38 |.define TISNIL, x26 // Constant -1LL.
39 |.define fp, x29 // Yes, we have to maintain a frame pointer.
41 |.define ST_INTERP, w26 // Constant -1.
43 |// The following temporaries are not saved across C calls, except for RA/RC.
62 |// Calling conventions. Also used as temporaries.
80 |//-----------------------------------------------------------------------
82 |// ARM64e pointer authentication codes (PAC).
84 |.macro sp_auth; pacibsp; .endmacro
85 |.macro br_auth, reg; braaz reg; .endmacro
86 |.macro blr_auth, reg; blraaz reg; .endmacro
87 |.macro ret_auth; retab; .endmacro
89 |.macro sp_auth; .endmacro
90 |.macro br_auth, reg; br reg; .endmacro
91 |.macro blr_auth, reg; blr reg; .endmacro
92 |.macro ret_auth; ret; .endmacro
95 |//-----------------------------------------------------------------------
97 |// Stack layout while in interpreter. Must match with lj_frame.h.
99 |.define CFRAME_SPACE, 208
100 |//----- 16 byte aligned, <-- sp entering interpreter
101 |.define SAVE_FP_LR_, 192
102 |.define SAVE_GPR_, 112 // 112+10*8: 64 bit GPR saves
103 |.define SAVE_FPR_, 48 // 48+8*8: 64 bit FPR saves
104 |// Unused [sp, #44] // 32 bit values
105 |.define SAVE_NRES, [sp, #40]
106 |.define SAVE_ERRF, [sp, #36]
107 |.define SAVE_MULTRES, [sp, #32]
108 |.define TMPD, [sp, #24] // 64 bit values
109 |.define SAVE_L, [sp, #16]
110 |.define SAVE_PC, [sp, #8]
111 |.define SAVE_CFRAME, [sp, #0]
112 |//----- 16 byte aligned, <-- sp while in interpreter.
114 |.define TMPDofs, #24
117 |// Windows unwind data is suited to r1 stored first.
118 |.macro stp_unwind, r1, r2, where
121 |.macro ldp_unwind, r1, r2, where
124 |.macro ldp_unwind, r1, r2, where, post_index
125 | ldp r1, r2, where, post_index
128 |// Otherwise store r2 first for compact unwind info (OSX).
129 |.macro stp_unwind, r1, r2, where
132 |.macro ldp_unwind, r1, r2, where
135 |.macro ldp_unwind, r1, r2, where, post_index
136 | ldp r2, r1, where, post_index
140 |.macro save_, gpr1, gpr2, fpr1, fpr2
141 | stp_unwind d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(14-fpr1)*8]
142 | stp_unwind x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(27-gpr1)*8]
144 |.macro rest_, gpr1, gpr2, fpr1, fpr2
145 | ldp_unwind d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(14-fpr1)*8]
146 | ldp_unwind x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(27-gpr1)*8]
151 | sub sp, sp, # CFRAME_SPACE
152 | stp fp, lr, [sp, # SAVE_FP_LR_]
153 | add fp, sp, # SAVE_FP_LR_
154 | stp_unwind x19, x20, [sp, # SAVE_GPR_+(27-19)*8]
156 | save_ 23, 24, 10, 11
157 | save_ 25, 26, 12, 13
158 | save_ 27, 28, 14, 15
161 | ldp_unwind x19, x20, [sp, # SAVE_GPR_+(27-19)*8]
163 | rest_ 23, 24, 10, 11
164 | rest_ 25, 26, 12, 13
165 | rest_ 27, 28, 14, 15
166 | ldp fp, lr, [sp, # SAVE_FP_LR_]
167 | add sp, sp, # CFRAME_SPACE
170 |// Type definitions. Some of these are only used for documentation.
171 |.type L, lua_State, LREG
172 |.type GL, global_State, GLREG
173 |.type TVALUE, TValue
177 |.type LFUNC, GCfuncL
178 |.type CFUNC, GCfuncC
179 |.type PROTO, GCproto
180 |.type UPVAL, GCupval
183 |.type TRACE, GCtrace
186 |//-----------------------------------------------------------------------
188 |// Trap for not-yet-implemented parts.
189 |.macro NYI; brk; .endmacro
191 |//-----------------------------------------------------------------------
193 |// Access to frame relative to BASE.
194 |.define FRAME_FUNC, #-16
195 |.define FRAME_PC, #-8
197 |// Endian-specific defines.
212 |.macro decode_RA, dst, ins; ubfx dst, ins, #8, #8; .endmacro
213 |.macro decode_RB, dst, ins; ubfx dst, ins, #24, #8; .endmacro
214 |.macro decode_RC, dst, ins; ubfx dst, ins, #16, #8; .endmacro
215 |.macro decode_RD, dst, ins; ubfx dst, ins, #16, #16; .endmacro
216 |.macro decode_RC8RD, dst, src; ubfiz dst, src, #3, #8; .endmacro
218 |// Instruction decode+dispatch.
221 | add TMP1, GL, INS, uxtb #3
223 | ldr TMP0, [TMP1, #GG_G2DISP]
228 |// Instruction footer.
230 | // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
231 | .define ins_next, ins_NEXT
232 | .define ins_next_, ins_NEXT
234 | // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
235 | // Affects only certain kinds of benchmarks (and only with -j off).
245 |// Call decode and dispatch.
247 | // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
248 | ldr PC, LFUNC:CARG3->pc
250 | add TMP1, GL, INS, uxtb #3
252 | ldr TMP0, [TMP1, #GG_G2DISP]
253 | add RA, BASE, RA, lsl #3
258 | // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
259 | str PC, [BASE, FRAME_PC]
263 |//-----------------------------------------------------------------------
265 |// Macros to check the TValue type and extract the GCobj. Branch on failure.
266 |.macro checktp, reg, tp, target
267 | asr ITYPE, reg, #47
269 | and reg, reg, #LJ_GCVMASK
272 |.macro checktp, dst, reg, tp, target
273 | asr ITYPE, reg, #47
275 | and dst, reg, #LJ_GCVMASK
278 |.macro checkstr, reg, target; checktp reg, LJ_TSTR, target; .endmacro
279 |.macro checktab, reg, target; checktp reg, LJ_TTAB, target; .endmacro
280 |.macro checkfunc, reg, target; checktp reg, LJ_TFUNC, target; .endmacro
281 |.macro checkint, reg, target
282 | cmp TISNUMhi, reg, lsr #32
285 |.macro checknum, reg, target
286 | cmp TISNUMhi, reg, lsr #32
289 |.macro checknumber, reg, target
290 | cmp TISNUMhi, reg, lsr #32
294 |.macro init_constants
296 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
297 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
300 |.macro mov_false, reg; movn reg, #0x8000, lsl #32; .endmacro
301 |.macro mov_true, reg; movn reg, #0x0001, lsl #48; .endmacro
302 |.macro mov_nil, reg; mov reg, TISNIL; .endmacro
303 |.macro cmp_nil, reg; cmp reg, TISNIL; .endmacro
304 |.macro add_TISNUM, dst, src; add dst, src, TISNUM; .endmacro
306 #define GL_J(field) (GG_G2J + (int)offsetof(jit_State, field))
308 #define PC2PROTO(field) ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
310 |.macro hotcheck, delta
312 | and CARG1, CARG1, #126
313 | add CARG1, CARG1, #GG_G2DISP+GG_DISP2HOT
314 | ldrh CARG2w, [GL, CARG1]
315 | subs CARG2, CARG2, #delta
316 | strh CARG2w, [GL, CARG1]
320 | hotcheck HOTCOUNT_LOOP
325 | hotcheck HOTCOUNT_CALL
329 |// Set current VM state.
330 |.macro mv_vmstate, reg, st; movn reg, #LJ_VMST_..st; .endmacro
331 |.macro st_vmstate, reg; str reg, GL->vmstate; .endmacro
333 |// Move table write barrier back. Overwrites mark and tmp.
334 |.macro barrierback, tab, mark, tmp
335 | ldr tmp, GL->gc.grayagain
336 | and mark, mark, #~LJ_GC_BLACK // black2gray(tab)
337 | str tab, GL->gc.grayagain
338 | strb mark, tab->marked
339 | str tmp, tab->gclist
342 |//-----------------------------------------------------------------------
345 #error "Only dual-number mode supported for ARM64 target"
348 /* Generate subroutines used by opcodes and other parts of the VM. */
349 /* The .code_sub section should be last to help static branch prediction. */
350 static void build_subroutines(BuildCtx *ctx)
354 |//-----------------------------------------------------------------------
355 |//-- Return handling ----------------------------------------------------
356 |//-----------------------------------------------------------------------
359 | // See vm_return. Also: RB = previous base.
360 | tbz PC, #2, ->cont_dispatch // (PC & FRAME_P) == 0?
362 | // Return from pcall or xpcall fast func.
363 | ldr PC, [RB, FRAME_PC] // Fetch PC of previous frame.
366 | // Prepending may overwrite the pcall frame, so do it at the end.
367 | str TMP0, [RA, #-8]! // Prepend true to results.
370 | adds RC, RC, #8 // RC = (nresults+1)*8.
371 | mov CRET1, #LUA_YIELD
372 | beq ->vm_unwind_c_eh
373 | str RCw, SAVE_MULTRES
374 | ands CARG1, PC, #FRAME_TYPE
375 | beq ->BC_RET_Z // Handle regular return to Lua.
378 | // BASE = base, RA = resultptr, RC/MULTRES = (nresults+1)*8, PC = return
379 | // CARG1 = PC & FRAME_TYPE
380 | and RB, PC, #~FRAME_TYPEP
381 | cmp CARG1, #FRAME_C
382 | sub RB, BASE, RB // RB = previous base.
386 | ldrsw CARG2, SAVE_NRES // CARG2 = nresults+1.
387 | mv_vmstate TMP0w, C
388 | sub BASE, BASE, #16
393 | subs TMP2, TMP2, #8
395 | str TMP0, [BASE], #8
398 | cmp RC, CARG2, lsl #3 // More/less results wanted?
401 | str BASE, L->top // Store new top.
404 | ldr RC, SAVE_CFRAME // Restore previous C frame.
405 | mov CRET1, #0 // Ok return status for vm_pcall.
413 | bgt >7 // Less results wanted?
414 | // More results wanted. Check stack size and fill up results with nil.
415 | ldr CARG3, L->maxstack
418 | str TISNIL, [BASE], #8
422 |7: // Less results wanted.
423 | cbz CARG2, <3 // LUA_MULTRET+1 case?
424 | sub CARG1, RC, CARG2, lsl #3
425 | sub BASE, BASE, CARG1 // Shrink top.
428 |8: // Corner case: need to grow stack for filling up results.
429 | // This can happen if:
430 | // - A C function grows the stack (a lot).
431 | // - The GC shrinks the stack in between.
432 | // - A return back from a lua_call() with (high) nresults adjustment.
433 | str BASE, L->top // Save current top held in BASE (yes).
435 | bl extern lj_state_growstack // (lua_State *L, int n)
436 | ldr BASE, L->top // Need the (realloced) L->top in BASE.
437 | ldrsw CARG2, SAVE_NRES
440 |->vm_unwind_c: // Unwind C stack, return from vm_pcall.
441 | // (void *cframe, int errcode)
442 | add fp, CARG1, # SAVE_FP_LR_
447 |->vm_unwind_c_eh: // Landing pad for external unwinder.
448 | mv_vmstate TMP0w, C
452 |->vm_unwind_ff: // Unwind C stack, return from ff pcall.
454 | add fp, CARG1, # SAVE_FP_LR_
458 | ldr GL, L->glref // Setup pointer to global state.
459 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
460 | mov RC, #16 // 2 results: false + error message.
463 | sub RA, BASE, #8 // Results start at BASE-8.
464 | ldr PC, [BASE, FRAME_PC] // Fetch PC of previous frame.
465 | str TMP0, [BASE, #-8] // Prepend false to error message.
466 | st_vmstate ST_INTERP
469 |//-----------------------------------------------------------------------
470 |//-- Grow stack for calls -----------------------------------------------
471 |//-----------------------------------------------------------------------
473 |->vm_growstack_c: // Grow stack for C function.
475 | mov CARG2, #LUA_MINSTACK
478 |->vm_growstack_l: // Grow stack for Lua function.
479 | // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
483 | stp BASE, RC, L->base
484 | add PC, PC, #4 // Must point after first instruction.
487 | // L->base = new base, L->top = top
489 | bl extern lj_state_growstack // (lua_State *L, int n)
490 | ldp BASE, RC, L->base
491 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
492 | sub NARGS8:RC, RC, BASE
493 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
494 | // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
495 | ins_callt // Just retry the call.
497 |//-----------------------------------------------------------------------
498 |//-- Entry points into the assembler VM ---------------------------------
499 |//-----------------------------------------------------------------------
501 |->vm_resume: // Setup C frame and resume thread.
502 | // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
505 | ldr GL, L->glref // Setup pointer to global state.
510 | add TMP0, sp, #CFRAME_RESUME
511 | ldrb TMP1w, L->status
513 | str L, SAVE_PC // Any value outside of bytecode is ok.
514 | str xzr, SAVE_CFRAME
515 | str TMP0, L->cframe
518 | // Resume after yield (like a return).
521 | ldp BASE, CARG1, L->base
523 | ldr PC, [BASE, FRAME_PC]
524 | strb wzr, L->status
525 | sub RC, CARG1, BASE
526 | ands CARG1, PC, #FRAME_TYPE
528 | st_vmstate ST_INTERP
529 | str RCw, SAVE_MULTRES
533 |->vm_pcall: // Setup protected C frame and enter VM.
534 | // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
537 | str CARG4w, SAVE_ERRF
540 |->vm_call: // Setup C frame and enter VM.
541 | // (lua_State *L, TValue *base, int nres1)
545 |1: // Entry point for vm_pcall above (PC = ftype).
546 | ldr RC, L:CARG1->cframe
547 | str CARG3w, SAVE_NRES
550 | ldr GL, L->glref // Setup pointer to global state.
552 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
554 | str RC, SAVE_CFRAME
555 | str TMP0, L->cframe // Add our C frame to cframe chain.
557 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
559 | ldp RB, CARG1, L->base // RB = old base (for vmeta_call).
562 | sub PC, PC, RB // PC = frame delta + frame type
563 | sub NARGS8:RC, CARG1, BASE
564 | st_vmstate ST_INTERP
567 | // RB = old base, BASE = new base, RC = nargs*8, PC = caller PC
568 | ldr CARG3, [BASE, FRAME_FUNC]
569 | checkfunc CARG3, ->vmeta_call
571 |->vm_call_dispatch_f:
573 | // BASE = new base, CARG3 = func, RC = nargs*8, PC = caller PC
575 |->vm_cpcall: // Setup protected C frame, call C.
576 | // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
579 | ldr RA, L:CARG1->stack
581 | ldr GL, L->glref // Setup pointer to global state.
583 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
585 | sub RA, RA, RB // Compute -savestack(L, L->top).
586 | str RAw, SAVE_NRES // Neg. delta means cframe w/o frame.
587 | str wzr, SAVE_ERRF // No error function.
589 | str RC, SAVE_CFRAME
590 | str TMP0, L->cframe // Add our C frame to cframe chain.
592 | blr_auth CARG4 // (lua_State *L, lua_CFunction func, void *ud)
595 | cbnz BASE, <3 // Else continue with the call.
596 | b ->vm_leave_cp // No base? Just remove C frame.
598 |//-----------------------------------------------------------------------
599 |//-- Metamethod handling ------------------------------------------------
600 |//-----------------------------------------------------------------------
602 |//-- Continuation dispatch ----------------------------------------------
605 | // BASE = meta base, RA = resultptr, RC = (nresults+1)*8
606 | ldr LFUNC:CARG3, [RB, FRAME_FUNC]
607 | ldr CARG1, [BASE, #-32] // Get continuation.
609 | mov BASE, RB // Restore caller BASE.
610 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
614 | ldr PC, [CARG4, #-24] // Restore PC from [cont|PC].
616 | str TISNIL, [TMP0, #-8] // Ensure one valid arg.
620 | ldr CARG3, LFUNC:CARG3->pc
621 | ldr KBASE, [CARG3, #PC2PROTO(k)]
622 | // BASE = base, RA = resultptr, CARG4 = meta base
627 | beq ->cont_ffi_callback // cont = 1: return from FFI callback.
628 | // cont = 0: tailcall from C function.
629 | sub CARG4, CARG4, #32
630 | sub RC, CARG4, BASE
634 |->cont_cat: // RA = resultptr, CARG4 = meta base
635 | ldr INSw, [PC, #-4]
636 | sub CARG2, CARG4, #32
641 | add TMP1, BASE, RB, lsl #3
642 | subs TMP1, CARG2, TMP1
645 | lsr CARG3, TMP1, #3
649 | str TMP0, [BASE, RA, lsl #3]
652 |//-- Table indexing metamethods -----------------------------------------
655 | movn CARG4, #~LJ_TSTR
656 | add CARG2, BASE, RB, lsl #3
657 | add CARG4, STR:RC, CARG4, lsl #47
661 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
662 | str CARG2, GL->tmptv
663 | add CARG2, GL, #offsetof(global_State, tmptv)
665 | add CARG3, sp, TMPDofs
669 |->vmeta_tgetb: // RB = table, RC = index
671 | add CARG2, BASE, RB, lsl #3
672 | add CARG3, sp, TMPDofs
676 |->vmeta_tgetv: // RB = table, RC = key
677 | add CARG2, BASE, RB, lsl #3
678 | add CARG3, BASE, RC, lsl #3
683 | bl extern lj_meta_tget // (lua_State *L, TValue *o, TValue *k)
684 | // Returns TValue * (finished) or NULL (metamethod).
687 | str TMP0, [BASE, RA, lsl #3]
690 |3: // Call __index metamethod.
691 | // BASE = base, L->top = new base, stack = cont/func/t/k
692 | sub TMP1, BASE, #FRAME_CONT
694 | mov NARGS8:RC, #16 // 2 args for func(t, k).
695 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
696 | str PC, [BASE, #-24] // [cont|PC]
698 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
699 | b ->vm_call_dispatch_f
703 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
704 | // Returns cTValue * or NULL.
706 | cbz CRET1, ->BC_TGETR_Z
710 |//-----------------------------------------------------------------------
713 | movn CARG4, #~LJ_TSTR
714 | add CARG2, BASE, RB, lsl #3
715 | add CARG4, STR:RC, CARG4, lsl #47
719 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
720 | str CARG2, GL->tmptv
721 | add CARG2, GL, #offsetof(global_State, tmptv)
723 | add CARG3, sp, TMPDofs
727 |->vmeta_tsetb: // RB = table, RC = index
729 | add CARG2, BASE, RB, lsl #3
730 | add CARG3, sp, TMPDofs
735 | add CARG2, BASE, RB, lsl #3
736 | add CARG3, BASE, RC, lsl #3
741 | bl extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
742 | // Returns TValue * (finished) or NULL (metamethod).
743 | ldr TMP0, [BASE, RA, lsl #3]
745 | // NOBARRIER: lj_meta_tset ensures the table is not black.
749 |3: // Call __newindex metamethod.
750 | // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
751 | sub TMP1, BASE, #FRAME_CONT
753 | mov NARGS8:RC, #24 // 3 args for func(t, k, v).
754 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
755 | str TMP0, [BASE, #16] // Copy value to third argument.
756 | str PC, [BASE, #-24] // [cont|PC]
758 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
759 | b ->vm_call_dispatch_f
766 | bl extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
767 | // Returns TValue *.
770 |//-- Comparison metamethods ---------------------------------------------
773 | add CARG2, BASE, RA, lsl #3
775 | add CARG3, BASE, RC, lsl #3
780 | bl extern lj_meta_comp // (lua_State *L, TValue *o1, *o2, int op)
781 | // Returns 0/1 or TValue * (metamethod).
786 | ldrh RBw, [PC, # OFS_RD]
788 | add RB, PC, RB, lsl #2
789 | sub RB, RB, #0x20000
790 | csel PC, PC, RB, lo
794 |->cont_ra: // RA = resultptr
795 | ldr INSw, [PC, #-4]
797 | decode_RA TMP1, INS
798 | str TMP0, [BASE, TMP1, lsl #3]
801 |->cont_condt: // RA = resultptr
804 | cmp TMP1, TMP0 // Branch if result is true.
807 |->cont_condf: // RA = resultptr
810 | cmp TMP0, TMP1 // Branch if result is false.
814 | // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
815 | and TAB:CARG3, CARG3, #LJ_GCVMASK
820 | bl extern lj_meta_equal // (lua_State *L, GCobj *o1, *o2, int ne)
821 | // Returns 0/1 or TValue * (metamethod).
831 | bl extern lj_meta_equal_cd // (lua_State *L, BCIns op)
832 | // Returns 0/1 or TValue * (metamethod).
843 | bl extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
846 |//-- Arithmetic metamethods ---------------------------------------------
849 | add CARG3, BASE, RB, lsl #3
850 | add CARG4, KBASE, RC, lsl #3
854 | add CARG4, BASE, RB, lsl #3
855 | add CARG3, KBASE, RC, lsl #3
859 | add CARG3, BASE, RC, lsl #3
864 | add CARG3, BASE, RB, lsl #3
865 | add CARG4, BASE, RC, lsl #3
868 | add CARG2, BASE, RA, lsl #3
872 | bl extern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
873 | // Returns NULL (finished) or TValue * (metamethod).
874 | cbz CRET1, ->cont_nop
876 | // Call metamethod for binary op.
878 | // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
879 | sub TMP1, CRET1, BASE
880 | str PC, [CRET1, #-24] // [cont|PC]
881 | add PC, TMP1, #FRAME_CONT
883 | mov NARGS8:RC, #16 // 2 args for func(o1, o2).
884 | b ->vm_call_dispatch
887 | add CARG2, BASE, RC, lsl #3
889 | mov TAB:RC, TAB:CARG1 // Save table (ignored for other types).
894 | bl extern lj_meta_len // (lua_State *L, TValue *o)
895 | // Returns NULL (retry) or TValue * (metamethod base).
897 | cbnz CRET1, ->vmeta_binop // Binop call for compatibility.
898 | mov TAB:CARG1, TAB:RC
901 | b ->vmeta_binop // Binop call for compatibility.
904 |//-- Call metamethod ----------------------------------------------------
906 |->vmeta_call: // Resolve and call __call metamethod.
907 | // RB = old base, BASE = new base, RC = nargs*8
909 | str RB, L->base // This is the callers base!
910 | sub CARG2, BASE, #16
912 | add CARG3, BASE, NARGS8:RC
913 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
914 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
915 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
916 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
919 |->vmeta_callt: // Resolve __call for BC_CALLT.
920 | // BASE = old base, RA = new base, RC = nargs*8
925 | add CARG3, RA, NARGS8:RC
926 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
927 | ldr TMP1, [RA, FRAME_FUNC] // Guaranteed to be a function here.
928 | ldr PC, [BASE, FRAME_PC]
929 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
930 | and LFUNC:CARG3, TMP1, #LJ_GCVMASK
933 |//-- Argument coercion for 'for' statement ------------------------------
940 | bl extern lj_meta_for // (lua_State *L, TValue *base)
941 | ldr INSw, [PC, #-4]
948 | cmp TMP0, #BC_JFORI
953 |//-----------------------------------------------------------------------
954 |//-- Fast functions -----------------------------------------------------
955 |//-----------------------------------------------------------------------
961 |.macro .ffunc_1, name
968 |.macro .ffunc_2, name
970 | ldp CARG1, CARG2, [BASE]
975 |.macro .ffunc_n, name
981 | checknum CARG1, ->fff_fallback
984 |.macro .ffunc_nn, name
986 | ldp CARG1, CARG2, [BASE]
988 | ldp FARG1, FARG2, [BASE]
990 | checknum CARG1, ->fff_fallback
991 | checknum CARG2, ->fff_fallback
994 |// Inlined GC threshold check. Caveat: uses CARG1 and CARG2.
996 | ldp CARG1, CARG2, GL->gc.total // Assumes threshold follows total.
1003 |//-- Base library: checks -----------------------------------------------
1006 | ldr PC, [BASE, FRAME_PC]
1009 | bhs ->fff_fallback
1010 | str CARG1, [BASE, #-16]
1012 | subs RA, NARGS8:RC, #8
1013 | add RC, NARGS8:RC, #8 // Compute (nresults+1)*8.
1014 | cbz RA, ->fff_res // Done if exactly 1 argument.
1016 | ldr CARG1, [RB, #16]
1018 | str CARG1, [RB], #8
1023 | mov TMP0, #~LJ_TISNUM
1024 | asr ITYPE, CARG1, #47
1025 | cmn ITYPE, #~LJ_TISNUM
1026 | csinv TMP1, TMP0, ITYPE, lo
1027 | add TMP1, TMP1, #offsetof(GCfuncC, upvalue)/8
1028 | ldr CARG1, [CFUNC:CARG3, TMP1, lsl #3]
1031 |//-- Base library: getters and setters ---------------------------------
1033 |.ffunc_1 getmetatable
1034 | asr ITYPE, CARG1, #47
1035 | cmn ITYPE, #-LJ_TTAB
1036 | ccmn ITYPE, #-LJ_TUDATA, #4, ne
1037 | and TAB:CARG1, CARG1, #LJ_GCVMASK
1039 |1: // Field metatable must be at same offset for GCtab and GCudata!
1040 | ldr TAB:RB, TAB:CARG1->metatable
1043 | ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable]
1044 | cbz TAB:RB, ->fff_restv
1045 | ldr TMP1w, TAB:RB->hmask
1046 | ldr TMP2w, STR:RC->sid
1047 | ldr NODE:CARG3, TAB:RB->node
1048 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
1049 | add TMP1, TMP1, TMP1, lsl #1
1050 | movn CARG4, #~LJ_TSTR
1051 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
1052 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
1053 |3: // Rearranged logic, because we expect _not_ to find the key.
1054 | ldp CARG1, TMP0, NODE:CARG3->val
1055 | ldr NODE:CARG3, NODE:CARG3->next
1058 | cbnz NODE:CARG3, <3
1060 | mov CARG1, RB // Use metatable as default result.
1061 | movk CARG1, #(LJ_TTAB>>1)&0xffff, lsl #48
1069 | movn TMP0, #~LJ_TISNUM
1071 | csel ITYPE, ITYPE, TMP0, hs
1072 | sub TMP1, GL, ITYPE, lsl #3
1073 | ldr TAB:RB, [TMP1, #offsetof(global_State, gcroot[GCROOT_BASEMT])-8]
1076 |.ffunc_2 setmetatable
1077 | // Fast path: no mt for table yet and not clearing the mt.
1078 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1079 | ldr TAB:TMP0, TAB:TMP1->metatable
1080 | asr ITYPE, CARG2, #47
1081 | ldrb TMP2w, TAB:TMP1->marked
1082 | cmn ITYPE, #-LJ_TTAB
1083 | and TAB:CARG2, CARG2, #LJ_GCVMASK
1084 | ccmp TAB:TMP0, #0, #0, eq
1085 | bne ->fff_fallback
1086 | str TAB:CARG2, TAB:TMP1->metatable
1087 | tbz TMP2w, #2, ->fff_restv // isblack(table)
1088 | barrierback TAB:TMP1, TMP2w, TMP0
1093 | cmp NARGS8:RC, #16
1094 | blo ->fff_fallback
1095 | checktab CARG2, ->fff_fallback
1097 | add CARG3, BASE, #8
1098 | bl extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1099 | // Returns cTValue *.
1100 | ldr CARG1, [CRET1]
1103 |//-- Base library: conversions ------------------------------------------
1106 | // Only handles the number case inline (without a base argument).
1109 | bne ->fff_fallback
1110 | checknumber CARG1, ->fff_fallback
1114 | // Only handles the string or number case inline.
1115 | asr ITYPE, CARG1, #47
1116 | cmn ITYPE, #-LJ_TSTR
1117 | // A __tostring method in the string base metatable is ignored.
1119 | // Handle numbers inline, unless a number base metatable is present.
1120 | ldr TMP1, GL->gcroot[GCROOT_BASEMT_NUM]
1122 | cmn ITYPE, #-LJ_TISNUM
1123 | ccmp TMP1, #0, #0, ls
1124 | str PC, SAVE_PC // Redundant (but a defined value).
1125 | bne ->fff_fallback
1129 | bl extern lj_strfmt_number // (lua_State *L, cTValue *o)
1130 | // Returns GCstr *.
1131 | movn TMP1, #~LJ_TSTR
1133 | add CARG1, CARG1, TMP1, lsl #47
1136 |//-- Base library: iterators -------------------------------------------
1139 | checktp CARG1, LJ_TTAB, ->fff_fallback
1140 | str TISNIL, [BASE, NARGS8:RC] // Set missing 2nd arg to nil.
1141 | ldr PC, [BASE, FRAME_PC]
1142 | add CARG2, BASE, #8
1143 | sub CARG3, BASE, #16
1144 | bl extern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1145 | // Returns 1=found, 0=end, -1=error.
1147 | tbnz CRET1w, #31, ->fff_fallback // Invalid key.
1148 | cbnz CRET1, ->fff_res // Found key/value.
1149 | // End of traversal: return nil.
1150 | str TISNIL, [BASE, #-16]
1154 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1156 | ldr TAB:CARG2, TAB:TMP1->metatable
1158 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1159 | ldr PC, [BASE, FRAME_PC]
1161 | cbnz TAB:CARG2, ->fff_fallback
1164 | stp CFUNC:CARG4, CARG1, [BASE, #-16]
1165 | str TISNIL, [BASE]
1168 |.ffunc_2 ipairs_aux
1169 | checktab CARG1, ->fff_fallback
1170 | checkint CARG2, ->fff_fallback
1171 | ldr TMP1w, TAB:CARG1->asize
1172 | ldr CARG3, TAB:CARG1->array
1173 | ldr TMP0w, TAB:CARG1->hmask
1174 | add CARG2w, CARG2w, #1
1176 | ldr PC, [BASE, FRAME_PC]
1177 | add_TISNUM TMP2, CARG2
1179 | str TMP2, [BASE, #-16]
1180 | bhs >2 // Not in array part?
1181 | ldr TMP0, [CARG3, CARG2, lsl #3]
1183 | mov TMP1, #(2+1)*8
1185 | str TMP0, [BASE, #-8]
1186 | csel RC, RC, TMP1, eq
1188 |2: // Check for empty hash part first. Otherwise call C function.
1189 | cbz TMP0w, ->fff_res
1190 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
1191 | // Returns cTValue * or NULL.
1192 | cbz CRET1, ->fff_res
1197 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1199 | ldr TAB:CARG2, TAB:TMP1->metatable
1201 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1202 | ldr PC, [BASE, FRAME_PC]
1204 | cbnz TAB:CARG2, ->fff_fallback
1207 | stp CFUNC:CARG4, CARG1, [BASE, #-16]
1208 | str TISNUM, [BASE]
1211 |//-- Base library: catch errors ----------------------------------------
1215 | ldrb TMP0w, GL->hookmask
1216 | blo ->fff_fallback
1217 | sub NARGS8:RC, NARGS8:RC, #8
1219 | add BASE, BASE, #16
1220 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1221 | add PC, TMP0, #16+FRAME_PCALL
1222 | beq ->vm_call_dispatch
1224 | add TMP2, BASE, NARGS8:RC
1226 | ldr TMP0, [TMP2, #-16]
1227 | str TMP0, [TMP2, #-8]!
1230 | b ->vm_call_dispatch
1233 | ldp CARG1, CARG2, [BASE]
1234 | ldrb TMP0w, GL->hookmask
1235 | subs NARGS8:TMP1, NARGS8:RC, #16
1236 | blo ->fff_fallback
1238 | asr ITYPE, CARG2, #47
1239 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1240 | cmn ITYPE, #-LJ_TFUNC
1241 | add PC, TMP0, #24+FRAME_PCALL
1242 | bne ->fff_fallback // Traceback must be a function.
1243 | mov NARGS8:RC, NARGS8:TMP1
1244 | add BASE, BASE, #24
1245 | stp CARG2, CARG1, [RB] // Swap function and traceback.
1246 | cbz NARGS8:RC, ->vm_call_dispatch
1249 |//-- Coroutine library --------------------------------------------------
1251 |.macro coroutine_resume_wrap, resume
1253 |.ffunc_1 coroutine_resume
1254 | checktp CARG1, LJ_TTHREAD, ->fff_fallback
1256 |.ffunc coroutine_wrap_aux
1257 | ldr L:CARG1, CFUNC:CARG3->upvalue[0].gcr
1258 | and L:CARG1, CARG1, #LJ_GCVMASK
1260 | ldr PC, [BASE, FRAME_PC]
1262 | ldp RB, CARG2, L:CARG1->base
1263 | ldrb TMP1w, L:CARG1->status
1264 | add TMP0, CARG2, TMP1
1267 | beq ->fff_fallback
1268 | cmp TMP1, #LUA_YIELD
1269 | add TMP0, CARG2, #8
1270 | csel CARG2, CARG2, TMP0, hs
1271 | ldr CARG4, L:CARG1->maxstack
1272 | add CARG3, CARG2, NARGS8:RC
1273 | ldr RB, L:CARG1->cframe
1274 | ccmp CARG3, CARG4, #2, ls
1275 | ccmp RB, #0, #2, ls
1276 | bhi ->fff_fallback
1278 | sub CARG3, CARG3, #8 // Keep resumed thread in stack for GC.
1279 | add BASE, BASE, #8
1280 | sub NARGS8:RC, NARGS8:RC, #8
1282 | str CARG3, L:CARG1->top
1285 |2: // Move args to coroutine.
1286 | ldr TMP0, [BASE, RB]
1288 | str TMP0, [CARG2, RB]
1295 | bl ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1296 | // Returns thread status.
1298 | ldp CARG3, CARG4, L:RA->base
1299 | cmp CRET1, #LUA_YIELD
1302 | st_vmstate ST_INTERP
1304 | sub RC, CARG4, CARG3
1305 | ldr CARG1, L->maxstack
1306 | add CARG2, BASE, RC
1307 | cbz RC, >6 // No results?
1310 | bhi >9 // Need to grow stack?
1313 | str CARG3, L:RA->top // Clear coroutine stack.
1314 |5: // Move results from coroutine.
1315 | ldr TMP0, [CARG3, RB]
1317 | str TMP0, [BASE, RB]
1325 | str TMP1, [BASE, #-8] // Prepend true/false to results.
1331 | ands CARG1, PC, #FRAME_TYPE
1333 | str RCw, SAVE_MULTRES
1337 |8: // Coroutine returned with error (at co->top-1).
1339 | ldr TMP0, [CARG4, #-8]!
1342 | str CARG4, L:RA->top // Remove error from coroutine stack.
1343 | str TMP0, [BASE] // Copy error message.
1348 | bl extern lj_ffh_coroutine_wrap_err // (lua_State *L, lua_State *co)
1352 |9: // Handle stack expansion on return from yield.
1355 | bl extern lj_state_growstack // (lua_State *L, int n)
1360 | coroutine_resume_wrap 1 // coroutine.resume
1361 | coroutine_resume_wrap 0 // coroutine.wrap
1363 |.ffunc coroutine_yield
1364 | ldr TMP0, L->cframe
1365 | add TMP1, BASE, NARGS8:RC
1366 | mov CRET1, #LUA_YIELD
1367 | stp BASE, TMP1, L->base
1368 | tbz TMP0, #0, ->fff_fallback
1369 | str xzr, L->cframe
1370 | strb CRET1w, L->status
1373 |//-- Math library -------------------------------------------------------
1375 |.macro math_round, func, round
1376 | .ffunc math_ .. func
1380 | blo ->fff_fallback
1381 | cmp TISNUMhi, CARG1, lsr #32
1383 | blo ->fff_fallback
1388 | math_round floor, frintm
1389 | math_round ceil, frintp
1392 | checknumber CARG1, ->fff_fallback
1393 | and CARG1, CARG1, #U64x(7fffffff,ffffffff)
1395 | eor CARG2w, CARG1w, CARG1w, asr #31
1396 | movz CARG3, #0x41e0, lsl #48 // 2^31.
1397 | subs CARG1w, CARG2w, CARG1w, asr #31
1398 | add_TISNUM CARG1, CARG1
1399 | csel CARG1, CARG1, CARG3, pl
1403 | // CARG1 = TValue result.
1404 | ldr PC, [BASE, FRAME_PC]
1405 | str CARG1, [BASE, #-16]
1410 | // RC = (nresults+1)*8, PC = return.
1411 | ands CARG1, PC, #FRAME_TYPE
1412 | str RCw, SAVE_MULTRES
1415 | ldr INSw, [PC, #-4]
1418 | cmp RC, RB, lsl #3 // More results expected?
1420 | decode_RA TMP1, INS
1421 | // Adjust BASE. KBASE is assumed to be set for the calling frame.
1422 | sub BASE, RA, TMP1, lsl #3
1425 |6: // Fill up results with nil.
1428 | str TISNIL, [TMP1, #-8]
1431 |.macro math_extern, func
1432 | .ffunc_n math_ .. func
1437 |.macro math_extern2, func
1438 | .ffunc_nn math_ .. func
1446 | ldr PC, [BASE, FRAME_PC]
1447 | str d0, [BASE, #-16]
1454 | bne ->fff_fallback // Need exactly 1 argument.
1455 | checknum CARG1, ->fff_fallback
1471 | math_extern2 atan2
1474 |.ffunc_2 math_ldexp
1476 | checknum CARG1, ->fff_fallback
1477 | checkint CARG2, ->fff_fallback
1478 | sxtw CARG1, CARG2w
1479 | bl extern ldexp // (double x, int exp)
1482 |.ffunc_n math_frexp
1483 | add CARG1, sp, TMPDofs
1486 | ldr PC, [BASE, FRAME_PC]
1487 | str d0, [BASE, #-16]
1489 | add_TISNUM CARG2, CARG2
1490 | str CARG2, [BASE, #-8]
1494 | sub CARG1, BASE, #16
1495 | ldr PC, [BASE, FRAME_PC]
1498 | str d0, [BASE, #-8]
1501 |.macro math_minmax, name, cond, fcond
1505 | checkint CARG1, >4
1506 |1: // Handle integers.
1510 | checkint CARG2, >3
1511 | cmp CARG1w, CARG2w
1513 | csel CARG1, CARG2, CARG1, cond
1515 |3: // Convert intermediate result to number and continue below.
1517 | blo ->fff_fallback
1523 | blo ->fff_fallback
1524 |5: // Handle numbers.
1529 | checknum CARG2, >7
1533 | fcsel d0, d1, d0, fcond
1535 |7: // Convert integer to number and continue above.
1537 | blo ->fff_fallback
1541 | math_minmax math_min, gt, pl
1542 | math_minmax math_max, lt, le
1544 |//-- String library -----------------------------------------------------
1546 |.ffunc string_byte // Only handle the 1-arg case here.
1547 | ldp PC, CARG1, [BASE, FRAME_PC]
1549 | asr ITYPE, CARG1, #47
1550 | ccmn ITYPE, #-LJ_TSTR, #0, eq
1551 | and STR:CARG1, CARG1, #LJ_GCVMASK
1552 | bne ->fff_fallback
1553 | ldrb TMP0w, STR:CARG1[1] // Access is always ok (NUL at end).
1554 | ldr CARG3w, STR:CARG1->len
1555 | add_TISNUM TMP0, TMP0
1556 | str TMP0, [BASE, #-16]
1558 | cbz CARG3, ->fff_res
1561 |.ffunc string_char // Only handle the 1-arg case here.
1563 | ldp PC, CARG1, [BASE, FRAME_PC]
1565 | ccmp NARGS8:RC, #8, #0, ls // Need exactly 1 argument.
1566 | bne ->fff_fallback
1567 | checkint CARG1, ->fff_fallback
1569 | // Point to the char inside the integer in the stack slot.
1573 | add CARG2, BASE, #7
1576 | // CARG2 = str, CARG3 = len.
1580 | bl extern lj_str_new // (lua_State *L, char *str, size_t l)
1582 | // Returns GCstr *.
1584 | movn TMP1, #~LJ_TSTR
1585 | add CARG1, CARG1, TMP1, lsl #47
1591 | ldr CARG3, [BASE, #16]
1592 | cmp NARGS8:RC, #16
1595 | blo ->fff_fallback
1596 | checkint CARG3, ->fff_fallback
1599 | ldr CARG2, [BASE, #8]
1600 | checkstr CARG1, ->fff_fallback
1601 | ldr TMP1w, STR:CARG1->len
1602 | checkint CARG2, ->fff_fallback
1603 | sxtw CARG2, CARG2w
1604 | // CARG1 = str, TMP1 = str->len, CARG2 = start, RB = end
1605 | add TMP2, RB, TMP1
1607 | add TMP0, CARG2, TMP1
1608 | csinc RB, RB, TMP2, ge // if (end < 0) end += len+1
1610 | csinc CARG2, CARG2, TMP0, ge // if (start < 0) start += len+1
1612 | csel RB, RB, xzr, ge // if (end < 0) end = 0
1614 | csinc CARG2, CARG2, xzr, ge // if (start < 1) start = 1
1616 | csel RB, RB, TMP1, le // if (end > len) end = len
1617 | add CARG1, STR:CARG1, #sizeof(GCstr)-1
1618 | subs CARG3, RB, CARG2 // len = end - start
1619 | add CARG2, CARG1, CARG2
1620 | add CARG3, CARG3, #1 // len += 1
1622 | add STR:CARG1, GL, #offsetof(global_State, strempty)
1623 | movn TMP1, #~LJ_TSTR
1624 | add CARG1, CARG1, TMP1, lsl #47
1627 |.macro ffstring_op, name
1628 | .ffunc string_ .. name
1632 | asr ITYPE, CARG2, #47
1633 | ccmn ITYPE, #-LJ_TSTR, #0, hs
1634 | and STR:CARG2, CARG2, #LJ_GCVMASK
1635 | bne ->fff_fallback
1636 | ldr TMP0, GL->tmpbuf.b
1637 | add SBUF:CARG1, GL, #offsetof(global_State, tmpbuf)
1640 | str L, GL->tmpbuf.L
1641 | str TMP0, GL->tmpbuf.w
1642 | bl extern lj_buf_putstr_ .. name
1643 | bl extern lj_buf_tostr
1647 |ffstring_op reverse
1651 |//-- Bit library --------------------------------------------------------
1653 |// FP number to bit conversion for soft-float. Clobbers CARG1-CARG3
1655 | bls ->fff_fallback
1656 | add CARG2, CARG1, CARG1
1658 | sub CARG3, CARG3, CARG2, lsr #53
1661 | and CARG2, CARG2, #U64x(001fffff,ffffffff)
1662 | orr CARG2, CARG2, #U64x(00200000,00000000)
1664 | lsr CARG2, CARG2, CARG3
1665 | cneg CARG1w, CARG2w, mi
1671 |.macro .ffunc_bit, name
1672 | .ffunc_1 bit_..name
1674 | checkint CARG1, ->vm_tobit_fb
1678 |.macro .ffunc_bit_op, name, ins
1684 | ldr CARG1, [BASE, RA]
1688 | checkint CARG1, ->vm_tobit_fb
1690 | ins TMP0w, TMP0w, CARG1w
1694 |.ffunc_bit_op band, and
1695 |.ffunc_bit_op bor, orr
1696 |.ffunc_bit_op bxor, eor
1700 |9: // Label reused by .ffunc_bit_op users.
1701 | add_TISNUM CARG1, TMP0
1706 | add_TISNUM CARG1, TMP0
1711 | add_TISNUM CARG1, TMP0
1714 |.macro .ffunc_bit_sh, name, ins, shmod
1716 | ldp TMP0, CARG1, [BASE]
1717 | cmp NARGS8:RC, #16
1718 | blo ->fff_fallback
1720 | checkint CARG1, ->vm_tobit_fb
1729 | checkint CARG1, ->vm_tobit_fb
1731 | ins TMP0w, CARG1w, TMP1w
1732 | add_TISNUM CARG1, TMP0
1736 |.ffunc_bit_sh lshift, lsl, 0
1737 |.ffunc_bit_sh rshift, lsr, 0
1738 |.ffunc_bit_sh arshift, asr, 0
1739 |.ffunc_bit_sh rol, ror, 1
1740 |.ffunc_bit_sh ror, ror, 0
1742 |//-----------------------------------------------------------------------
1744 |->fff_fallback: // Call fast function fallback handler.
1745 | // BASE = new base, RC = nargs*8
1746 | ldp CFUNC:CARG3, PC, [BASE, FRAME_FUNC] // Fallback may overwrite PC.
1747 | ldr TMP2, L->maxstack
1748 | add TMP1, BASE, NARGS8:RC
1749 | stp BASE, TMP1, L->base
1750 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1751 | add TMP1, TMP1, #8*LUA_MINSTACK
1752 | ldr CARG3, CFUNC:CARG3->f
1753 | str PC, SAVE_PC // Redundant (but a defined value).
1756 | bhi >5 // Need to grow stack.
1757 | blr_auth CARG3 // (lua_State *L)
1758 | // Either throws an error, or recovers and returns -1, 0 or nresults+1.
1763 | bgt ->fff_res // Returned nresults+1?
1764 |1: // Returned 0 or -1: retry fast path.
1766 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1767 | sub NARGS8:RC, CARG1, BASE
1768 | bne ->vm_call_tail // Returned -1?
1769 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1770 | ins_callt // Returned 0: retry fast path.
1772 |// Reconstruct previous base for vmeta_call during tailcall.
1774 | ands TMP0, PC, #FRAME_TYPE
1775 | and TMP1, PC, #~FRAME_TYPEP
1777 | ldrb RAw, [PC, #-4+OFS_RA]
1781 | sub RB, BASE, TMP1
1782 | b ->vm_call_dispatch // Resolve again for tailcall.
1784 |5: // Grow stack for fallback handler.
1785 | mov CARG2, #LUA_MINSTACK
1786 | bl extern lj_state_growstack // (lua_State *L, int n)
1788 | cmp CARG1, CARG1 // Set zero-flag to force retry.
1791 |->fff_gcstep: // Call GC step function.
1792 | // BASE = new base, RC = nargs*8
1794 | add CARG2, BASE, NARGS8:RC // Calculate L->top.
1796 | stp BASE, CARG2, L->base
1797 | str PC, SAVE_PC // Redundant (but a defined value).
1799 | bl extern lj_gc_step // (lua_State *L)
1800 | ldp BASE, CARG2, L->base
1801 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1802 | mov lr, RA // Help return address predictor.
1803 | sub NARGS8:RC, CARG2, BASE // Calculate nargs*8.
1804 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1807 |//-----------------------------------------------------------------------
1808 |//-- Special dispatch targets -------------------------------------------
1809 |//-----------------------------------------------------------------------
1811 |->vm_record: // Dispatch target for recording phase.
1813 | ldrb CARG1w, GL->hookmask
1814 | tst CARG1, #HOOK_VMEVENT // No recording while in vmevent.
1816 | // Decrement the hookcount for consistency, but always do the call.
1817 | ldr CARG2w, GL->hookcount
1818 | tst CARG1, #HOOK_ACTIVE
1820 | sub CARG2w, CARG2w, #1
1821 | tst CARG1, #LUA_MASKLINE|LUA_MASKCOUNT
1823 | str CARG2w, GL->hookcount
1827 |->vm_rethook: // Dispatch target for return hooks.
1828 | ldrb TMP2w, GL->hookmask
1829 | tbz TMP2w, #HOOK_ACTIVE_SHIFT, >1 // Hook already active?
1830 |5: // Re-dispatch to static ins.
1831 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1834 |->vm_inshook: // Dispatch target for instr/line hooks.
1835 | ldrb TMP2w, GL->hookmask
1836 | ldr TMP3w, GL->hookcount
1837 | tbnz TMP2w, #HOOK_ACTIVE_SHIFT, <5 // Hook already active?
1838 | tst TMP2w, #LUA_MASKLINE|LUA_MASKCOUNT
1840 | sub TMP3w, TMP3w, #1
1841 | str TMP3w, GL->hookcount
1843 | tbz TMP2w, #LUA_HOOKLINE, <5
1848 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
1849 | bl extern lj_dispatch_ins // (lua_State *L, const BCIns *pc)
1852 |4: // Re-dispatch to static ins.
1853 | ldr INSw, [PC, #-4]
1854 | add TMP1, GL, INS, uxtb #3
1856 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1860 |->cont_hook: // Continue from hook yield.
1861 | ldr CARG1, [CARG4, #-40]
1863 | str CARG1w, SAVE_MULTRES // Restore MULTRES for *M ins.
1866 |->vm_hotloop: // Hot loop counter underflow.
1868 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Same as curr_topL(L).
1869 | add CARG1, GL, #GG_G2DISP+GG_DISP2J
1870 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1872 | ldr CARG3, LFUNC:CARG3->pc
1874 | str L, [GL, #GL_J(L)]
1875 | ldrb CARG3w, [CARG3, #PC2PROTO(framesize)]
1877 | add CARG3, BASE, CARG3, lsl #3
1879 | bl extern lj_trace_hot // (jit_State *J, const BCIns *pc)
1883 |->vm_callhook: // Dispatch target for call hooks.
1889 |->vm_hotcall: // Hot call counter underflow.
1894 | add TMP1, BASE, NARGS8:RC
1898 | stp BASE, TMP1, L->base
1899 | bl extern lj_dispatch_call // (lua_State *L, const BCIns *pc)
1900 | // Returns ASMFunction.
1901 | ldp BASE, TMP1, L->base
1902 | str xzr, SAVE_PC // Invalidate for subsequent line hook.
1903 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
1905 | sub NARGS8:RC, TMP1, BASE
1906 | ldr INSw, [PC, #-4]
1907 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1910 |->cont_stitch: // Trace stitching.
1912 | // RA = resultptr, CARG4 = meta base
1913 | ldr RBw, SAVE_MULTRES
1914 | ldr INSw, [PC, #-4]
1915 | ldr TRACE:CARG3, [CARG4, #-40] // Save previous trace.
1917 | decode_RA RC, INS // Call base.
1918 | and CARG3, CARG3, #LJ_GCVMASK
1920 |1: // Move results down.
1921 | ldr CARG1, [RA], #8
1923 | str CARG1, [BASE, RC, lsl #3]
1932 | bhi >9 // More results wanted?
1934 | ldrh RAw, TRACE:CARG3->traceno
1935 | ldrh RCw, TRACE:CARG3->link
1937 | beq ->cont_nop // Blacklisted.
1939 | bne =>BC_JLOOP // Jump to stitched trace.
1941 | // Stitch a new trace to the previous trace.
1942 | mov CARG1, #GL_J(exitno)
1943 | str RAw, [GL, CARG1]
1944 | mov CARG1, #GL_J(L)
1945 | str L, [GL, CARG1]
1947 | add CARG1, GL, #GG_G2J
1949 | bl extern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
1953 |9: // Fill up results with nil.
1954 | str TISNIL, [BASE, RC, lsl #3]
1959 |->vm_profhook: // Dispatch target for profiler hook.
1964 | bl extern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
1965 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
1971 |//-----------------------------------------------------------------------
1972 |//-- Trace exit handler -------------------------------------------------
1973 |//-----------------------------------------------------------------------
1975 |.macro savex_, a, b
1976 | stp d..a, d..b, [sp, #a*8]
1977 | stp x..a, x..b, [sp, #32*8+a*8]
1982 | sub sp, sp, #(64*8)
1998 | stp d30, d31, [sp, #30*8]
1999 | ldr CARG1, [sp, #64*8] // Load original value of lr.
2000 | add CARG3, sp, #64*8 // Recompute original value of sp.
2001 | mv_vmstate CARG4w, EXIT
2002 | stp xzr, CARG3, [sp, #62*8] // Store 0/sp in RID_LR/RID_SP.
2003 | sub CARG1, CARG1, lr
2005 | lsr CARG1, CARG1, #2
2006 | ldr BASE, GL->jit_base
2007 | sub CARG1, CARG1, #2
2008 | ldr CARG2w, [lr] // Load trace number.
2011 | rev32 CARG2, CARG2
2014 | ubfx CARG2w, CARG2w, #5, #16
2015 | str CARG1w, [GL, #GL_J(exitno)]
2016 | str CARG2w, [GL, #GL_J(parent)]
2017 | str L, [GL, #GL_J(L)]
2018 | str xzr, GL->jit_base
2019 | add CARG1, GL, #GG_G2J
2021 | bl extern lj_trace_exit // (jit_State *J, ExitState *ex)
2022 | // Returns MULTRES (unscaled) or negated error code.
2023 | ldr CARG2, L->cframe
2025 | and sp, CARG2, #CFRAME_RAWMASK
2026 | ldr PC, SAVE_PC // Get SAVE_PC.
2027 | str L, SAVE_L // Set SAVE_L (on-trace resume/yield).
2032 | // CARG1 = MULTRES or negated error code, BASE, PC and GL set.
2037 | cmn CARG1w, #LUA_ERRERR
2038 | bhs >9 // Check for error from exit.
2039 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2041 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2042 | str RCw, SAVE_MULTRES
2044 | ldr CARG2, LFUNC:CARG2->pc
2045 | str xzr, GL->jit_base
2046 | mv_vmstate CARG4w, INTERP
2047 | ldr KBASE, [CARG2, #PC2PROTO(k)]
2048 | // Modified copy of ins_next which handles function header dispatch, too.
2049 | ldrb RBw, [PC, # OFS_OP]
2050 | ldr INSw, [PC], #4
2052 | cmn CARG1w, #17 // Static dispatch?
2054 | cmp RBw, #BC_FUNCC+2 // Fast function?
2055 | add TMP1, GL, INS, uxtb #3
2058 | cmp RBw, #BC_FUNCF // Function header?
2059 | add TMP0, GL, RB, uxtb #3
2060 | ldr RB, [TMP0, #GG_G2DISP]
2062 | lsr TMP0, INS, #16
2063 | csel RC, TMP0, RC, lo
2065 | ldr CARG3, [BASE, FRAME_FUNC]
2067 | add RA, BASE, RA, lsl #3 // Yes: RA = BASE+framesize*8, RC = nargs*8
2068 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2072 |4: // Check frame below fast function.
2073 | ldr CARG1, [BASE, FRAME_PC]
2074 | ands CARG2, CARG1, #FRAME_TYPE
2075 | bne <2 // Trace stitching continuation?
2076 | // Otherwise set KBASE for Lua function below fast function.
2077 | ldr CARG3w, [CARG1, #-4]
2078 | decode_RA CARG1, CARG3
2079 | sub CARG2, BASE, CARG1, lsl #3
2080 | ldr LFUNC:CARG3, [CARG2, #-32]
2081 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2082 | ldr CARG3, LFUNC:CARG3->pc
2083 | ldr KBASE, [CARG3, #PC2PROTO(k)]
2086 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2087 | ldr RA, [GL, #GL_J(trace)]
2089 | ldr TRACE:RA, [RA, RC, lsl #3]
2090 | ldr INSw, TRACE:RA->startins
2091 | add TMP0, GL, INS, uxtb #3
2093 | ldr RB, [TMP0, #GG_G2DISP+GG_DISP2STATIC]
2097 |9: // Rethrow error from the right C frame.
2098 | neg CARG2w, CARG1w
2100 | bl extern lj_err_trace // (lua_State *L, int errcode)
2103 |//-----------------------------------------------------------------------
2104 |//-- Math helper functions ----------------------------------------------
2105 |//-----------------------------------------------------------------------
2107 | // int lj_vm_modi(int dividend, int divisor);
2109 | eor CARG4w, CARG1w, CARG2w
2111 | eor CARG3w, CARG1w, CARG1w, asr #31
2112 | eor CARG4w, CARG2w, CARG2w, asr #31
2113 | sub CARG3w, CARG3w, CARG1w, asr #31
2114 | sub CARG4w, CARG4w, CARG2w, asr #31
2115 | udiv CARG1w, CARG3w, CARG4w
2116 | msub CARG1w, CARG1w, CARG4w, CARG3w
2117 | ccmp CARG1w, #0, #4, mi
2118 | sub CARG3w, CARG1w, CARG4w
2119 | csel CARG1w, CARG1w, CARG3w, eq
2120 | eor CARG3w, CARG1w, CARG2w
2122 | cneg CARG1w, CARG1w, mi
2125 |//-----------------------------------------------------------------------
2126 |//-- Miscellaneous functions --------------------------------------------
2127 |//-----------------------------------------------------------------------
2129 |.define NEXT_TAB, TAB:CARG1
2130 |.define NEXT_RES, CARG1
2131 |.define NEXT_IDX, CARG2w
2132 |.define NEXT_LIM, CARG3w
2133 |.define NEXT_TMP0, TMP0
2134 |.define NEXT_TMP0w, TMP0w
2135 |.define NEXT_TMP1, TMP1
2136 |.define NEXT_TMP1w, TMP1w
2137 |.define NEXT_RES_PTR, sp
2138 |.define NEXT_RES_VAL, [sp]
2139 |.define NEXT_RES_KEY, [sp, #8]
2141 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2142 |// Next idx returned in CRET2w.
2145 | ldr NEXT_LIM, NEXT_TAB->asize
2146 | ldr NEXT_TMP1, NEXT_TAB->array
2147 |1: // Traverse array part.
2148 | subs NEXT_TMP0w, NEXT_IDX, NEXT_LIM
2149 | bhs >5 // Index points after array part?
2150 | ldr NEXT_TMP0, [NEXT_TMP1, NEXT_IDX, uxtw #3]
2151 | cmn NEXT_TMP0, #-LJ_TNIL
2152 | cinc NEXT_IDX, NEXT_IDX, eq
2153 | beq <1 // Skip holes in array part.
2154 | str NEXT_TMP0, NEXT_RES_VAL
2155 | movz NEXT_TMP0w, #(LJ_TISNUM>>1)&0xffff, lsl #16
2156 | stp NEXT_IDX, NEXT_TMP0w, NEXT_RES_KEY
2157 | add NEXT_IDX, NEXT_IDX, #1
2158 | mov NEXT_RES, NEXT_RES_PTR
2162 |5: // Traverse hash part.
2163 | ldr NEXT_TMP1w, NEXT_TAB->hmask
2164 | ldr NODE:NEXT_RES, NEXT_TAB->node
2165 | add NEXT_TMP0w, NEXT_TMP0w, NEXT_TMP0w, lsl #1
2166 | add NEXT_LIM, NEXT_LIM, NEXT_TMP1w
2167 | add NODE:NEXT_RES, NODE:NEXT_RES, NEXT_TMP0w, uxtw #3
2169 | cmp NEXT_IDX, NEXT_LIM
2171 | ldr NEXT_TMP0, NODE:NEXT_RES->val
2172 | cmn NEXT_TMP0, #-LJ_TNIL
2173 | add NEXT_IDX, NEXT_IDX, #1
2175 | // Skip holes in hash part.
2176 | add NODE:NEXT_RES, NODE:NEXT_RES, #sizeof(Node)
2179 |9: // End of iteration. Set the key to nil (not the value).
2180 | movn NEXT_TMP0, #0
2181 | str NEXT_TMP0, NEXT_RES_KEY
2182 | mov NEXT_RES, NEXT_RES_PTR
2186 |//-----------------------------------------------------------------------
2187 |//-- FFI helper functions -----------------------------------------------
2188 |//-----------------------------------------------------------------------
2190 |// Handler for callback functions.
2191 |// Saveregs already performed. Callback slot number in w9, g in x10.
2194 |.type CTSTATE, CTState, PC
2196 | ldr CTSTATE, GL:x10->ctype_state
2198 | add x10, sp, # CFRAME_SPACE
2199 | str w9, CTSTATE->cb.slot
2200 | stp x0, x1, CTSTATE->cb.gpr[0]
2201 | stp d0, d1, CTSTATE->cb.fpr[0]
2202 | stp x2, x3, CTSTATE->cb.gpr[2]
2203 | stp d2, d3, CTSTATE->cb.fpr[2]
2204 | stp x4, x5, CTSTATE->cb.gpr[4]
2205 | stp d4, d5, CTSTATE->cb.fpr[4]
2206 | stp x6, x7, CTSTATE->cb.gpr[6]
2207 | stp d6, d7, CTSTATE->cb.fpr[6]
2208 | str x10, CTSTATE->cb.stack
2209 | mov CARG1, CTSTATE
2210 | str CTSTATE, SAVE_PC // Any value outside of bytecode is ok.
2212 | bl extern lj_ccallback_enter // (CTState *cts, void *cf)
2213 | // Returns lua_State *.
2214 | ldp BASE, RC, L:CRET1->base
2217 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2219 | st_vmstate ST_INTERP
2220 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2224 |->cont_ffi_callback: // Return from FFI callback.
2226 | ldr CTSTATE, GL->ctype_state
2227 | stp BASE, CARG4, L->base
2229 | mov CARG1, CTSTATE
2231 | bl extern lj_ccallback_leave // (CTState *cts, TValue *o)
2232 | ldp x0, x1, CTSTATE->cb.gpr[0]
2233 | ldp d0, d1, CTSTATE->cb.fpr[0]
2237 |->vm_ffi_call: // Call C function via FFI.
2238 | // Caveat: needs special frame unwinding, see below.
2240 | .type CCSTATE, CCallState, x19
2242 | stp_unwind CCSTATE, x20, [sp, #-32]!
2243 | stp fp, lr, [sp, #16]
2246 | ldr TMP0w, CCSTATE:x0->spadj
2247 | ldrb TMP1w, CCSTATE->nsp
2248 | add TMP2, CCSTATE, #offsetof(CCallState, stack)
2249 | subs TMP1, TMP1, #8
2250 | ldr TMP3, CCSTATE->func
2253 |1: // Copy stack slots
2254 | ldr TMP0, [TMP2, TMP1]
2255 | str TMP0, [sp, TMP1]
2256 | subs TMP1, TMP1, #8
2259 | ldp x0, x1, CCSTATE->gpr[0]
2260 | ldp d0, d1, CCSTATE->fpr[0]
2261 | ldp x2, x3, CCSTATE->gpr[2]
2262 | ldp d2, d3, CCSTATE->fpr[2]
2263 | ldp x4, x5, CCSTATE->gpr[4]
2264 | ldp d4, d5, CCSTATE->fpr[4]
2265 | ldp x6, x7, CCSTATE->gpr[6]
2266 | ldp d6, d7, CCSTATE->fpr[6]
2267 | ldr x8, CCSTATE->retp
2270 | stp x0, x1, CCSTATE->gpr[0]
2271 | stp d0, d1, CCSTATE->fpr[0]
2272 | stp d2, d3, CCSTATE->fpr[2]
2273 | ldp fp, lr, [sp, #16]
2274 | ldp_unwind CCSTATE, x20, [sp], #32
2277 |// Note: vm_ffi_call must be the last function in this object file!
2279 |//-----------------------------------------------------------------------
2282 /* Generate the code for a single instruction. */
2283 static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2290 /* -- Comparison ops ---------------------------------------------------- */
2292 /* Remember: all ops branch for a true comparison, fall through otherwise. */
2294 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2295 | // RA = src1, RC = src2, JMP with RC = target
2296 | ldr CARG1, [BASE, RA, lsl #3]
2297 | ldrh RBw, [PC, # OFS_RD]
2298 | ldr CARG2, [BASE, RC, lsl #3]
2300 | add RB, PC, RB, lsl #2
2301 | sub RB, RB, #0x20000
2302 | checkint CARG1, >3
2303 | checkint CARG2, >4
2304 | cmp CARG1w, CARG2w
2305 if (op == BC_ISLT) {
2306 | csel PC, RB, PC, lt
2307 } else if (op == BC_ISGE) {
2308 | csel PC, RB, PC, ge
2309 } else if (op == BC_ISLE) {
2310 | csel PC, RB, PC, le
2312 | csel PC, RB, PC, gt
2318 | ldr FARG1, [BASE, RA, lsl #3]
2320 | ldr FARG2, [BASE, RC, lsl #3]
2321 | cmp TISNUMhi, CARG2, lsr #32
2324 | // RA number, RC int.
2325 | scvtf FARG2, CARG2w
2328 |4: // RA int, RC not int
2329 | ldr FARG2, [BASE, RC, lsl #3]
2331 | // RA int, RC number.
2332 | scvtf FARG1, CARG1w
2334 |5: // RA number, RC number
2336 | // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2337 if (op == BC_ISLT) {
2338 | csel PC, RB, PC, lo
2339 } else if (op == BC_ISGE) {
2340 | csel PC, RB, PC, hs
2341 } else if (op == BC_ISLE) {
2342 | csel PC, RB, PC, ls
2344 | csel PC, RB, PC, hi
2349 case BC_ISEQV: case BC_ISNEV:
2350 vk = op == BC_ISEQV;
2351 | // RA = src1, RC = src2, JMP with RC = target
2352 | ldr CARG1, [BASE, RA, lsl #3]
2353 | add RC, BASE, RC, lsl #3
2354 | ldrh RBw, [PC, # OFS_RD]
2357 | add RB, PC, RB, lsl #2
2358 | sub RB, RB, #0x20000
2359 | asr ITYPE, CARG3, #47
2360 | cmn ITYPE, #-LJ_TISNUM
2366 | // RC is not a number.
2367 | asr TMP0, CARG1, #47
2369 | // Check if RC or RA is a cdata.
2370 | cmn ITYPE, #-LJ_TCDATA
2371 | ccmn TMP0, #-LJ_TCDATA, #4, ne
2372 | beq ->vmeta_equal_cd
2376 | // Tag and value are equal.
2379 | mov PC, RB // Perform branch.
2384 |2: // Check if the tags are the same and it's a table or userdata.
2386 | ccmn ITYPE, #-LJ_TISTABUD, #2, eq
2390 | bhi ->BC_ISEQV_Z // Reuse code from opposite instruction.
2392 | // Different tables or userdatas. Need to check __eq metamethod.
2393 | // Field metatable must be at same offset for GCtab and GCudata!
2394 | and TAB:CARG2, CARG1, #LJ_GCVMASK
2395 | ldr TAB:TMP2, TAB:CARG2->metatable
2397 | cbz TAB:TMP2, <1 // No metatable?
2398 | ldrb TMP1w, TAB:TMP2->nomm
2399 | mov CARG4, #0 // ne = 0
2400 | tbnz TMP1w, #MM_eq, <1 // 'no __eq' flag set: done.
2402 | cbz TAB:TMP2, ->BC_ISEQV_Z // No metatable?
2403 | ldrb TMP1w, TAB:TMP2->nomm
2404 | mov CARG4, #1 // ne = 1.
2405 | tbnz TMP1w, #MM_eq, ->BC_ISEQV_Z // 'no __eq' flag set: done.
2410 case BC_ISEQS: case BC_ISNES:
2411 vk = op == BC_ISEQS;
2412 | // RA = src, RC = str_const (~), JMP with RC = target
2413 | ldr CARG1, [BASE, RA, lsl #3]
2415 | ldrh RBw, [PC, # OFS_RD]
2416 | ldr CARG2, [KBASE, RC, lsl #3]
2418 | movn TMP0, #~LJ_TSTR
2420 | asr ITYPE, CARG1, #47
2422 | add RB, PC, RB, lsl #2
2423 | add CARG2, CARG2, TMP0, lsl #47
2424 | sub RB, RB, #0x20000
2426 | cmn ITYPE, #-LJ_TCDATA
2427 | beq ->vmeta_equal_cd
2431 | csel PC, RB, PC, eq
2433 | csel PC, RB, PC, ne
2438 case BC_ISEQN: case BC_ISNEN:
2439 vk = op == BC_ISEQN;
2440 | // RA = src, RC = num_const (~), JMP with RC = target
2441 | ldr CARG1, [BASE, RA, lsl #3]
2442 | add RC, KBASE, RC, lsl #3
2443 | ldrh RBw, [PC, # OFS_RD]
2446 | add RB, PC, RB, lsl #2
2447 | sub RB, RB, #0x20000
2453 | checkint CARG1, >4
2454 | checkint CARG3, >6
2455 | cmp CARG1w, CARG3w
2458 | csel PC, RB, PC, eq
2462 | csel PC, RB, PC, ne
2473 | ldr FARG1, [BASE, RA, lsl #3]
2475 | cmp TISNUMhi, CARG3, lsr #32
2477 | // RA number, RC int.
2478 | scvtf FARG2, CARG3w
2480 | // RA number, RC number.
2484 |6: // RA int, RC number
2486 | scvtf FARG1, CARG1w
2492 | asr ITYPE, CARG1, #47
2493 | cmn ITYPE, #-LJ_TCDATA
2495 | b ->vmeta_equal_cd
2499 case BC_ISEQP: case BC_ISNEP:
2500 vk = op == BC_ISEQP;
2501 | // RA = src, RC = primitive_type (~), JMP with RC = target
2502 | ldr TMP0, [BASE, RA, lsl #3]
2503 | ldrh RBw, [PC, # OFS_RD]
2506 | add RB, PC, RB, lsl #2
2508 | asr ITYPE, TMP0, #47
2509 | cmn ITYPE, #-LJ_TCDATA
2510 | beq ->vmeta_equal_cd
2513 | cmn RC, TMP0, asr #47
2515 | sub RB, RB, #0x20000
2517 | csel PC, RB, PC, eq
2519 | csel PC, RB, PC, ne
2524 /* -- Unary test and copy ops ------------------------------------------- */
2526 case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
2527 | // RA = dst or unused, RC = src, JMP with RC = target
2528 | ldrh RBw, [PC, # OFS_RD]
2529 | ldr TMP0, [BASE, RC, lsl #3]
2532 | add RB, PC, RB, lsl #2
2534 | sub RB, RB, #0x20000
2535 if (op == BC_ISTC || op == BC_IST) {
2536 if (op == BC_ISTC) {
2537 | csel RA, RA, RC, lo
2539 | csel PC, RB, PC, lo
2541 if (op == BC_ISFC) {
2542 | csel RA, RA, RC, hs
2544 | csel PC, RB, PC, hs
2546 if (op == BC_ISTC || op == BC_ISFC) {
2547 | str TMP0, [BASE, RA, lsl #3]
2553 | // RA = src, RC = -type
2554 | ldr TMP0, [BASE, RA, lsl #3]
2555 | cmn RC, TMP0, asr #47
2556 | bne ->vmeta_istype
2560 | // RA = src, RC = -(TISNUM-1)
2561 | ldr TMP0, [BASE, RA]
2562 | checknum TMP0, ->vmeta_istype
2566 /* -- Unary ops --------------------------------------------------------- */
2569 | // RA = dst, RC = src
2570 | ldr TMP0, [BASE, RC, lsl #3]
2571 | str TMP0, [BASE, RA, lsl #3]
2575 | // RA = dst, RC = src
2576 | ldr TMP0, [BASE, RC, lsl #3]
2580 | csel TMP0, TMP1, TMP2, lo
2581 | str TMP0, [BASE, RA, lsl #3]
2585 | // RA = dst, RC = src
2586 | ldr TMP0, [BASE, RC, lsl #3]
2587 | asr ITYPE, TMP0, #47
2588 | cmn ITYPE, #-LJ_TISNUM
2590 | eor TMP0, TMP0, #U64x(80000000,00000000)
2593 | movz CARG3, #0x41e0, lsl #48 // 2^31.
2594 | add_TISNUM TMP0, TMP0
2595 | csel TMP0, TMP0, CARG3, vc
2597 | str TMP0, [BASE, RA, lsl #3]
2601 | // RA = dst, RC = src
2602 | ldr CARG1, [BASE, RC, lsl #3]
2603 | asr ITYPE, CARG1, #47
2604 | cmn ITYPE, #-LJ_TSTR
2605 | and CARG1, CARG1, #LJ_GCVMASK
2607 | ldr CARG1w, STR:CARG1->len
2609 | add_TISNUM CARG1, CARG1
2610 | str CARG1, [BASE, RA, lsl #3]
2614 | cmn ITYPE, #-LJ_TTAB
2617 | ldr TAB:CARG2, TAB:CARG1->metatable
2618 | cbnz TAB:CARG2, >9
2622 | bl extern lj_tab_len // (GCtab *t)
2623 | // Returns uint32_t (but less than 2^31).
2628 | ldrb TMP1w, TAB:CARG2->nomm
2629 | tbnz TMP1w, #MM_len, <3 // 'no __len' flag set: done.
2634 /* -- Binary ops -------------------------------------------------------- */
2636 |.macro ins_arithcheck_int, target
2637 | checkint CARG1, target
2638 | checkint CARG2, target
2641 |.macro ins_arithcheck_num, target
2642 | checknum CARG1, target
2643 | checknum CARG2, target
2646 |.macro ins_arithcheck_nzdiv, target
2647 | cbz CARG2w, target
2650 |.macro ins_arithhead
2651 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
2661 |.macro ins_arithload, reg1, reg2
2662 | // RA = dst, RB = src1, RC = src2 | num_const
2665 | ldr reg1, [BASE, RB, lsl #3]
2666 | ldr reg2, [KBASE, RC, lsl #3]
2669 | ldr reg1, [KBASE, RC, lsl #3]
2670 | ldr reg2, [BASE, RB, lsl #3]
2673 | ldr reg1, [BASE, RB, lsl #3]
2674 | ldr reg2, [BASE, RC, lsl #3]
2679 |.macro ins_arithfallback, ins
2682 | ins ->vmeta_arith_vn
2685 | ins ->vmeta_arith_nv
2688 | ins ->vmeta_arith_vv
2693 |.macro ins_arithmod, res, reg1, reg2
2694 | fdiv d2, reg1, reg2
2696 | // Cannot use fmsub, because FMA is not enabled by default.
2698 | fsub res, reg1, d2
2701 |.macro ins_arithdn, intins, fpins
2703 | ins_arithload CARG1, CARG2
2704 | ins_arithcheck_int >5
2705 |.if "intins" == "smull"
2706 | smull CARG1, CARG1w, CARG2w
2707 | cmp CARG1, CARG1, sxtw
2708 | mov CARG1w, CARG1w
2709 | ins_arithfallback bne
2710 |.elif "intins" == "ins_arithmodi"
2711 | ins_arithfallback ins_arithcheck_nzdiv
2714 | intins CARG1w, CARG1w, CARG2w
2715 | ins_arithfallback bvs
2717 | add_TISNUM CARG1, CARG1
2718 | str CARG1, [BASE, RA, lsl #3]
2723 | ins_arithload FARG1, FARG2
2724 | ins_arithfallback ins_arithcheck_num
2725 | fpins FARG1, FARG1, FARG2
2726 | str FARG1, [BASE, RA, lsl #3]
2730 |.macro ins_arithfp, fpins
2732 | ins_arithload CARG1, CARG2
2733 | ins_arithload FARG1, FARG2
2734 | ins_arithfallback ins_arithcheck_num
2735 |.if "fpins" == "fpow"
2738 | fpins FARG1, FARG1, FARG2
2740 | str FARG1, [BASE, RA, lsl #3]
2744 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
2745 | ins_arithdn adds, fadd
2747 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
2748 | ins_arithdn subs, fsub
2750 case BC_MULVN: case BC_MULNV: case BC_MULVV:
2751 | ins_arithdn smull, fmul
2753 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
2756 case BC_MODVN: case BC_MODNV: case BC_MODVV:
2757 | ins_arithdn ins_arithmodi, ins_arithmod
2760 | // NYI: (partial) integer arithmetic.
2767 | // RA = dst, RB = src_start, RC = src_end
2770 | add CARG2, BASE, RC, lsl #3
2772 | // RA = dst, CARG2 = top-1, CARG3 = left
2775 | bl extern lj_meta_cat // (lua_State *L, TValue *top, int left)
2776 | // Returns NULL (finished) or TValue * (metamethod).
2777 | ldrb RBw, [PC, #-4+OFS_RB]
2779 | cbnz CRET1, ->vmeta_binop
2780 | ldr TMP0, [BASE, RB, lsl #3]
2781 | str TMP0, [BASE, RA, lsl #3] // Copy result to RA.
2785 /* -- Constant ops ------------------------------------------------------ */
2788 | // RA = dst, RC = str_const (~)
2790 | ldr TMP0, [KBASE, RC, lsl #3]
2791 | movn TMP1, #~LJ_TSTR
2792 | add TMP0, TMP0, TMP1, lsl #47
2793 | str TMP0, [BASE, RA, lsl #3]
2798 | // RA = dst, RC = cdata_const (~)
2800 | ldr TMP0, [KBASE, RC, lsl #3]
2801 | movn TMP1, #~LJ_TCDATA
2802 | add TMP0, TMP0, TMP1, lsl #47
2803 | str TMP0, [BASE, RA, lsl #3]
2808 | // RA = dst, RC = int16_literal
2810 | add_TISNUM TMP0, RC
2811 | str TMP0, [BASE, RA, lsl #3]
2815 | // RA = dst, RC = num_const
2816 | ldr TMP0, [KBASE, RC, lsl #3]
2817 | str TMP0, [BASE, RA, lsl #3]
2821 | // RA = dst, RC = primitive_type (~)
2822 | mvn TMP0, RC, lsl #47
2823 | str TMP0, [BASE, RA, lsl #3]
2827 | // RA = base, RC = end
2828 | add RA, BASE, RA, lsl #3
2829 | add RC, BASE, RC, lsl #3
2830 | str TISNIL, [RA], #8
2833 | str TISNIL, [RA], #8
2838 /* -- Upvalue and function ops ------------------------------------------ */
2841 | // RA = dst, RC = uvnum
2842 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2843 | add RC, RC, #offsetof(GCfuncL, uvptr)/8
2844 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2845 | ldr UPVAL:CARG2, [LFUNC:CARG2, RC, lsl #3]
2846 | ldr CARG2, UPVAL:CARG2->v
2848 | str TMP0, [BASE, RA, lsl #3]
2852 | // RA = uvnum, RC = src
2853 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2854 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2855 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2856 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2857 | ldr CARG3, [BASE, RC, lsl #3]
2858 | ldr CARG2, UPVAL:CARG1->v
2859 | ldrb TMP2w, UPVAL:CARG1->marked
2860 | ldrb TMP0w, UPVAL:CARG1->closed
2861 | asr ITYPE, CARG3, #47
2862 | str CARG3, [CARG2]
2863 | add ITYPE, ITYPE, #-LJ_TISGCV
2864 | tst TMP2w, #LJ_GC_BLACK // isblack(uv)
2865 | ccmp TMP0w, #0, #4, ne // && uv->closed
2866 | ccmn ITYPE, #-(LJ_TNUMX - LJ_TISGCV), #0, ne // && tvisgcv(v)
2871 |2: // Check if new value is white.
2872 | and GCOBJ:CARG3, CARG3, #LJ_GCVMASK
2873 | ldrb TMP1w, GCOBJ:CARG3->gch.marked
2874 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2876 | // Crossed a write barrier. Move the barrier forward.
2878 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2882 | // RA = uvnum, RC = str_const (~)
2883 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2884 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2886 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2887 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2888 | ldr STR:CARG3, [KBASE, RC, lsl #3]
2889 | movn TMP0, #~LJ_TSTR
2890 | ldr CARG2, UPVAL:CARG1->v
2891 | ldrb TMP2w, UPVAL:CARG1->marked
2892 | add TMP0, STR:CARG3, TMP0, lsl #47
2893 | ldrb TMP1w, STR:CARG3->marked
2895 | tbnz TMP2w, #2, >2 // isblack(uv)
2899 |2: // Check if string is white and ensure upvalue is closed.
2900 | ldrb TMP0w, UPVAL:CARG1->closed
2901 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2902 | ccmp TMP0w, #0, #4, ne
2904 | // Crossed a write barrier. Move the barrier forward.
2906 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2910 | // RA = uvnum, RC = num_const
2911 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2912 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2913 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2914 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2915 | ldr TMP0, [KBASE, RC, lsl #3]
2916 | ldr CARG2, UPVAL:CARG2->v
2921 | // RA = uvnum, RC = primitive_type (~)
2922 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2923 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2924 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2925 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2926 | mvn TMP0, RC, lsl #47
2927 | ldr CARG2, UPVAL:CARG2->v
2933 | // RA = level, RC = target
2934 | ldr CARG3, L->openupval
2935 | add RC, PC, RC, lsl #2
2937 | sub PC, RC, #0x20000
2940 | add CARG2, BASE, RA, lsl #3
2941 | bl extern lj_func_closeuv // (lua_State *L, TValue *level)
2948 | // RA = dst, RC = proto_const (~) (holding function prototype)
2951 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2953 | ldr CARG2, [KBASE, RC, lsl #3]
2955 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2956 | // (lua_State *L, GCproto *pt, GCfuncL *parent)
2957 | bl extern lj_func_newL_gc
2958 | // Returns GCfuncL *.
2960 | movn TMP0, #~LJ_TFUNC
2961 | add CRET1, CRET1, TMP0, lsl #47
2962 | str CRET1, [BASE, RA, lsl #3]
2966 /* -- Table ops --------------------------------------------------------- */
2970 | // RA = dst, RC = (hbits|asize) | tab_const (~)
2971 | ldp CARG3, CARG4, GL->gc.total // Assumes threshold follows total.
2978 if (op == BC_TNEW) {
2979 | and CARG2, RC, #0x7ff
2980 | lsr CARG3, RC, #11
2983 | csel CARG2, CARG2, TMP0, ne
2984 | bl extern lj_tab_new // (lua_State *L, int32_t asize, uint32_t hbits)
2985 | // Returns GCtab *.
2988 | ldr CARG2, [KBASE, RC, lsl #3]
2989 | bl extern lj_tab_dup // (lua_State *L, Table *kt)
2990 | // Returns GCtab *.
2993 | movk CRET1, #(LJ_TTAB>>1)&0xffff, lsl #48
2994 | str CRET1, [BASE, RA, lsl #3]
2998 | bl extern lj_gc_step_fixtop // (lua_State *L)
3004 | // RA = dst, RC = str_const (~)
3006 | // RA = src, RC = str_const (~)
3007 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3009 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3010 | ldr TAB:CARG2, LFUNC:CARG1->env
3011 | ldr STR:RC, [KBASE, RC, lsl #3]
3012 if (op == BC_GGET) {
3022 | // RA = dst, RB = table, RC = key
3023 | ldr CARG2, [BASE, RB, lsl #3]
3024 | ldr TMP1, [BASE, RC, lsl #3]
3025 | checktab CARG2, ->vmeta_tgetv
3026 | checkint TMP1, >9 // Integer key?
3027 | ldr CARG3, TAB:CARG2->array
3028 | ldr CARG1w, TAB:CARG2->asize
3029 | add CARG3, CARG3, TMP1, uxtw #3
3030 | cmp TMP1w, CARG1w // In array part?
3036 | str TMP0, [BASE, RA, lsl #3]
3039 |5: // Check for __index if table value is nil.
3040 | ldr TAB:CARG1, TAB:CARG2->metatable
3041 | cbz TAB:CARG1, <1 // No metatable: done.
3042 | ldrb TMP1w, TAB:CARG1->nomm
3043 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
3047 | asr ITYPE, TMP1, #47
3048 | cmn ITYPE, #-LJ_TSTR // String key?
3050 | and STR:RC, TMP1, #LJ_GCVMASK
3056 | // RA = dst, RB = table, RC = str_const (~)
3057 | ldr CARG2, [BASE, RB, lsl #3]
3059 | ldr STR:RC, [KBASE, RC, lsl #3]
3060 | checktab CARG2, ->vmeta_tgets1
3062 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst
3063 | ldr TMP1w, TAB:CARG2->hmask
3064 | ldr TMP2w, STR:RC->sid
3065 | ldr NODE:CARG3, TAB:CARG2->node
3066 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
3067 | add TMP1, TMP1, TMP1, lsl #1
3068 | movn CARG4, #~LJ_TSTR
3069 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
3070 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
3072 | ldp TMP0, CARG1, NODE:CARG3->val
3073 | ldr NODE:CARG3, NODE:CARG3->next
3079 | str TMP0, [BASE, RA, lsl #3]
3082 |4: // Follow hash chain.
3083 | cbnz NODE:CARG3, <1
3084 | // End of hash chain: key not found, nil result.
3087 |5: // Check for __index if table value is nil.
3088 | ldr TAB:CARG1, TAB:CARG2->metatable
3089 | cbz TAB:CARG1, <3 // No metatable: done.
3090 | ldrb TMP1w, TAB:CARG1->nomm
3091 | tbnz TMP1w, #MM_index, <3 // 'no __index' flag set: done.
3097 | // RA = dst, RB = table, RC = index
3098 | ldr CARG2, [BASE, RB, lsl #3]
3099 | checktab CARG2, ->vmeta_tgetb
3100 | ldr CARG3, TAB:CARG2->array
3101 | ldr CARG1w, TAB:CARG2->asize
3102 | add CARG3, CARG3, RC, lsl #3
3103 | cmp RCw, CARG1w // In array part?
3109 | str TMP0, [BASE, RA, lsl #3]
3112 |5: // Check for __index if table value is nil.
3113 | ldr TAB:CARG1, TAB:CARG2->metatable
3114 | cbz TAB:CARG1, <1 // No metatable: done.
3115 | ldrb TMP1w, TAB:CARG1->nomm
3116 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
3122 | // RA = dst, RB = table, RC = key
3123 | ldr CARG1, [BASE, RB, lsl #3]
3124 | ldr TMP1, [BASE, RC, lsl #3]
3125 | and TAB:CARG1, CARG1, #LJ_GCVMASK
3126 | ldr CARG3, TAB:CARG1->array
3127 | ldr TMP2w, TAB:CARG1->asize
3128 | add CARG3, CARG3, TMP1w, uxtw #3
3129 | cmp TMP1w, TMP2w // In array part?
3133 | str TMP0, [BASE, RA, lsl #3]
3140 | // RA = src, RB = table, RC = key
3141 | ldr CARG2, [BASE, RB, lsl #3]
3142 | ldr TMP1, [BASE, RC, lsl #3]
3143 | checktab CARG2, ->vmeta_tsetv
3144 | checkint TMP1, >9 // Integer key?
3145 | ldr CARG3, TAB:CARG2->array
3146 | ldr CARG1w, TAB:CARG2->asize
3147 | add CARG3, CARG3, TMP1, uxtw #3
3148 | cmp TMP1w, CARG1w // In array part?
3151 | ldr TMP0, [BASE, RA, lsl #3]
3152 | ldrb TMP2w, TAB:CARG2->marked
3153 | cmp_nil TMP1 // Previous value is nil?
3157 | tbnz TMP2w, #2, >7 // isblack(table)
3161 |5: // Check for __newindex if previous value is nil.
3162 | ldr TAB:CARG1, TAB:CARG2->metatable
3163 | cbz TAB:CARG1, <1 // No metatable: done.
3164 | ldrb TMP1w, TAB:CARG1->nomm
3165 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
3168 |7: // Possible table write barrier for the value. Skip valiswhite check.
3169 | barrierback TAB:CARG2, TMP2w, TMP1
3173 | asr ITYPE, TMP1, #47
3174 | cmn ITYPE, #-LJ_TSTR // String key?
3176 | and STR:RC, TMP1, #LJ_GCVMASK
3182 | // RA = dst, RB = table, RC = str_const (~)
3183 | ldr CARG2, [BASE, RB, lsl #3]
3185 | ldr STR:RC, [KBASE, RC, lsl #3]
3186 | checktab CARG2, ->vmeta_tsets1
3188 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src
3189 | ldr TMP1w, TAB:CARG2->hmask
3190 | ldr TMP2w, STR:RC->sid
3191 | ldr NODE:CARG3, TAB:CARG2->node
3192 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
3193 | add TMP1, TMP1, TMP1, lsl #1
3194 | movn CARG4, #~LJ_TSTR
3195 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
3196 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
3197 | strb wzr, TAB:CARG2->nomm // Clear metamethod cache.
3199 | ldp TMP1, CARG1, NODE:CARG3->val
3200 | ldr NODE:TMP3, NODE:CARG3->next
3201 | ldrb TMP2w, TAB:CARG2->marked
3204 | ldr TMP0, [BASE, RA, lsl #3]
3205 | cmp_nil TMP1 // Previous value is nil?
3208 | str TMP0, NODE:CARG3->val
3209 | tbnz TMP2w, #2, >7 // isblack(table)
3213 |4: // Check for __newindex if previous value is nil.
3214 | ldr TAB:CARG1, TAB:CARG2->metatable
3215 | cbz TAB:CARG1, <2 // No metatable: done.
3216 | ldrb TMP1w, TAB:CARG1->nomm
3217 | tbnz TMP1w, #MM_newindex, <2 // 'no __newindex' flag set: done.
3220 |5: // Follow hash chain.
3221 | mov NODE:CARG3, NODE:TMP3
3222 | cbnz NODE:TMP3, <1
3223 | // End of hash chain: key not found, add a new one.
3225 | // But check for __newindex first.
3226 | ldr TAB:CARG1, TAB:CARG2->metatable
3227 | cbz TAB:CARG1, >6 // No metatable: continue.
3228 | ldrb TMP1w, TAB:CARG1->nomm
3229 | // 'no __newindex' flag NOT set: check.
3230 | tbz TMP1w, #MM_newindex, ->vmeta_tsets
3232 | movn TMP1, #~LJ_TSTR
3234 | add TMP0, STR:RC, TMP1, lsl #47
3238 | add CARG3, sp, TMPDofs
3239 | bl extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k)
3240 | // Returns TValue *.
3242 | ldr TMP0, [BASE, RA, lsl #3]
3244 | b <3 // No 2nd write barrier needed.
3246 |7: // Possible table write barrier for the value. Skip valiswhite check.
3247 | barrierback TAB:CARG2, TMP2w, TMP1
3253 | // RA = src, RB = table, RC = index
3254 | ldr CARG2, [BASE, RB, lsl #3]
3255 | checktab CARG2, ->vmeta_tsetb
3256 | ldr CARG3, TAB:CARG2->array
3257 | ldr CARG1w, TAB:CARG2->asize
3258 | add CARG3, CARG3, RC, lsl #3
3259 | cmp RCw, CARG1w // In array part?
3262 | ldr TMP0, [BASE, RA, lsl #3]
3263 | ldrb TMP2w, TAB:CARG2->marked
3264 | cmp_nil TMP1 // Previous value is nil?
3268 | tbnz TMP2w, #2, >7 // isblack(table)
3272 |5: // Check for __newindex if previous value is nil.
3273 | ldr TAB:CARG1, TAB:CARG2->metatable
3274 | cbz TAB:CARG1, <1 // No metatable: done.
3275 | ldrb TMP1w, TAB:CARG1->nomm
3276 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
3279 |7: // Possible table write barrier for the value. Skip valiswhite check.
3280 | barrierback TAB:CARG2, TMP2w, TMP1
3286 | // RA = src, RB = table, RC = key
3287 | ldr CARG2, [BASE, RB, lsl #3]
3288 | ldr TMP1, [BASE, RC, lsl #3]
3289 | and TAB:CARG2, CARG2, #LJ_GCVMASK
3290 | ldr CARG1, TAB:CARG2->array
3291 | ldrb TMP2w, TAB:CARG2->marked
3292 | ldr CARG4w, TAB:CARG2->asize
3293 | add CARG1, CARG1, TMP1, uxtw #3
3294 | tbnz TMP2w, #2, >7 // isblack(table)
3296 | cmp TMP1w, CARG4w // In array part?
3299 | ldr TMP0, [BASE, RA, lsl #3]
3303 |7: // Possible table write barrier for the value. Skip valiswhite check.
3304 | barrierback TAB:CARG2, TMP2w, TMP0
3309 | // RA = base (table at base-1), RC = num_const (start index)
3310 | add RA, BASE, RA, lsl #3
3312 | ldr RBw, SAVE_MULTRES
3313 | ldr TAB:CARG2, [RA, #-8] // Guaranteed to be a table.
3314 | ldr TMP1, [KBASE, RC, lsl #3] // Integer constant is in lo-word.
3316 | cbz RB, >4 // Nothing to copy?
3317 | and TAB:CARG2, CARG2, #LJ_GCVMASK
3318 | ldr CARG1w, TAB:CARG2->asize
3319 | add CARG3w, TMP1w, RBw, lsr #3
3320 | ldr CARG4, TAB:CARG2->array
3324 | add TMP1, CARG4, TMP1w, uxtw #3
3325 | ldrb TMP2w, TAB:CARG2->marked
3326 |3: // Copy result slots to table.
3327 | ldr TMP0, [RA], #8
3328 | str TMP0, [TMP1], #8
3331 | tbnz TMP2w, #2, >7 // isblack(table)
3335 |5: // Need to resize array part.
3339 | bl extern lj_tab_reasize // (lua_State *L, GCtab *t, int nasize)
3340 | // Must not reallocate the stack.
3343 |7: // Possible table write barrier for any value. Skip valiswhite check.
3344 | barrierback TAB:CARG2, TMP2w, TMP1
3348 /* -- Calls and vararg handling ----------------------------------------- */
3351 | // RA = base, (RB = nresults+1,) RC = extra_nargs
3352 | ldr TMP0w, SAVE_MULTRES
3353 | decode_RC8RD NARGS8:RC, RC
3354 | add NARGS8:RC, NARGS8:RC, TMP0
3358 | decode_RC8RD NARGS8:RC, RC
3359 | // RA = base, (RB = nresults+1,) RC = (nargs+1)*8
3361 | mov RB, BASE // Save old BASE for vmeta_call.
3362 | add BASE, BASE, RA, lsl #3
3363 | ldr CARG3, [BASE], #16
3364 | sub NARGS8:RC, NARGS8:RC, #8
3365 | checkfunc CARG3, ->vmeta_call
3370 | // RA = base, (RB = 0,) RC = extra_nargs
3371 | ldr TMP0w, SAVE_MULTRES
3372 | add NARGS8:RC, TMP0, RC, lsl #3
3376 | lsl NARGS8:RC, RC, #3
3377 | // RA = base, (RB = 0,) RC = (nargs+1)*8
3379 | add RA, BASE, RA, lsl #3
3380 | ldr TMP1, [RA], #16
3381 | sub NARGS8:RC, NARGS8:RC, #8
3382 | checktp CARG3, TMP1, LJ_TFUNC, ->vmeta_callt
3383 | ldr PC, [BASE, FRAME_PC]
3386 | ldrb TMP2w, LFUNC:CARG3->ffid
3387 | tst PC, #FRAME_TYPE
3390 | str TMP1, [BASE, FRAME_FUNC] // Copy function down, but keep PC.
3393 | ldr TMP0, [RA, RB]
3395 | cmp TMP1, NARGS8:RC
3396 | str TMP0, [BASE, RB]
3400 | cmp TMP2, #1 // (> FF_C) Calling a fast function?
3405 |5: // Tailcall to a fast function with a Lua frame below.
3406 | ldrb RAw, [PC, #-4+OFS_RA]
3407 | sub CARG1, BASE, RA, lsl #3
3408 | ldr LFUNC:CARG1, [CARG1, #-32]
3409 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3410 | ldr CARG1, LFUNC:CARG1->pc
3411 | ldr KBASE, [CARG1, #PC2PROTO(k)]
3414 |7: // Tailcall from a vararg function.
3415 | eor PC, PC, #FRAME_VARG
3416 | tst PC, #FRAME_TYPEP // Vararg frame below?
3417 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3419 | sub BASE, BASE, PC
3420 | ldr PC, [BASE, FRAME_PC]
3421 | tst PC, #FRAME_TYPE
3422 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3427 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3428 | add RA, BASE, RA, lsl #3
3429 | ldr CARG3, [RA, #-24]
3430 | mov RB, BASE // Save old BASE for vmeta_call.
3431 | ldp CARG1, CARG2, [RA, #-16]
3433 | mov NARGS8:RC, #16 // Iterators get 2 arguments.
3434 | str CARG3, [RA] // Copy callable.
3435 | stp CARG1, CARG2, [RA, #16] // Copy state and control var.
3436 | checkfunc CARG3, ->vmeta_call
3445 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3446 | add RA, BASE, RA, lsl #3
3447 | ldr TAB:RB, [RA, #-16]
3448 | ldrh TMP3w, [PC, # OFS_RD]
3449 | ldr CARG1w, [RA, #-8+LO] // Get index from control var.
3451 | add TMP3, PC, TMP3, lsl #2
3452 | and TAB:RB, RB, #LJ_GCVMASK
3453 | sub TMP3, TMP3, #0x20000
3454 | ldr TMP1w, TAB:RB->asize
3455 | ldr CARG2, TAB:RB->array
3456 |1: // Traverse array part.
3457 | subs RC, CARG1, TMP1
3458 | add CARG3, CARG2, CARG1, lsl #3
3459 | bhs >5 // Index points after array part?
3462 | cinc CARG1, CARG1, eq // Skip holes in array part.
3464 | add_TISNUM CARG1, CARG1
3465 | stp CARG1, TMP0, [RA]
3466 | add CARG1, CARG1, #1
3468 | str CARG1w, [RA, #-8+LO] // Update control var.
3473 |5: // Traverse hash part.
3474 | ldr TMP2w, TAB:RB->hmask
3475 | ldr NODE:RB, TAB:RB->node
3477 | add CARG1, RC, RC, lsl #1
3478 | cmp RC, TMP2 // End of iteration? Branch to ITERN+1.
3479 | add NODE:CARG3, NODE:RB, CARG1, lsl #3 // node = tab->node + idx*3*8
3481 | ldp TMP0, CARG1, NODE:CARG3->val
3484 | beq <6 // Skip holes in hash part.
3485 | stp CARG1, TMP0, [RA]
3486 | add CARG1, RC, TMP1
3491 | // RA = base, RC = target (points to ITERN)
3492 | add RA, BASE, RA, lsl #3
3493 | ldr CFUNC:CARG1, [RA, #-24]
3494 | add RC, PC, RC, lsl #2
3495 | ldp TAB:CARG3, CARG4, [RA, #-16]
3496 | sub RC, RC, #0x20000
3497 | checkfunc CFUNC:CARG1, >5
3498 | asr TMP0, TAB:CARG3, #47
3499 | ldrb TMP1w, CFUNC:CARG1->ffid
3501 | ccmn TMP0, #-LJ_TTAB, #0, eq
3502 | ccmp TMP1w, #FF_next_N, #0, eq
3504 | mov TMP0w, #0xfffe7fff // LJ_KEYINDEX
3505 | lsl TMP0, TMP0, #32
3506 | str TMP0, [RA, #-8] // Initialize control var.
3511 |5: // Despecialize bytecode if any of the checks fail.
3513 | ldrb TMP2w, [RC, # OFS_OP]
3516 | mov TMP1, #BC_ITERC
3517 | strb TMP0w, [PC, #-4+OFS_OP]
3519 | cmp TMP2w, #BC_ITERN
3522 | strb TMP1w, [RC, # OFS_OP]
3525 |6: // Unpatch JLOOP.
3526 | ldr RA, [GL, #GL_J(trace)]
3527 | ldrh TMP2w, [RC, # OFS_RD]
3528 | ldr TRACE:RA, [RA, TMP2, lsl #3]
3529 | ldr TMP2w, TRACE:RA->startins
3530 | bfxil TMP2w, TMP1w, #0, #8
3539 | // RA = base, RB = (nresults+1), RC = numparams
3540 | ldr TMP1, [BASE, FRAME_PC]
3541 | add TMP0, BASE, RC, lsl #3
3542 | add RC, BASE, RA, lsl #3 // RC = destination
3543 | add TMP0, TMP0, #FRAME_VARG
3544 | add TMP2, RC, RB, lsl #3
3545 | sub RA, TMP0, TMP1 // RA = vbase
3546 | // Note: RA may now be even _above_ BASE if nargs was < numparams.
3547 | sub TMP3, BASE, #16 // TMP3 = vtop
3549 | sub TMP2, TMP2, #16
3550 |1: // Copy vararg slots to destination slots.
3552 | ldr TMP0, [RA], #8
3553 | csinv TMP0, TMP0, xzr, lo // TISNIL = ~xzr
3555 | str TMP0, [RC], #8
3560 |5: // Copy all varargs.
3561 | ldr TMP0, L->maxstack
3562 | subs TMP2, TMP3, RA
3563 | csel RB, xzr, TMP2, le // MULTRES = (max(vtop-vbase,0)+1)*8
3565 | add TMP1, RC, TMP2
3566 | str RBw, SAVE_MULTRES
3567 | ble <2 // Nothing to copy.
3571 | ldr TMP0, [RA], #8
3572 | str TMP0, [RC], #8
3577 |7: // Grow stack for varargs.
3578 | lsr CARG2, TMP2, #3
3579 | stp BASE, RC, L->base
3581 | sub RA, RA, BASE // Need delta, because BASE may change.
3583 | bl extern lj_state_growstack // (lua_State *L, int n)
3584 | ldp BASE, RC, L->base
3586 | sub TMP3, BASE, #16
3590 /* -- Returns ----------------------------------------------------------- */
3593 | // RA = results, RC = extra results
3594 | ldr TMP0w, SAVE_MULTRES
3595 | ldr PC, [BASE, FRAME_PC]
3596 | add RA, BASE, RA, lsl #3
3597 | add RC, TMP0, RC, lsl #3
3602 | // RA = results, RC = nresults+1
3603 | ldr PC, [BASE, FRAME_PC]
3605 | add RA, BASE, RA, lsl #3
3607 | str RCw, SAVE_MULTRES
3609 | ands CARG1, PC, #FRAME_TYPE
3610 | eor CARG2, PC, #FRAME_VARG
3614 | // BASE = base, RA = resultptr, RC = (nresults+1)*8, PC = return
3615 | ldr INSw, [PC, #-4]
3617 | sub CARG3, BASE, #16
3620 | ldr TMP0, [RA], #8
3621 | add BASE, BASE, #8
3622 | sub TMP1, TMP1, #8
3623 | str TMP0, [BASE, #-24]
3627 | sub CARG4, CARG3, RA, lsl #3
3629 | ldr LFUNC:CARG1, [CARG4, FRAME_FUNC]
3631 | cmp RC, RB, lsl #3 // More results expected?
3633 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3635 | ldr CARG2, LFUNC:CARG1->pc
3636 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3639 |6: // Fill up results with nil.
3640 | add BASE, BASE, #8
3642 | str TISNIL, [BASE, #-24]
3645 |->BC_RETV1_Z: // Non-standard return case.
3646 | add RA, BASE, RA, lsl #3
3648 | tst CARG2, #FRAME_TYPEP
3650 | // Return from vararg function: relocate BASE down.
3651 | sub BASE, BASE, CARG2
3652 | ldr PC, [BASE, FRAME_PC]
3656 case BC_RET0: case BC_RET1:
3657 | // RA = results, RC = nresults+1
3658 | ldr PC, [BASE, FRAME_PC]
3660 | str RCw, SAVE_MULTRES
3661 | ands CARG1, PC, #FRAME_TYPE
3662 | eor CARG2, PC, #FRAME_VARG
3664 | ldr INSw, [PC, #-4]
3665 if (op == BC_RET1) {
3666 | ldr TMP0, [BASE, RA, lsl #3]
3668 | sub CARG4, BASE, #16
3670 | sub BASE, CARG4, RA, lsl #3
3671 if (op == BC_RET1) {
3672 | str TMP0, [CARG4], #8
3675 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3677 | cmp RC, RB, lsl #3
3679 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3680 | ldr CARG2, LFUNC:CARG1->pc
3681 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3684 |6: // Fill up results with nil.
3686 | str TISNIL, [CARG4], #8
3690 /* -- Loops and branches ------------------------------------------------ */
3692 |.define FOR_IDX, [RA]; .define FOR_TIDX, [RA, #4]
3693 |.define FOR_STOP, [RA, #8]; .define FOR_TSTOP, [RA, #12]
3694 |.define FOR_STEP, [RA, #16]; .define FOR_TSTEP, [RA, #20]
3695 |.define FOR_EXT, [RA, #24]; .define FOR_TEXT, [RA, #28]
3701 | // Fall through. Assumes BC_IFORL follows.
3711 | // RA = base, RC = target (after end of loop or start of loop)
3712 vk = (op == BC_IFORL || op == BC_JFORL);
3713 | add RA, BASE, RA, lsl #3
3714 | ldp CARG1, CARG2, FOR_IDX // CARG1 = IDX, CARG2 = STOP
3715 | ldr CARG3, FOR_STEP // CARG3 = STEP
3716 if (op != BC_JFORL) {
3717 | add RC, PC, RC, lsl #2
3718 | sub RC, RC, #0x20000
3720 | checkint CARG1, >5
3722 | checkint CARG2, ->vmeta_for
3723 | checkint CARG3, ->vmeta_for
3724 | tbnz CARG3w, #31, >4
3725 | cmp CARG1w, CARG2w
3727 | adds CARG1w, CARG1w, CARG3w
3729 | add_TISNUM TMP0, CARG1
3730 | tbnz CARG3w, #31, >4
3731 | cmp CARG1w, CARG2w
3734 if (op == BC_FORI) {
3735 | csel PC, RC, PC, gt
3736 } else if (op == BC_JFORI) {
3738 | ldrh RCw, [RC, #-4+OFS_RD]
3739 } else if (op == BC_IFORL) {
3740 | csel PC, RC, PC, le
3746 | str CARG1, FOR_EXT
3748 if (op == BC_JFORI || op == BC_JFORL) {
3754 |4: // Invert check for negative step.
3755 | cmp CARG2w, CARG1w
3759 | ldp d0, d1, FOR_IDX
3762 | checknum CARG2, ->vmeta_for
3763 | checknum CARG3, ->vmeta_for
3769 | tbnz CARG3, #63, >7
3776 if (op == BC_FORI) {
3777 | csel PC, RC, PC, hi
3778 } else if (op == BC_JFORI) {
3779 | ldrh RCw, [RC, #-4+OFS_RD]
3781 } else if (op == BC_IFORL) {
3782 | csel PC, RC, PC, ls
3788 |7: // Invert check for negative step.
3797 | // Fall through. Assumes BC_IITERL follows.
3805 | // RA = base, RC = target
3806 | ldr CARG1, [BASE, RA, lsl #3]
3807 | add TMP1, BASE, RA, lsl #3
3809 | beq >1 // Stop if iterator returned nil.
3810 if (op == BC_JITERL) {
3811 | str CARG1, [TMP1, #-8]
3814 | add TMP0, PC, RC, lsl #2 // Otherwise save control var + branch.
3815 | sub PC, TMP0, #0x20000
3816 | str CARG1, [TMP1, #-8]
3823 | // RA = base, RC = target (loop extent)
3824 | // Note: RA/RC is only used by trace recorder to determine scope/extent
3825 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
3829 | // Fall through. Assumes BC_ILOOP follows.
3833 | // RA = base, RC = target (loop extent)
3839 | // RA = base (ignored), RC = traceno
3840 | ldr CARG1, [GL, #GL_J(trace)]
3841 | st_vmstate wzr // Traces on ARM64 don't store the trace #, so use 0.
3842 | ldr TRACE:RC, [CARG1, RC, lsl #3]
3844 | ldr RA, TRACE:RC->mcauth
3846 | ldr RA, TRACE:RC->mcode
3848 | str BASE, GL->jit_base
3849 | str L, GL->tmpbuf.L
3850 | sub sp, sp, #16 // See SPS_FIXED. Avoids sp adjust in every root trace.
3860 | // RA = base (only used by trace recorder), RC = target
3861 | add RC, PC, RC, lsl #2
3862 | sub PC, RC, #0x20000
3866 /* -- Function headers -------------------------------------------------- */
3872 case BC_FUNCV: /* NYI: compiled vararg functions. */
3873 | // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
3881 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3882 | ldr CARG1, L->maxstack
3883 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3884 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3886 | bhi ->vm_growstack_l
3888 | cmp NARGS8:RC, TMP1, lsl #3 // Check for missing parameters.
3890 if (op == BC_JFUNCF) {
3897 |3: // Clear missing parameters.
3898 | str TISNIL, [BASE, NARGS8:RC]
3899 | add NARGS8:RC, NARGS8:RC, #8
3907 | NYI // NYI: compiled vararg functions
3908 break; /* NYI: compiled vararg functions. */
3911 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3912 | ldr CARG1, L->maxstack
3913 | movn TMP0, #~LJ_TFUNC
3914 | add TMP2, BASE, RC
3915 | add LFUNC:CARG3, CARG3, TMP0, lsl #47
3917 | sub CARG1, CARG1, #8
3918 | add TMP0, RC, #16+FRAME_VARG
3919 | str LFUNC:CARG3, [TMP2], #8 // Store (tagged) copy of LFUNC.
3920 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3922 | str TMP0, [TMP2], #8 // Store delta + FRAME_VARG.
3923 | bhs ->vm_growstack_l
3925 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3930 | cmp RA, RC // Less args than parameters?
3933 | sub TMP1, TMP1, #1
3934 | str TISNIL, [RA], #8 // Clear old fixarg slot (help the GC).
3935 | str TMP0, [TMP2], #8
3941 | sub TMP1, TMP1, #1
3942 | str TISNIL, [TMP2], #8
3949 | // BASE = new base, RA = BASE+framesize*8, CARG3 = CFUNC, RC = nargs*8
3950 if (op == BC_FUNCC) {
3951 | ldr CARG4, CFUNC:CARG3->f
3953 | ldr CARG4, GL->wrapf
3955 | add CARG2, RA, NARGS8:RC
3956 | ldr CARG1, L->maxstack
3957 | add RC, BASE, NARGS8:RC
3959 | stp BASE, RC, L->base
3960 if (op == BC_FUNCCW) {
3961 | ldr CARG2, CFUNC:CARG3->f
3963 | mv_vmstate TMP0w, C
3965 | bhi ->vm_growstack_c // Need to grow stack.
3967 | blr_auth CARG4 // (lua_State *L [, lua_CFunction f])
3968 | // Returns nresults.
3969 | ldp BASE, TMP1, L->base
3971 | sbfiz RC, CRET1, #3, #32
3972 | st_vmstate ST_INTERP
3973 | ldr PC, [BASE, FRAME_PC]
3974 | sub RA, TMP1, RC // RA = L->top - nresults*8
3978 /* ---------------------------------------------------------------------- */
3981 fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
3987 static int build_backend(BuildCtx *ctx)
3991 dasm_growpc(Dst, BC__MAX);
3993 build_subroutines(ctx);
3996 for (op = 0; op < BC__MAX; op++)
3997 build_ins(ctx, (BCOp)op, op);
4002 /* Emit pseudo frame-info for all assembler functions. */
4003 static void emit_asm_debug(BuildCtx *ctx)
4005 int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
4007 switch (ctx->mode) {
4009 fprintf(ctx->fp, "\t.section .debug_frame,\"\",%%progbits\n");
4012 "\t.long .LECIE0-.LSCIE0\n"
4014 "\t.long 0xffffffff\n"
4019 "\t.byte 30\n" /* Return address is in lr. */
4020 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4025 "\t.long .LEFDE0-.LASFDE0\n"
4027 "\t.long .Lframe0\n"
4030 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4031 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4033 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4034 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4035 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4036 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4037 64+i, i+(3+(28-19+1)-8));
4044 "\t.long .LEFDE1-.LASFDE1\n"
4046 "\t.long .Lframe0\n"
4047 "\t.quad lj_vm_ffi_call\n"
4049 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4050 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4051 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4052 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4054 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
4057 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",%%progbits\n");
4060 "\t.long .LECIE1-.LSCIE1\n"
4064 "\t.string \"zPR\"\n"
4067 "\t.byte 30\n" /* Return address is in lr. */
4068 "\t.uleb128 6\n" /* augmentation length */
4069 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4070 "\t.long lj_err_unwind_dwarf-.\n"
4071 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4072 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4077 "\t.long .LEFDE2-.LASFDE2\n"
4079 "\t.long .LASFDE2-.Lframe1\n"
4080 "\t.long .Lbegin-.\n"
4082 "\t.uleb128 0\n" /* augmentation length */
4083 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4084 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4086 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4087 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4088 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4089 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4090 64+i, i+(3+(28-19+1)-8));
4097 "\t.long .LECIE2-.LSCIE2\n"
4101 "\t.string \"zR\"\n"
4104 "\t.byte 30\n" /* Return address is in lr. */
4105 "\t.uleb128 1\n" /* augmentation length */
4106 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4107 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4112 "\t.long .LEFDE3-.LASFDE3\n"
4114 "\t.long .LASFDE3-.Lframe2\n"
4115 "\t.long lj_vm_ffi_call-.\n"
4117 "\t.uleb128 0\n" /* augmentation length */
4118 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4119 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4120 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4121 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4123 ".LEFDE3:\n\n", (int)ctx->codesz - fcofs);
4128 case BUILD_machasm: {
4133 fprintf(ctx->fp, "\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support\n");
4136 "\t.set L$set$x,LECIEX-LSCIEX\n"
4141 "\t.ascii \"zPR\\0\"\n"
4144 "\t.byte 30\n" /* Return address is in lr. */
4145 "\t.uleb128 6\n" /* augmentation length */
4146 "\t.byte 0x9b\n" /* indirect|pcrel|sdata4 */
4147 "\t.long _lj_err_unwind_dwarf@GOT-.\n"
4148 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4149 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4152 for (j = 0; j < ctx->nsym; j++) {
4153 const char *name = ctx->sym[j].name;
4154 int32_t size = ctx->sym[j+1].ofs - ctx->sym[j].ofs;
4155 if (size == 0) continue;
4157 if (!strcmp(name, "_lj_vm_ffi_call")) { fcsize = size; continue; }
4161 "\t.set L$set$%d,LEFDE%d-LASFDE%d\n"
4162 "\t.long L$set$%d\n"
4164 "\t.long LASFDE%d-EH_frame1\n"
4167 "\t.uleb128 0\n" /* augmentation length */
4168 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4169 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4170 j, j, j, j, j, j, j, name, size);
4171 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4172 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4173 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4174 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4175 64+i, i+(3+(28-19+1)-8));
4184 "\t.set L$set$y,LECIEY-LSCIEY\n"
4189 "\t.ascii \"zR\\0\"\n"
4192 "\t.byte 30\n" /* Return address is in lr. */
4193 "\t.uleb128 1\n" /* augmentation length */
4194 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4195 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4200 "\t.set L$set$yy,LEFDEY-LASFDEY\n"
4201 "\t.long L$set$yy\n"
4203 "\t.long LASFDEY-EH_frame2\n"
4204 "\t.long _lj_vm_ffi_call-.\n"
4206 "\t.uleb128 0\n" /* augmentation length */
4207 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4208 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4209 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4210 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4212 "LEFDEY:\n\n", fcsize);
4215 fprintf(ctx->fp, ".subsections_via_symbols\n");