1 |// Low-level VM code for ARM64 CPUs.
2 |// Bytecode interpreter, fast functions and helper functions.
3 |// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
6 |.section code_op, code_sub
8 |.actionlist build_actionlist
10 |.globalnames globnames
11 |.externnames extnames
13 |// Note: The ragged indentation of the instructions is intentional.
14 |// The starting columns indicate data dependencies.
16 |//-----------------------------------------------------------------------
18 |// ARM64 registers and the AAPCS64 ABI 1.0 at a glance:
20 |// x0-x17 temp, x19-x28 callee-saved, x29 fp, x30 lr
21 |// x18 is reserved on most platforms. Don't use it, save it or restore it.
22 |// x31 doesn't exist. Register number 31 either means xzr/wzr (zero) or sp,
23 |// depending on the instruction.
24 |// v0-v7 temp, v8-v15 callee-saved (only d8-d15 preserved), v16-v31 temp
26 |// x0-x7/v0-v7 hold parameters and results.
28 |// Fixed register assignments for the interpreter.
30 |// The following must be C callee-save.
31 |.define BASE, x19 // Base of current Lua stack frame.
32 |.define KBASE, x20 // Constants of current Lua function.
33 |.define PC, x21 // Next PC.
34 |.define GLREG, x22 // Global state.
35 |.define LREG, x23 // Register holding lua_State (also in SAVE_L).
36 |.define TISNUM, x24 // Constant LJ_TISNUM << 47.
37 |.define TISNUMhi, x25 // Constant LJ_TISNUM << 15.
38 |.define TISNIL, x26 // Constant -1LL.
39 |.define fp, x29 // Yes, we have to maintain a frame pointer.
41 |.define ST_INTERP, w26 // Constant -1.
43 |// The following temporaries are not saved across C calls, except for RA/RC.
62 |// Calling conventions. Also used as temporaries.
80 |//-----------------------------------------------------------------------
82 |// ARM64e pointer authentication codes (PAC).
84 |.macro sp_auth; pacibsp; .endmacro
85 |.macro br_auth, reg; braaz reg; .endmacro
86 |.macro blr_auth, reg; blraaz reg; .endmacro
87 |.macro ret_auth; retab; .endmacro
89 |.macro sp_auth; .endmacro
90 |.macro br_auth, reg; br reg; .endmacro
91 |.macro blr_auth, reg; blr reg; .endmacro
92 |.macro ret_auth; ret; .endmacro
95 |//-----------------------------------------------------------------------
97 |// Stack layout while in interpreter. Must match with lj_frame.h.
99 |.define CFRAME_SPACE, 208
100 |//----- 16 byte aligned, <-- sp entering interpreter
101 |.define SAVE_FP_LR_, 192
102 |.define SAVE_GPR_, 112 // 112+10*8: 64 bit GPR saves
103 |.define SAVE_FPR_, 48 // 48+8*8: 64 bit FPR saves
104 |// Unused [sp, #44] // 32 bit values
105 |.define SAVE_NRES, [sp, #40]
106 |.define SAVE_ERRF, [sp, #36]
107 |.define SAVE_MULTRES, [sp, #32]
108 |.define TMPD, [sp, #24] // 64 bit values
109 |.define SAVE_L, [sp, #16]
110 |.define SAVE_PC, [sp, #8]
111 |.define SAVE_CFRAME, [sp, #0]
112 |//----- 16 byte aligned, <-- sp while in interpreter.
114 |.define TMPDofs, #24
117 |// Windows unwind data is suited to r1 stored first.
118 |.macro stp_unwind, r1, r2, where
121 |.macro ldp_unwind, r1, r2, where
124 |.macro ldp_unwind, r1, r2, where, post_index
125 | ldp r1, r2, where, post_index
128 |// Otherwise store r2 first for compact unwind info (OSX).
129 |.macro stp_unwind, r1, r2, where
132 |.macro ldp_unwind, r1, r2, where
135 |.macro ldp_unwind, r1, r2, where, post_index
136 | ldp r2, r1, where, post_index
140 |.macro save_, gpr1, gpr2, fpr1, fpr2
141 | stp_unwind d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(14-fpr1)*8]
142 | stp_unwind x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(27-gpr1)*8]
144 |.macro rest_, gpr1, gpr2, fpr1, fpr2
145 | ldp_unwind d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(14-fpr1)*8]
146 | ldp_unwind x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(27-gpr1)*8]
151 | sub sp, sp, # CFRAME_SPACE
152 | stp fp, lr, [sp, # SAVE_FP_LR_]
153 | add fp, sp, # SAVE_FP_LR_
154 | stp_unwind x19, x20, [sp, # SAVE_GPR_+(27-19)*8]
156 | save_ 23, 24, 10, 11
157 | save_ 25, 26, 12, 13
158 | save_ 27, 28, 14, 15
161 | ldp_unwind x19, x20, [sp, # SAVE_GPR_+(27-19)*8]
163 | rest_ 23, 24, 10, 11
164 | rest_ 25, 26, 12, 13
165 | rest_ 27, 28, 14, 15
166 | ldp fp, lr, [sp, # SAVE_FP_LR_]
167 | add sp, sp, # CFRAME_SPACE
170 |// Type definitions. Some of these are only used for documentation.
171 |.type L, lua_State, LREG
172 |.type GL, global_State, GLREG
173 |.type TVALUE, TValue
177 |.type LFUNC, GCfuncL
178 |.type CFUNC, GCfuncC
179 |.type PROTO, GCproto
180 |.type UPVAL, GCupval
183 |.type TRACE, GCtrace
186 |//-----------------------------------------------------------------------
188 |// Trap for not-yet-implemented parts.
189 |.macro NYI; brk; .endmacro
191 |//-----------------------------------------------------------------------
193 |// Access to frame relative to BASE.
194 |.define FRAME_FUNC, #-16
195 |.define FRAME_PC, #-8
197 |// Endian-specific defines.
212 |.macro decode_RA, dst, ins; ubfx dst, ins, #8, #8; .endmacro
213 |.macro decode_RB, dst, ins; ubfx dst, ins, #24, #8; .endmacro
214 |.macro decode_RC, dst, ins; ubfx dst, ins, #16, #8; .endmacro
215 |.macro decode_RD, dst, ins; ubfx dst, ins, #16, #16; .endmacro
216 |.macro decode_RC8RD, dst, src; ubfiz dst, src, #3, #8; .endmacro
218 |// Instruction decode+dispatch.
221 | add TMP1, GL, INS, uxtb #3
223 | ldr TMP0, [TMP1, #GG_G2DISP]
228 |// Instruction footer.
230 | // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
231 | .define ins_next, ins_NEXT
232 | .define ins_next_, ins_NEXT
234 | // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
235 | // Affects only certain kinds of benchmarks (and only with -j off).
245 |// Call decode and dispatch.
247 | // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
248 | ldr PC, LFUNC:CARG3->pc
250 | add TMP1, GL, INS, uxtb #3
252 | ldr TMP0, [TMP1, #GG_G2DISP]
253 | add RA, BASE, RA, lsl #3
258 | // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
259 | str PC, [BASE, FRAME_PC]
263 |//-----------------------------------------------------------------------
265 |// Macros to check the TValue type and extract the GCobj. Branch on failure.
266 |.macro checktp, reg, tp, target
267 | asr ITYPE, reg, #47
269 | and reg, reg, #LJ_GCVMASK
272 |.macro checktp, dst, reg, tp, target
273 | asr ITYPE, reg, #47
275 | and dst, reg, #LJ_GCVMASK
278 |.macro checkstr, reg, target; checktp reg, LJ_TSTR, target; .endmacro
279 |.macro checktab, reg, target; checktp reg, LJ_TTAB, target; .endmacro
280 |.macro checkfunc, reg, target; checktp reg, LJ_TFUNC, target; .endmacro
281 |.macro checkint, reg, target
282 | cmp TISNUMhi, reg, lsr #32
285 |.macro checknum, reg, target
286 | cmp TISNUMhi, reg, lsr #32
289 |.macro checknumber, reg, target
290 | cmp TISNUMhi, reg, lsr #32
294 |.macro init_constants
296 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
297 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
300 |.macro mov_false, reg; movn reg, #0x8000, lsl #32; .endmacro
301 |.macro mov_true, reg; movn reg, #0x0001, lsl #48; .endmacro
302 |.macro mov_nil, reg; mov reg, TISNIL; .endmacro
303 |.macro cmp_nil, reg; cmp reg, TISNIL; .endmacro
304 |.macro add_TISNUM, dst, src; add dst, src, TISNUM; .endmacro
306 #define GL_J(field) (GG_G2J + (int)offsetof(jit_State, field))
308 #define PC2PROTO(field) ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
310 |.macro hotcheck, delta
312 | and CARG1, CARG1, #126
313 | add CARG1, CARG1, #GG_G2DISP+GG_DISP2HOT
314 | ldrh CARG2w, [GL, CARG1]
315 | subs CARG2, CARG2, #delta
316 | strh CARG2w, [GL, CARG1]
320 | hotcheck HOTCOUNT_LOOP
325 | hotcheck HOTCOUNT_CALL
329 |// Set current VM state.
330 |.macro mv_vmstate, reg, st; movn reg, #LJ_VMST_..st; .endmacro
331 |.macro st_vmstate, reg; str reg, GL->vmstate; .endmacro
333 |// Move table write barrier back. Overwrites mark and tmp.
334 |.macro barrierback, tab, mark, tmp
335 | ldr tmp, GL->gc.grayagain
336 | and mark, mark, #~LJ_GC_BLACK // black2gray(tab)
337 | str tab, GL->gc.grayagain
338 | strb mark, tab->marked
339 | str tmp, tab->gclist
342 |//-----------------------------------------------------------------------
345 #error "Only dual-number mode supported for ARM64 target"
348 /* Generate subroutines used by opcodes and other parts of the VM. */
349 /* The .code_sub section should be last to help static branch prediction. */
350 static void build_subroutines(BuildCtx *ctx)
354 |//-----------------------------------------------------------------------
355 |//-- Return handling ----------------------------------------------------
356 |//-----------------------------------------------------------------------
359 | // See vm_return. Also: RB = previous base.
360 | tbz PC, #2, ->cont_dispatch // (PC & FRAME_P) == 0?
362 | // Return from pcall or xpcall fast func.
363 | ldr PC, [RB, FRAME_PC] // Fetch PC of previous frame.
366 | // Prepending may overwrite the pcall frame, so do it at the end.
367 | str TMP0, [RA, #-8]! // Prepend true to results.
370 | adds RC, RC, #8 // RC = (nresults+1)*8.
371 | mov CRET1, #LUA_YIELD
372 | beq ->vm_unwind_c_eh
373 | str RCw, SAVE_MULTRES
374 | ands CARG1, PC, #FRAME_TYPE
375 | beq ->BC_RET_Z // Handle regular return to Lua.
378 | // BASE = base, RA = resultptr, RC/MULTRES = (nresults+1)*8, PC = return
379 | // CARG1 = PC & FRAME_TYPE
380 | and RB, PC, #~FRAME_TYPEP
381 | cmp CARG1, #FRAME_C
382 | sub RB, BASE, RB // RB = previous base.
386 | ldrsw CARG2, SAVE_NRES // CARG2 = nresults+1.
387 | mv_vmstate TMP0w, C
388 | sub BASE, BASE, #16
393 | subs TMP2, TMP2, #8
395 | str TMP0, [BASE], #8
398 | cmp RC, CARG2, lsl #3 // More/less results wanted?
401 | str BASE, L->top // Store new top.
404 | ldr RC, SAVE_CFRAME // Restore previous C frame.
405 | mov CRET1, #0 // Ok return status for vm_pcall.
413 | bgt >7 // Less results wanted?
414 | // More results wanted. Check stack size and fill up results with nil.
415 | ldr CARG3, L->maxstack
418 | str TISNIL, [BASE], #8
422 |7: // Less results wanted.
423 | cbz CARG2, <3 // LUA_MULTRET+1 case?
424 | sub CARG1, RC, CARG2, lsl #3
425 | sub BASE, BASE, CARG1 // Shrink top.
428 |8: // Corner case: need to grow stack for filling up results.
429 | // This can happen if:
430 | // - A C function grows the stack (a lot).
431 | // - The GC shrinks the stack in between.
432 | // - A return back from a lua_call() with (high) nresults adjustment.
433 | str BASE, L->top // Save current top held in BASE (yes).
435 | bl extern lj_state_growstack // (lua_State *L, int n)
436 | ldr BASE, L->top // Need the (realloced) L->top in BASE.
437 | ldrsw CARG2, SAVE_NRES
440 |->vm_unwind_c: // Unwind C stack, return from vm_pcall.
441 | // (void *cframe, int errcode)
442 | add fp, CARG1, # SAVE_FP_LR_
447 |->vm_unwind_c_eh: // Landing pad for external unwinder.
448 | mv_vmstate TMP0w, C
452 |->vm_unwind_ff: // Unwind C stack, return from ff pcall.
454 | add fp, CARG1, # SAVE_FP_LR_
458 | ldr GL, L->glref // Setup pointer to global state.
459 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
460 | mov RC, #16 // 2 results: false + error message.
463 | sub RA, BASE, #8 // Results start at BASE-8.
464 | ldr PC, [BASE, FRAME_PC] // Fetch PC of previous frame.
465 | str TMP0, [BASE, #-8] // Prepend false to error message.
466 | st_vmstate ST_INTERP
469 |//-----------------------------------------------------------------------
470 |//-- Grow stack for calls -----------------------------------------------
471 |//-----------------------------------------------------------------------
473 |->vm_growstack_c: // Grow stack for C function.
475 | mov CARG2, #LUA_MINSTACK
478 |->vm_growstack_l: // Grow stack for Lua function.
479 | // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
483 | stp BASE, RC, L->base
484 | add PC, PC, #4 // Must point after first instruction.
487 | // L->base = new base, L->top = top
489 | bl extern lj_state_growstack // (lua_State *L, int n)
490 | ldp BASE, RC, L->base
491 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
492 | sub NARGS8:RC, RC, BASE
493 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
494 | // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
495 | ins_callt // Just retry the call.
497 |//-----------------------------------------------------------------------
498 |//-- Entry points into the assembler VM ---------------------------------
499 |//-----------------------------------------------------------------------
501 |->vm_resume: // Setup C frame and resume thread.
502 | // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
505 | ldr GL, L->glref // Setup pointer to global state.
510 | add TMP0, sp, #CFRAME_RESUME
511 | ldrb TMP1w, L->status
513 | str L, SAVE_PC // Any value outside of bytecode is ok.
514 | str xzr, SAVE_CFRAME
515 | str TMP0, L->cframe
518 | // Resume after yield (like a return).
521 | ldp BASE, CARG1, L->base
523 | ldr PC, [BASE, FRAME_PC]
524 | strb wzr, L->status
525 | sub RC, CARG1, BASE
526 | ands CARG1, PC, #FRAME_TYPE
528 | st_vmstate ST_INTERP
529 | str RCw, SAVE_MULTRES
533 |->vm_pcall: // Setup protected C frame and enter VM.
534 | // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
537 | str CARG4w, SAVE_ERRF
540 |->vm_call: // Setup C frame and enter VM.
541 | // (lua_State *L, TValue *base, int nres1)
545 |1: // Entry point for vm_pcall above (PC = ftype).
546 | ldr RC, L:CARG1->cframe
547 | str CARG3w, SAVE_NRES
550 | ldr GL, L->glref // Setup pointer to global state.
552 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
554 | str RC, SAVE_CFRAME
555 | str TMP0, L->cframe // Add our C frame to cframe chain.
557 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
559 | ldp RB, CARG1, L->base // RB = old base (for vmeta_call).
562 | sub PC, PC, RB // PC = frame delta + frame type
563 | sub NARGS8:RC, CARG1, BASE
564 | st_vmstate ST_INTERP
567 | // RB = old base, BASE = new base, RC = nargs*8, PC = caller PC
568 | ldr CARG3, [BASE, FRAME_FUNC]
569 | checkfunc CARG3, ->vmeta_call
571 |->vm_call_dispatch_f:
573 | // BASE = new base, CARG3 = func, RC = nargs*8, PC = caller PC
575 |->vm_cpcall: // Setup protected C frame, call C.
576 | // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
579 | ldr RA, L:CARG1->stack
581 | ldr GL, L->glref // Setup pointer to global state.
583 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
585 | sub RA, RA, RB // Compute -savestack(L, L->top).
586 | str RAw, SAVE_NRES // Neg. delta means cframe w/o frame.
587 | str wzr, SAVE_ERRF // No error function.
589 | str RC, SAVE_CFRAME
590 | str TMP0, L->cframe // Add our C frame to cframe chain.
592 | blr_auth CARG4 // (lua_State *L, lua_CFunction func, void *ud)
595 | cbnz BASE, <3 // Else continue with the call.
596 | b ->vm_leave_cp // No base? Just remove C frame.
598 |//-----------------------------------------------------------------------
599 |//-- Metamethod handling ------------------------------------------------
600 |//-----------------------------------------------------------------------
602 |//-- Continuation dispatch ----------------------------------------------
605 | // BASE = meta base, RA = resultptr, RC = (nresults+1)*8
606 | ldr LFUNC:CARG3, [RB, FRAME_FUNC]
607 | ldr CARG1, [BASE, #-32] // Get continuation.
609 | mov BASE, RB // Restore caller BASE.
610 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
614 | ldr PC, [CARG4, #-24] // Restore PC from [cont|PC].
616 | str TISNIL, [TMP0, #-8] // Ensure one valid arg.
620 | ldr CARG3, LFUNC:CARG3->pc
621 | ldr KBASE, [CARG3, #PC2PROTO(k)]
622 | // BASE = base, RA = resultptr, CARG4 = meta base
627 | beq ->cont_ffi_callback // cont = 1: return from FFI callback.
628 | // cont = 0: tailcall from C function.
629 | sub CARG4, CARG4, #32
630 | sub RC, CARG4, BASE
634 |->cont_cat: // RA = resultptr, CARG4 = meta base
635 | ldr INSw, [PC, #-4]
636 | sub CARG2, CARG4, #32
641 | add TMP1, BASE, RB, lsl #3
642 | subs TMP1, CARG2, TMP1
645 | lsr CARG3, TMP1, #3
649 | str TMP0, [BASE, RA, lsl #3]
652 |//-- Table indexing metamethods -----------------------------------------
655 | movn CARG4, #~LJ_TSTR
656 | add CARG2, BASE, RB, lsl #3
657 | add CARG4, STR:RC, CARG4, lsl #47
661 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
662 | str CARG2, GL->tmptv
663 | add CARG2, GL, #offsetof(global_State, tmptv)
665 | add CARG3, sp, TMPDofs
669 |->vmeta_tgetb: // RB = table, RC = index
671 | add CARG2, BASE, RB, lsl #3
672 | add CARG3, sp, TMPDofs
676 |->vmeta_tgetv: // RB = table, RC = key
677 | add CARG2, BASE, RB, lsl #3
678 | add CARG3, BASE, RC, lsl #3
683 | bl extern lj_meta_tget // (lua_State *L, TValue *o, TValue *k)
684 | // Returns TValue * (finished) or NULL (metamethod).
687 | str TMP0, [BASE, RA, lsl #3]
690 |3: // Call __index metamethod.
691 | // BASE = base, L->top = new base, stack = cont/func/t/k
692 | sub TMP1, BASE, #FRAME_CONT
694 | mov NARGS8:RC, #16 // 2 args for func(t, k).
695 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
696 | str PC, [BASE, #-24] // [cont|PC]
698 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
699 | b ->vm_call_dispatch_f
703 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
704 | // Returns cTValue * or NULL.
706 | cbz CRET1, ->BC_TGETR_Z
710 |//-----------------------------------------------------------------------
713 | movn CARG4, #~LJ_TSTR
714 | add CARG2, BASE, RB, lsl #3
715 | add CARG4, STR:RC, CARG4, lsl #47
719 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
720 | str CARG2, GL->tmptv
721 | add CARG2, GL, #offsetof(global_State, tmptv)
723 | add CARG3, sp, TMPDofs
727 |->vmeta_tsetb: // RB = table, RC = index
729 | add CARG2, BASE, RB, lsl #3
730 | add CARG3, sp, TMPDofs
735 | add CARG2, BASE, RB, lsl #3
736 | add CARG3, BASE, RC, lsl #3
741 | bl extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
742 | // Returns TValue * (finished) or NULL (metamethod).
743 | ldr TMP0, [BASE, RA, lsl #3]
745 | // NOBARRIER: lj_meta_tset ensures the table is not black.
749 |3: // Call __newindex metamethod.
750 | // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
751 | sub TMP1, BASE, #FRAME_CONT
753 | mov NARGS8:RC, #24 // 3 args for func(t, k, v).
754 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
755 | str TMP0, [BASE, #16] // Copy value to third argument.
756 | str PC, [BASE, #-24] // [cont|PC]
758 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
759 | b ->vm_call_dispatch_f
766 | bl extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
767 | // Returns TValue *.
770 |//-- Comparison metamethods ---------------------------------------------
773 | add CARG2, BASE, RA, lsl #3
775 | add CARG3, BASE, RC, lsl #3
780 | bl extern lj_meta_comp // (lua_State *L, TValue *o1, *o2, int op)
781 | // Returns 0/1 or TValue * (metamethod).
786 | ldrh RBw, [PC, # OFS_RD]
788 | add RB, PC, RB, lsl #2
789 | sub RB, RB, #0x20000
790 | csel PC, PC, RB, lo
794 |->cont_ra: // RA = resultptr
795 | ldr INSw, [PC, #-4]
797 | decode_RA TMP1, INS
798 | str TMP0, [BASE, TMP1, lsl #3]
801 |->cont_condt: // RA = resultptr
804 | cmp TMP1, TMP0 // Branch if result is true.
807 |->cont_condf: // RA = resultptr
810 | cmp TMP0, TMP1 // Branch if result is false.
814 | // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
815 | and TAB:CARG3, CARG3, #LJ_GCVMASK
820 | bl extern lj_meta_equal // (lua_State *L, GCobj *o1, *o2, int ne)
821 | // Returns 0/1 or TValue * (metamethod).
831 | bl extern lj_meta_equal_cd // (lua_State *L, BCIns op)
832 | // Returns 0/1 or TValue * (metamethod).
843 | bl extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
846 |//-- Arithmetic metamethods ---------------------------------------------
849 | add CARG3, BASE, RB, lsl #3
850 | add CARG4, KBASE, RC, lsl #3
854 | add CARG4, BASE, RB, lsl #3
855 | add CARG3, KBASE, RC, lsl #3
859 | add CARG3, BASE, RC, lsl #3
864 | add CARG3, BASE, RB, lsl #3
865 | add CARG4, BASE, RC, lsl #3
868 | add CARG2, BASE, RA, lsl #3
872 | bl extern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
873 | // Returns NULL (finished) or TValue * (metamethod).
874 | cbz CRET1, ->cont_nop
876 | // Call metamethod for binary op.
878 | // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
879 | sub TMP1, CRET1, BASE
880 | str PC, [CRET1, #-24] // [cont|PC]
881 | add PC, TMP1, #FRAME_CONT
883 | mov NARGS8:RC, #16 // 2 args for func(o1, o2).
884 | b ->vm_call_dispatch
887 | add CARG2, BASE, RC, lsl #3
889 | mov TAB:RC, TAB:CARG1 // Save table (ignored for other types).
894 | bl extern lj_meta_len // (lua_State *L, TValue *o)
895 | // Returns NULL (retry) or TValue * (metamethod base).
897 | cbnz CRET1, ->vmeta_binop // Binop call for compatibility.
898 | mov TAB:CARG1, TAB:RC
901 | b ->vmeta_binop // Binop call for compatibility.
904 |//-- Call metamethod ----------------------------------------------------
906 |->vmeta_call: // Resolve and call __call metamethod.
907 | // RB = old base, BASE = new base, RC = nargs*8
909 | str RB, L->base // This is the callers base!
910 | sub CARG2, BASE, #16
912 | add CARG3, BASE, NARGS8:RC
913 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
914 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
915 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
916 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
919 |->vmeta_callt: // Resolve __call for BC_CALLT.
920 | // BASE = old base, RA = new base, RC = nargs*8
925 | add CARG3, RA, NARGS8:RC
926 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
927 | ldr TMP1, [RA, FRAME_FUNC] // Guaranteed to be a function here.
928 | ldr PC, [BASE, FRAME_PC]
929 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
930 | and LFUNC:CARG3, TMP1, #LJ_GCVMASK
933 |//-- Argument coercion for 'for' statement ------------------------------
940 | bl extern lj_meta_for // (lua_State *L, TValue *base)
941 | ldr INSw, [PC, #-4]
948 | cmp TMP0, #BC_JFORI
953 |//-----------------------------------------------------------------------
954 |//-- Fast functions -----------------------------------------------------
955 |//-----------------------------------------------------------------------
961 |.macro .ffunc_1, name
968 |.macro .ffunc_2, name
970 | ldp CARG1, CARG2, [BASE]
975 |.macro .ffunc_n, name
981 | checknum CARG1, ->fff_fallback
984 |.macro .ffunc_nn, name
986 | ldp CARG1, CARG2, [BASE]
988 | ldp FARG1, FARG2, [BASE]
990 | checknum CARG1, ->fff_fallback
991 | checknum CARG2, ->fff_fallback
994 |// Inlined GC threshold check. Caveat: uses CARG1 and CARG2.
996 | ldp CARG1, CARG2, GL->gc.total // Assumes threshold follows total.
1003 |//-- Base library: checks -----------------------------------------------
1006 | ldr PC, [BASE, FRAME_PC]
1009 | bhs ->fff_fallback
1010 | str CARG1, [BASE, #-16]
1012 | subs RA, NARGS8:RC, #8
1013 | add RC, NARGS8:RC, #8 // Compute (nresults+1)*8.
1014 | cbz RA, ->fff_res // Done if exactly 1 argument.
1016 | ldr CARG1, [RB, #16]
1018 | str CARG1, [RB], #8
1023 | mov TMP0, #~LJ_TISNUM
1024 | asr ITYPE, CARG1, #47
1025 | cmn ITYPE, #~LJ_TISNUM
1026 | csinv TMP1, TMP0, ITYPE, lo
1027 | add TMP1, TMP1, #offsetof(GCfuncC, upvalue)/8
1028 | ldr CARG1, [CFUNC:CARG3, TMP1, lsl #3]
1031 |//-- Base library: getters and setters ---------------------------------
1033 |.ffunc_1 getmetatable
1034 | asr ITYPE, CARG1, #47
1035 | cmn ITYPE, #-LJ_TTAB
1036 | ccmn ITYPE, #-LJ_TUDATA, #4, ne
1037 | and TAB:CARG1, CARG1, #LJ_GCVMASK
1039 |1: // Field metatable must be at same offset for GCtab and GCudata!
1040 | ldr TAB:RB, TAB:CARG1->metatable
1043 | ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable]
1044 | cbz TAB:RB, ->fff_restv
1045 | ldr TMP1w, TAB:RB->hmask
1046 | ldr TMP2w, STR:RC->sid
1047 | ldr NODE:CARG3, TAB:RB->node
1048 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
1049 | add TMP1, TMP1, TMP1, lsl #1
1050 | movn CARG4, #~LJ_TSTR
1051 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
1052 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
1053 |3: // Rearranged logic, because we expect _not_ to find the key.
1054 | ldp CARG1, TMP0, NODE:CARG3->val
1055 | ldr NODE:CARG3, NODE:CARG3->next
1058 | cbnz NODE:CARG3, <3
1060 | mov CARG1, RB // Use metatable as default result.
1061 | movk CARG1, #(LJ_TTAB>>1)&0xffff, lsl #48
1069 | movn TMP0, #~LJ_TISNUM
1071 | csel ITYPE, ITYPE, TMP0, hs
1072 | sub TMP1, GL, ITYPE, lsl #3
1073 | ldr TAB:RB, [TMP1, #offsetof(global_State, gcroot[GCROOT_BASEMT])-8]
1076 |.ffunc_2 setmetatable
1077 | // Fast path: no mt for table yet and not clearing the mt.
1078 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1079 | ldr TAB:TMP0, TAB:TMP1->metatable
1080 | asr ITYPE, CARG2, #47
1081 | ldrb TMP2w, TAB:TMP1->marked
1082 | cmn ITYPE, #-LJ_TTAB
1083 | and TAB:CARG2, CARG2, #LJ_GCVMASK
1084 | ccmp TAB:TMP0, #0, #0, eq
1085 | bne ->fff_fallback
1086 | str TAB:CARG2, TAB:TMP1->metatable
1087 | tbz TMP2w, #2, ->fff_restv // isblack(table)
1088 | barrierback TAB:TMP1, TMP2w, TMP0
1093 | cmp NARGS8:RC, #16
1094 | blo ->fff_fallback
1095 | checktab CARG2, ->fff_fallback
1097 | add CARG3, BASE, #8
1098 | bl extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1099 | // Returns cTValue *.
1100 | ldr CARG1, [CRET1]
1103 |//-- Base library: conversions ------------------------------------------
1106 | // Only handles the number case inline (without a base argument).
1109 | bne ->fff_fallback
1110 | checknumber CARG1, ->fff_fallback
1114 | // Only handles the string or number case inline.
1115 | asr ITYPE, CARG1, #47
1116 | cmn ITYPE, #-LJ_TSTR
1117 | // A __tostring method in the string base metatable is ignored.
1119 | // Handle numbers inline, unless a number base metatable is present.
1120 | ldr TMP1, GL->gcroot[GCROOT_BASEMT_NUM]
1122 | cmn ITYPE, #-LJ_TISNUM
1123 | ccmp TMP1, #0, #0, ls
1124 | str PC, SAVE_PC // Redundant (but a defined value).
1125 | bne ->fff_fallback
1129 | bl extern lj_strfmt_number // (lua_State *L, cTValue *o)
1130 | // Returns GCstr *.
1131 | movn TMP1, #~LJ_TSTR
1133 | add CARG1, CARG1, TMP1, lsl #47
1136 |//-- Base library: iterators -------------------------------------------
1139 | checktp CARG1, LJ_TTAB, ->fff_fallback
1140 | str TISNIL, [BASE, NARGS8:RC] // Set missing 2nd arg to nil.
1141 | ldr PC, [BASE, FRAME_PC]
1142 | add CARG2, BASE, #8
1143 | sub CARG3, BASE, #16
1144 | bl extern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1145 | // Returns 1=found, 0=end, -1=error.
1147 | tbnz CRET1w, #31, ->fff_fallback // Invalid key.
1148 | cbnz CRET1, ->fff_res // Found key/value.
1149 | // End of traversal: return nil.
1150 | str TISNIL, [BASE, #-16]
1154 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1156 | ldr TAB:CARG2, TAB:TMP1->metatable
1158 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1159 | ldr PC, [BASE, FRAME_PC]
1161 | cbnz TAB:CARG2, ->fff_fallback
1164 | stp CFUNC:CARG4, CARG1, [BASE, #-16]
1165 | str TISNIL, [BASE]
1168 |.ffunc_2 ipairs_aux
1169 | checktab CARG1, ->fff_fallback
1170 | checkint CARG2, ->fff_fallback
1171 | ldr TMP1w, TAB:CARG1->asize
1172 | ldr CARG3, TAB:CARG1->array
1173 | ldr TMP0w, TAB:CARG1->hmask
1174 | add CARG2w, CARG2w, #1
1176 | ldr PC, [BASE, FRAME_PC]
1177 | add_TISNUM TMP2, CARG2
1179 | str TMP2, [BASE, #-16]
1180 | bhs >2 // Not in array part?
1181 | ldr TMP0, [CARG3, CARG2, lsl #3]
1183 | mov TMP1, #(2+1)*8
1185 | str TMP0, [BASE, #-8]
1186 | csel RC, RC, TMP1, eq
1188 |2: // Check for empty hash part first. Otherwise call C function.
1189 | cbz TMP0w, ->fff_res
1190 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
1191 | // Returns cTValue * or NULL.
1192 | cbz CRET1, ->fff_res
1197 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1199 | ldr TAB:CARG2, TAB:TMP1->metatable
1201 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1202 | ldr PC, [BASE, FRAME_PC]
1204 | cbnz TAB:CARG2, ->fff_fallback
1207 | stp CFUNC:CARG4, CARG1, [BASE, #-16]
1208 | str TISNUM, [BASE]
1211 |//-- Base library: catch errors ----------------------------------------
1214 | ldr TMP1, L->maxstack
1215 | add TMP2, BASE, NARGS8:RC
1217 | blo ->fff_fallback
1219 | ldrb TMP0w, GL->hookmask
1220 | blo ->fff_fallback
1221 | sub NARGS8:RC, NARGS8:RC, #8
1223 | add BASE, BASE, #16
1224 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1225 | add PC, TMP0, #16+FRAME_PCALL
1226 | beq ->vm_call_dispatch
1228 | add TMP2, BASE, NARGS8:RC
1230 | ldr TMP0, [TMP2, #-16]
1231 | str TMP0, [TMP2, #-8]!
1234 | b ->vm_call_dispatch
1237 | ldr TMP1, L->maxstack
1238 | add TMP2, BASE, NARGS8:RC
1240 | blo ->fff_fallback
1241 | ldp CARG1, CARG2, [BASE]
1242 | ldrb TMP0w, GL->hookmask
1243 | subs NARGS8:TMP1, NARGS8:RC, #16
1244 | blo ->fff_fallback
1246 | asr ITYPE, CARG2, #47
1247 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1248 | cmn ITYPE, #-LJ_TFUNC
1249 | add PC, TMP0, #24+FRAME_PCALL
1250 | bne ->fff_fallback // Traceback must be a function.
1251 | mov NARGS8:RC, NARGS8:TMP1
1252 | add BASE, BASE, #24
1253 | stp CARG2, CARG1, [RB] // Swap function and traceback.
1254 | cbz NARGS8:RC, ->vm_call_dispatch
1257 |//-- Coroutine library --------------------------------------------------
1259 |.macro coroutine_resume_wrap, resume
1261 |.ffunc_1 coroutine_resume
1262 | checktp CARG1, LJ_TTHREAD, ->fff_fallback
1264 |.ffunc coroutine_wrap_aux
1265 | ldr L:CARG1, CFUNC:CARG3->upvalue[0].gcr
1266 | and L:CARG1, CARG1, #LJ_GCVMASK
1268 | ldr PC, [BASE, FRAME_PC]
1270 | ldp RB, CARG2, L:CARG1->base
1271 | ldrb TMP1w, L:CARG1->status
1272 | add TMP0, CARG2, TMP1
1275 | beq ->fff_fallback
1276 | cmp TMP1, #LUA_YIELD
1277 | add TMP0, CARG2, #8
1278 | csel CARG2, CARG2, TMP0, hs
1279 | ldr CARG4, L:CARG1->maxstack
1280 | add CARG3, CARG2, NARGS8:RC
1281 | ldr RB, L:CARG1->cframe
1282 | ccmp CARG3, CARG4, #2, ls
1283 | ccmp RB, #0, #2, ls
1284 | bhi ->fff_fallback
1286 | sub CARG3, CARG3, #8 // Keep resumed thread in stack for GC.
1287 | add BASE, BASE, #8
1288 | sub NARGS8:RC, NARGS8:RC, #8
1290 | str CARG3, L:CARG1->top
1293 |2: // Move args to coroutine.
1294 | ldr TMP0, [BASE, RB]
1296 | str TMP0, [CARG2, RB]
1303 | bl ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1304 | // Returns thread status.
1306 | ldp CARG3, CARG4, L:RA->base
1307 | cmp CRET1, #LUA_YIELD
1310 | st_vmstate ST_INTERP
1312 | sub RC, CARG4, CARG3
1313 | ldr CARG1, L->maxstack
1314 | add CARG2, BASE, RC
1315 | cbz RC, >6 // No results?
1318 | bhi >9 // Need to grow stack?
1321 | str CARG3, L:RA->top // Clear coroutine stack.
1322 |5: // Move results from coroutine.
1323 | ldr TMP0, [CARG3, RB]
1325 | str TMP0, [BASE, RB]
1333 | str TMP1, [BASE, #-8] // Prepend true/false to results.
1339 | ands CARG1, PC, #FRAME_TYPE
1341 | str RCw, SAVE_MULTRES
1345 |8: // Coroutine returned with error (at co->top-1).
1347 | ldr TMP0, [CARG4, #-8]!
1350 | str CARG4, L:RA->top // Remove error from coroutine stack.
1351 | str TMP0, [BASE] // Copy error message.
1356 | bl extern lj_ffh_coroutine_wrap_err // (lua_State *L, lua_State *co)
1360 |9: // Handle stack expansion on return from yield.
1363 | bl extern lj_state_growstack // (lua_State *L, int n)
1368 | coroutine_resume_wrap 1 // coroutine.resume
1369 | coroutine_resume_wrap 0 // coroutine.wrap
1371 |.ffunc coroutine_yield
1372 | ldr TMP0, L->cframe
1373 | add TMP1, BASE, NARGS8:RC
1374 | mov CRET1, #LUA_YIELD
1375 | stp BASE, TMP1, L->base
1376 | tbz TMP0, #0, ->fff_fallback
1377 | str xzr, L->cframe
1378 | strb CRET1w, L->status
1381 |//-- Math library -------------------------------------------------------
1383 |.macro math_round, func, round
1384 | .ffunc math_ .. func
1388 | blo ->fff_fallback
1389 | cmp TISNUMhi, CARG1, lsr #32
1391 | blo ->fff_fallback
1396 | math_round floor, frintm
1397 | math_round ceil, frintp
1400 | checknumber CARG1, ->fff_fallback
1401 | and CARG1, CARG1, #U64x(7fffffff,ffffffff)
1403 | eor CARG2w, CARG1w, CARG1w, asr #31
1404 | movz CARG3, #0x41e0, lsl #48 // 2^31.
1405 | subs CARG1w, CARG2w, CARG1w, asr #31
1406 | add_TISNUM CARG1, CARG1
1407 | csel CARG1, CARG1, CARG3, pl
1411 | // CARG1 = TValue result.
1412 | ldr PC, [BASE, FRAME_PC]
1413 | str CARG1, [BASE, #-16]
1418 | // RC = (nresults+1)*8, PC = return.
1419 | ands CARG1, PC, #FRAME_TYPE
1420 | str RCw, SAVE_MULTRES
1423 | ldr INSw, [PC, #-4]
1426 | cmp RC, RB, lsl #3 // More results expected?
1428 | decode_RA TMP1, INS
1429 | // Adjust BASE. KBASE is assumed to be set for the calling frame.
1430 | sub BASE, RA, TMP1, lsl #3
1433 |6: // Fill up results with nil.
1436 | str TISNIL, [TMP1, #-8]
1439 |.macro math_extern, func
1440 | .ffunc_n math_ .. func
1445 |.macro math_extern2, func
1446 | .ffunc_nn math_ .. func
1454 | ldr PC, [BASE, FRAME_PC]
1455 | str d0, [BASE, #-16]
1462 | bne ->fff_fallback // Need exactly 1 argument.
1463 | checknum CARG1, ->fff_fallback
1479 | math_extern2 atan2
1482 |.ffunc_2 math_ldexp
1484 | checknum CARG1, ->fff_fallback
1485 | checkint CARG2, ->fff_fallback
1486 | sxtw CARG1, CARG2w
1487 | bl extern ldexp // (double x, int exp)
1490 |.ffunc_n math_frexp
1491 | add CARG1, sp, TMPDofs
1494 | ldr PC, [BASE, FRAME_PC]
1495 | str d0, [BASE, #-16]
1497 | add_TISNUM CARG2, CARG2
1498 | str CARG2, [BASE, #-8]
1502 | sub CARG1, BASE, #16
1503 | ldr PC, [BASE, FRAME_PC]
1506 | str d0, [BASE, #-8]
1509 |.macro math_minmax, name, cond, fcond
1513 | checkint CARG1, >4
1514 |1: // Handle integers.
1518 | checkint CARG2, >3
1519 | cmp CARG1w, CARG2w
1521 | csel CARG1, CARG2, CARG1, cond
1523 |3: // Convert intermediate result to number and continue below.
1525 | blo ->fff_fallback
1531 | blo ->fff_fallback
1532 |5: // Handle numbers.
1537 | checknum CARG2, >7
1541 | fcsel d0, d1, d0, fcond
1543 |7: // Convert integer to number and continue above.
1545 | blo ->fff_fallback
1549 | math_minmax math_min, gt, pl
1550 | math_minmax math_max, lt, le
1552 |//-- String library -----------------------------------------------------
1554 |.ffunc string_byte // Only handle the 1-arg case here.
1555 | ldp PC, CARG1, [BASE, FRAME_PC]
1557 | asr ITYPE, CARG1, #47
1558 | ccmn ITYPE, #-LJ_TSTR, #0, eq
1559 | and STR:CARG1, CARG1, #LJ_GCVMASK
1560 | bne ->fff_fallback
1561 | ldrb TMP0w, STR:CARG1[1] // Access is always ok (NUL at end).
1562 | ldr CARG3w, STR:CARG1->len
1563 | add_TISNUM TMP0, TMP0
1564 | str TMP0, [BASE, #-16]
1566 | cbz CARG3, ->fff_res
1569 |.ffunc string_char // Only handle the 1-arg case here.
1571 | ldp PC, CARG1, [BASE, FRAME_PC]
1573 | ccmp NARGS8:RC, #8, #0, ls // Need exactly 1 argument.
1574 | bne ->fff_fallback
1575 | checkint CARG1, ->fff_fallback
1577 | // Point to the char inside the integer in the stack slot.
1581 | add CARG2, BASE, #7
1584 | // CARG2 = str, CARG3 = len.
1588 | bl extern lj_str_new // (lua_State *L, char *str, size_t l)
1590 | // Returns GCstr *.
1592 | movn TMP1, #~LJ_TSTR
1593 | add CARG1, CARG1, TMP1, lsl #47
1599 | ldr CARG3, [BASE, #16]
1600 | cmp NARGS8:RC, #16
1603 | blo ->fff_fallback
1604 | checkint CARG3, ->fff_fallback
1607 | ldr CARG2, [BASE, #8]
1608 | checkstr CARG1, ->fff_fallback
1609 | ldr TMP1w, STR:CARG1->len
1610 | checkint CARG2, ->fff_fallback
1611 | sxtw CARG2, CARG2w
1612 | // CARG1 = str, TMP1 = str->len, CARG2 = start, RB = end
1613 | add TMP2, RB, TMP1
1615 | add TMP0, CARG2, TMP1
1616 | csinc RB, RB, TMP2, ge // if (end < 0) end += len+1
1618 | csinc CARG2, CARG2, TMP0, ge // if (start < 0) start += len+1
1620 | csel RB, RB, xzr, ge // if (end < 0) end = 0
1622 | csinc CARG2, CARG2, xzr, ge // if (start < 1) start = 1
1624 | csel RB, RB, TMP1, le // if (end > len) end = len
1625 | add CARG1, STR:CARG1, #sizeof(GCstr)-1
1626 | subs CARG3, RB, CARG2 // len = end - start
1627 | add CARG2, CARG1, CARG2
1628 | add CARG3, CARG3, #1 // len += 1
1630 | add STR:CARG1, GL, #offsetof(global_State, strempty)
1631 | movn TMP1, #~LJ_TSTR
1632 | add CARG1, CARG1, TMP1, lsl #47
1635 |.macro ffstring_op, name
1636 | .ffunc string_ .. name
1640 | asr ITYPE, CARG2, #47
1641 | ccmn ITYPE, #-LJ_TSTR, #0, hs
1642 | and STR:CARG2, CARG2, #LJ_GCVMASK
1643 | bne ->fff_fallback
1644 | ldr TMP0, GL->tmpbuf.b
1645 | add SBUF:CARG1, GL, #offsetof(global_State, tmpbuf)
1648 | str L, GL->tmpbuf.L
1649 | str TMP0, GL->tmpbuf.w
1650 | bl extern lj_buf_putstr_ .. name
1651 | bl extern lj_buf_tostr
1655 |ffstring_op reverse
1659 |//-- Bit library --------------------------------------------------------
1661 |// FP number to bit conversion for soft-float. Clobbers CARG1-CARG3
1663 | bls ->fff_fallback
1664 | add CARG2, CARG1, CARG1
1666 | sub CARG3, CARG3, CARG2, lsr #53
1669 | and CARG2, CARG2, #U64x(001fffff,ffffffff)
1670 | orr CARG2, CARG2, #U64x(00200000,00000000)
1672 | lsr CARG2, CARG2, CARG3
1673 | cneg CARG1w, CARG2w, mi
1679 |.macro .ffunc_bit, name
1680 | .ffunc_1 bit_..name
1682 | checkint CARG1, ->vm_tobit_fb
1686 |.macro .ffunc_bit_op, name, ins
1692 | ldr CARG1, [BASE, RA]
1696 | checkint CARG1, ->vm_tobit_fb
1698 | ins TMP0w, TMP0w, CARG1w
1702 |.ffunc_bit_op band, and
1703 |.ffunc_bit_op bor, orr
1704 |.ffunc_bit_op bxor, eor
1708 |9: // Label reused by .ffunc_bit_op users.
1709 | add_TISNUM CARG1, TMP0
1714 | add_TISNUM CARG1, TMP0
1719 | add_TISNUM CARG1, TMP0
1722 |.macro .ffunc_bit_sh, name, ins, shmod
1724 | ldp TMP0, CARG1, [BASE]
1725 | cmp NARGS8:RC, #16
1726 | blo ->fff_fallback
1728 | checkint CARG1, ->vm_tobit_fb
1737 | checkint CARG1, ->vm_tobit_fb
1739 | ins TMP0w, CARG1w, TMP1w
1740 | add_TISNUM CARG1, TMP0
1744 |.ffunc_bit_sh lshift, lsl, 0
1745 |.ffunc_bit_sh rshift, lsr, 0
1746 |.ffunc_bit_sh arshift, asr, 0
1747 |.ffunc_bit_sh rol, ror, 1
1748 |.ffunc_bit_sh ror, ror, 0
1750 |//-----------------------------------------------------------------------
1752 |->fff_fallback: // Call fast function fallback handler.
1753 | // BASE = new base, RC = nargs*8
1754 | ldp CFUNC:CARG3, PC, [BASE, FRAME_FUNC] // Fallback may overwrite PC.
1755 | ldr TMP2, L->maxstack
1756 | add TMP1, BASE, NARGS8:RC
1757 | stp BASE, TMP1, L->base
1758 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1759 | add TMP1, TMP1, #8*LUA_MINSTACK
1760 | ldr CARG3, CFUNC:CARG3->f
1761 | str PC, SAVE_PC // Redundant (but a defined value).
1764 | bhi >5 // Need to grow stack.
1765 | blr_auth CARG3 // (lua_State *L)
1766 | // Either throws an error, or recovers and returns -1, 0 or nresults+1.
1771 | bgt ->fff_res // Returned nresults+1?
1772 |1: // Returned 0 or -1: retry fast path.
1774 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1775 | sub NARGS8:RC, CARG1, BASE
1776 | bne ->vm_call_tail // Returned -1?
1777 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1778 | ins_callt // Returned 0: retry fast path.
1780 |// Reconstruct previous base for vmeta_call during tailcall.
1782 | ands TMP0, PC, #FRAME_TYPE
1783 | and TMP1, PC, #~FRAME_TYPEP
1785 | ldrb RAw, [PC, #-4+OFS_RA]
1789 | sub RB, BASE, TMP1
1790 | b ->vm_call_dispatch // Resolve again for tailcall.
1792 |5: // Grow stack for fallback handler.
1793 | mov CARG2, #LUA_MINSTACK
1794 | bl extern lj_state_growstack // (lua_State *L, int n)
1796 | cmp CARG1, CARG1 // Set zero-flag to force retry.
1799 |->fff_gcstep: // Call GC step function.
1800 | // BASE = new base, RC = nargs*8
1802 | add CARG2, BASE, NARGS8:RC // Calculate L->top.
1804 | stp BASE, CARG2, L->base
1805 | str PC, SAVE_PC // Redundant (but a defined value).
1807 | bl extern lj_gc_step // (lua_State *L)
1808 | ldp BASE, CARG2, L->base
1809 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1810 | mov lr, RA // Help return address predictor.
1811 | sub NARGS8:RC, CARG2, BASE // Calculate nargs*8.
1812 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1815 |//-----------------------------------------------------------------------
1816 |//-- Special dispatch targets -------------------------------------------
1817 |//-----------------------------------------------------------------------
1819 |->vm_record: // Dispatch target for recording phase.
1821 | ldrb CARG1w, GL->hookmask
1822 | tst CARG1, #HOOK_VMEVENT // No recording while in vmevent.
1824 | // Decrement the hookcount for consistency, but always do the call.
1825 | ldr CARG2w, GL->hookcount
1826 | tst CARG1, #HOOK_ACTIVE
1828 | sub CARG2w, CARG2w, #1
1829 | tst CARG1, #LUA_MASKLINE|LUA_MASKCOUNT
1831 | str CARG2w, GL->hookcount
1835 |->vm_rethook: // Dispatch target for return hooks.
1836 | ldrb TMP2w, GL->hookmask
1837 | tbz TMP2w, #HOOK_ACTIVE_SHIFT, >1 // Hook already active?
1838 |5: // Re-dispatch to static ins.
1839 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1842 |->vm_inshook: // Dispatch target for instr/line hooks.
1843 | ldrb TMP2w, GL->hookmask
1844 | ldr TMP3w, GL->hookcount
1845 | tbnz TMP2w, #HOOK_ACTIVE_SHIFT, <5 // Hook already active?
1846 | tst TMP2w, #LUA_MASKLINE|LUA_MASKCOUNT
1848 | sub TMP3w, TMP3w, #1
1849 | str TMP3w, GL->hookcount
1851 | tbz TMP2w, #LUA_HOOKLINE, <5
1856 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
1857 | bl extern lj_dispatch_ins // (lua_State *L, const BCIns *pc)
1860 |4: // Re-dispatch to static ins.
1861 | ldr INSw, [PC, #-4]
1862 | add TMP1, GL, INS, uxtb #3
1864 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1868 |->cont_hook: // Continue from hook yield.
1869 | ldr CARG1, [CARG4, #-40]
1871 | str CARG1w, SAVE_MULTRES // Restore MULTRES for *M ins.
1874 |->vm_hotloop: // Hot loop counter underflow.
1876 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Same as curr_topL(L).
1877 | add CARG1, GL, #GG_G2DISP+GG_DISP2J
1878 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1880 | ldr CARG3, LFUNC:CARG3->pc
1882 | str L, [GL, #GL_J(L)]
1883 | ldrb CARG3w, [CARG3, #PC2PROTO(framesize)]
1885 | add CARG3, BASE, CARG3, lsl #3
1887 | bl extern lj_trace_hot // (jit_State *J, const BCIns *pc)
1891 |->vm_callhook: // Dispatch target for call hooks.
1897 |->vm_hotcall: // Hot call counter underflow.
1902 | add TMP1, BASE, NARGS8:RC
1906 | stp BASE, TMP1, L->base
1907 | bl extern lj_dispatch_call // (lua_State *L, const BCIns *pc)
1908 | // Returns ASMFunction.
1909 | ldp BASE, TMP1, L->base
1910 | str xzr, SAVE_PC // Invalidate for subsequent line hook.
1911 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
1913 | sub NARGS8:RC, TMP1, BASE
1914 | ldr INSw, [PC, #-4]
1915 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1918 |->cont_stitch: // Trace stitching.
1920 | // RA = resultptr, CARG4 = meta base
1921 | ldr RBw, SAVE_MULTRES
1922 | ldr INSw, [PC, #-4]
1923 | ldr TRACE:CARG3, [CARG4, #-40] // Save previous trace.
1925 | decode_RA RC, INS // Call base.
1926 | and CARG3, CARG3, #LJ_GCVMASK
1928 |1: // Move results down.
1929 | ldr CARG1, [RA], #8
1931 | str CARG1, [BASE, RC, lsl #3]
1940 | bhi >9 // More results wanted?
1942 | ldrh RAw, TRACE:CARG3->traceno
1943 | ldrh RCw, TRACE:CARG3->link
1945 | beq ->cont_nop // Blacklisted.
1947 | bne =>BC_JLOOP // Jump to stitched trace.
1949 | // Stitch a new trace to the previous trace.
1950 | mov CARG1, #GL_J(exitno)
1951 | str RAw, [GL, CARG1]
1952 | mov CARG1, #GL_J(L)
1953 | str L, [GL, CARG1]
1955 | add CARG1, GL, #GG_G2J
1957 | bl extern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
1961 |9: // Fill up results with nil.
1962 | str TISNIL, [BASE, RC, lsl #3]
1967 |->vm_profhook: // Dispatch target for profiler hook.
1972 | bl extern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
1973 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
1979 |//-----------------------------------------------------------------------
1980 |//-- Trace exit handler -------------------------------------------------
1981 |//-----------------------------------------------------------------------
1983 |.macro savex_, a, b
1984 | stp d..a, d..b, [sp, #a*8]
1985 | stp x..a, x..b, [sp, #32*8+a*8]
1990 | sub sp, sp, #(64*8)
2006 | stp d30, d31, [sp, #30*8]
2007 | ldr CARG1, [sp, #64*8] // Load original value of lr.
2008 | add CARG3, sp, #64*8 // Recompute original value of sp.
2009 | mv_vmstate CARG4w, EXIT
2010 | stp xzr, CARG3, [sp, #62*8] // Store 0/sp in RID_LR/RID_SP.
2011 | sub CARG1, CARG1, lr
2013 | lsr CARG1, CARG1, #2
2014 | ldr BASE, GL->jit_base
2015 | sub CARG1, CARG1, #2
2016 | ldr CARG2w, [lr] // Load trace number.
2019 | rev32 CARG2, CARG2
2022 | ubfx CARG2w, CARG2w, #5, #16
2023 | str CARG1w, [GL, #GL_J(exitno)]
2024 | str CARG2w, [GL, #GL_J(parent)]
2025 | str L, [GL, #GL_J(L)]
2026 | str xzr, GL->jit_base
2027 | add CARG1, GL, #GG_G2J
2029 | bl extern lj_trace_exit // (jit_State *J, ExitState *ex)
2030 | // Returns MULTRES (unscaled) or negated error code.
2031 | ldr CARG2, L->cframe
2033 | and sp, CARG2, #CFRAME_RAWMASK
2034 | ldr PC, SAVE_PC // Get SAVE_PC.
2035 | str L, SAVE_L // Set SAVE_L (on-trace resume/yield).
2040 | // CARG1 = MULTRES or negated error code, BASE, PC and GL set.
2045 | cmn CARG1w, #LUA_ERRERR
2046 | bhs >9 // Check for error from exit.
2047 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2049 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2050 | str RCw, SAVE_MULTRES
2052 | ldr CARG2, LFUNC:CARG2->pc
2053 | str xzr, GL->jit_base
2054 | mv_vmstate CARG4w, INTERP
2055 | ldr KBASE, [CARG2, #PC2PROTO(k)]
2056 | // Modified copy of ins_next which handles function header dispatch, too.
2057 | ldrb RBw, [PC, # OFS_OP]
2058 | ldr INSw, [PC], #4
2060 | cmn CARG1w, #17 // Static dispatch?
2062 | cmp RBw, #BC_FUNCC+2 // Fast function?
2063 | add TMP1, GL, INS, uxtb #3
2066 | cmp RBw, #BC_FUNCF // Function header?
2067 | add TMP0, GL, RB, uxtb #3
2068 | ldr RB, [TMP0, #GG_G2DISP]
2070 | lsr TMP0, INS, #16
2071 | csel RC, TMP0, RC, lo
2073 | ldr CARG3, [BASE, FRAME_FUNC]
2075 | add RA, BASE, RA, lsl #3 // Yes: RA = BASE+framesize*8, RC = nargs*8
2076 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2080 |4: // Check frame below fast function.
2081 | ldr CARG1, [BASE, FRAME_PC]
2082 | ands CARG2, CARG1, #FRAME_TYPE
2083 | bne <2 // Trace stitching continuation?
2084 | // Otherwise set KBASE for Lua function below fast function.
2085 | ldr CARG3w, [CARG1, #-4]
2086 | decode_RA CARG1, CARG3
2087 | sub CARG2, BASE, CARG1, lsl #3
2088 | ldr LFUNC:CARG3, [CARG2, #-32]
2089 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2090 | ldr CARG3, LFUNC:CARG3->pc
2091 | ldr KBASE, [CARG3, #PC2PROTO(k)]
2094 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2095 | ldr RA, [GL, #GL_J(trace)]
2097 | ldr TRACE:RA, [RA, RC, lsl #3]
2098 | ldr INSw, TRACE:RA->startins
2099 | add TMP0, GL, INS, uxtb #3
2101 | ldr RB, [TMP0, #GG_G2DISP+GG_DISP2STATIC]
2105 |9: // Rethrow error from the right C frame.
2106 | neg CARG2w, CARG1w
2108 | bl extern lj_err_trace // (lua_State *L, int errcode)
2111 |//-----------------------------------------------------------------------
2112 |//-- Math helper functions ----------------------------------------------
2113 |//-----------------------------------------------------------------------
2115 | // int lj_vm_modi(int dividend, int divisor);
2117 | eor CARG4w, CARG1w, CARG2w
2119 | eor CARG3w, CARG1w, CARG1w, asr #31
2120 | eor CARG4w, CARG2w, CARG2w, asr #31
2121 | sub CARG3w, CARG3w, CARG1w, asr #31
2122 | sub CARG4w, CARG4w, CARG2w, asr #31
2123 | udiv CARG1w, CARG3w, CARG4w
2124 | msub CARG1w, CARG1w, CARG4w, CARG3w
2125 | ccmp CARG1w, #0, #4, mi
2126 | sub CARG3w, CARG1w, CARG4w
2127 | csel CARG1w, CARG1w, CARG3w, eq
2128 | eor CARG3w, CARG1w, CARG2w
2130 | cneg CARG1w, CARG1w, mi
2133 |//-----------------------------------------------------------------------
2134 |//-- Miscellaneous functions --------------------------------------------
2135 |//-----------------------------------------------------------------------
2137 |.define NEXT_TAB, TAB:CARG1
2138 |.define NEXT_RES, CARG1
2139 |.define NEXT_IDX, CARG2w
2140 |.define NEXT_LIM, CARG3w
2141 |.define NEXT_TMP0, TMP0
2142 |.define NEXT_TMP0w, TMP0w
2143 |.define NEXT_TMP1, TMP1
2144 |.define NEXT_TMP1w, TMP1w
2145 |.define NEXT_RES_PTR, sp
2146 |.define NEXT_RES_VAL, [sp]
2147 |.define NEXT_RES_KEY, [sp, #8]
2149 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2150 |// Next idx returned in CRET2w.
2153 | ldr NEXT_LIM, NEXT_TAB->asize
2154 | ldr NEXT_TMP1, NEXT_TAB->array
2155 |1: // Traverse array part.
2156 | subs NEXT_TMP0w, NEXT_IDX, NEXT_LIM
2157 | bhs >5 // Index points after array part?
2158 | ldr NEXT_TMP0, [NEXT_TMP1, NEXT_IDX, uxtw #3]
2159 | cmn NEXT_TMP0, #-LJ_TNIL
2160 | cinc NEXT_IDX, NEXT_IDX, eq
2161 | beq <1 // Skip holes in array part.
2162 | str NEXT_TMP0, NEXT_RES_VAL
2163 | movz NEXT_TMP0w, #(LJ_TISNUM>>1)&0xffff, lsl #16
2164 | stp NEXT_IDX, NEXT_TMP0w, NEXT_RES_KEY
2165 | add NEXT_IDX, NEXT_IDX, #1
2166 | mov NEXT_RES, NEXT_RES_PTR
2170 |5: // Traverse hash part.
2171 | ldr NEXT_TMP1w, NEXT_TAB->hmask
2172 | ldr NODE:NEXT_RES, NEXT_TAB->node
2173 | add NEXT_TMP0w, NEXT_TMP0w, NEXT_TMP0w, lsl #1
2174 | add NEXT_LIM, NEXT_LIM, NEXT_TMP1w
2175 | add NODE:NEXT_RES, NODE:NEXT_RES, NEXT_TMP0w, uxtw #3
2177 | cmp NEXT_IDX, NEXT_LIM
2179 | ldr NEXT_TMP0, NODE:NEXT_RES->val
2180 | cmn NEXT_TMP0, #-LJ_TNIL
2181 | add NEXT_IDX, NEXT_IDX, #1
2183 | // Skip holes in hash part.
2184 | add NODE:NEXT_RES, NODE:NEXT_RES, #sizeof(Node)
2187 |9: // End of iteration. Set the key to nil (not the value).
2188 | movn NEXT_TMP0, #0
2189 | str NEXT_TMP0, NEXT_RES_KEY
2190 | mov NEXT_RES, NEXT_RES_PTR
2194 |//-----------------------------------------------------------------------
2195 |//-- FFI helper functions -----------------------------------------------
2196 |//-----------------------------------------------------------------------
2198 |// Handler for callback functions.
2199 |// Saveregs already performed. Callback slot number in w9, g in x10.
2202 |.type CTSTATE, CTState, PC
2204 | ldr CTSTATE, GL:x10->ctype_state
2206 | add x10, sp, # CFRAME_SPACE
2207 | str w9, CTSTATE->cb.slot
2208 | stp x0, x1, CTSTATE->cb.gpr[0]
2209 | stp d0, d1, CTSTATE->cb.fpr[0]
2210 | stp x2, x3, CTSTATE->cb.gpr[2]
2211 | stp d2, d3, CTSTATE->cb.fpr[2]
2212 | stp x4, x5, CTSTATE->cb.gpr[4]
2213 | stp d4, d5, CTSTATE->cb.fpr[4]
2214 | stp x6, x7, CTSTATE->cb.gpr[6]
2215 | stp d6, d7, CTSTATE->cb.fpr[6]
2216 | str x10, CTSTATE->cb.stack
2217 | mov CARG1, CTSTATE
2218 | str CTSTATE, SAVE_PC // Any value outside of bytecode is ok.
2220 | bl extern lj_ccallback_enter // (CTState *cts, void *cf)
2221 | // Returns lua_State *.
2222 | ldp BASE, RC, L:CRET1->base
2225 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2227 | st_vmstate ST_INTERP
2228 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2232 |->cont_ffi_callback: // Return from FFI callback.
2234 | ldr CTSTATE, GL->ctype_state
2235 | stp BASE, CARG4, L->base
2237 | mov CARG1, CTSTATE
2239 | bl extern lj_ccallback_leave // (CTState *cts, TValue *o)
2240 | ldp x0, x1, CTSTATE->cb.gpr[0]
2241 | ldp d0, d1, CTSTATE->cb.fpr[0]
2245 |->vm_ffi_call: // Call C function via FFI.
2246 | // Caveat: needs special frame unwinding, see below.
2248 | .type CCSTATE, CCallState, x19
2250 | stp_unwind CCSTATE, x20, [sp, #-32]!
2251 | stp fp, lr, [sp, #16]
2254 | ldr TMP0w, CCSTATE:x0->spadj
2255 | ldrb TMP1w, CCSTATE->nsp
2256 | add TMP2, CCSTATE, #offsetof(CCallState, stack)
2257 | subs TMP1, TMP1, #8
2258 | ldr TMP3, CCSTATE->func
2261 |1: // Copy stack slots
2262 | ldr TMP0, [TMP2, TMP1]
2263 | str TMP0, [sp, TMP1]
2264 | subs TMP1, TMP1, #8
2267 | ldp x0, x1, CCSTATE->gpr[0]
2268 | ldp d0, d1, CCSTATE->fpr[0]
2269 | ldp x2, x3, CCSTATE->gpr[2]
2270 | ldp d2, d3, CCSTATE->fpr[2]
2271 | ldp x4, x5, CCSTATE->gpr[4]
2272 | ldp d4, d5, CCSTATE->fpr[4]
2273 | ldp x6, x7, CCSTATE->gpr[6]
2274 | ldp d6, d7, CCSTATE->fpr[6]
2275 | ldr x8, CCSTATE->retp
2278 | stp x0, x1, CCSTATE->gpr[0]
2279 | stp d0, d1, CCSTATE->fpr[0]
2280 | stp d2, d3, CCSTATE->fpr[2]
2281 | ldp fp, lr, [sp, #16]
2282 | ldp_unwind CCSTATE, x20, [sp], #32
2285 |// Note: vm_ffi_call must be the last function in this object file!
2287 |//-----------------------------------------------------------------------
2290 /* Generate the code for a single instruction. */
2291 static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2298 /* -- Comparison ops ---------------------------------------------------- */
2300 /* Remember: all ops branch for a true comparison, fall through otherwise. */
2302 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2303 | // RA = src1, RC = src2, JMP with RC = target
2304 | ldr CARG1, [BASE, RA, lsl #3]
2305 | ldrh RBw, [PC, # OFS_RD]
2306 | ldr CARG2, [BASE, RC, lsl #3]
2308 | add RB, PC, RB, lsl #2
2309 | sub RB, RB, #0x20000
2310 | checkint CARG1, >3
2311 | checkint CARG2, >4
2312 | cmp CARG1w, CARG2w
2313 if (op == BC_ISLT) {
2314 | csel PC, RB, PC, lt
2315 } else if (op == BC_ISGE) {
2316 | csel PC, RB, PC, ge
2317 } else if (op == BC_ISLE) {
2318 | csel PC, RB, PC, le
2320 | csel PC, RB, PC, gt
2326 | ldr FARG1, [BASE, RA, lsl #3]
2328 | ldr FARG2, [BASE, RC, lsl #3]
2329 | cmp TISNUMhi, CARG2, lsr #32
2332 | // RA number, RC int.
2333 | scvtf FARG2, CARG2w
2336 |4: // RA int, RC not int
2337 | ldr FARG2, [BASE, RC, lsl #3]
2339 | // RA int, RC number.
2340 | scvtf FARG1, CARG1w
2342 |5: // RA number, RC number
2344 | // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2345 if (op == BC_ISLT) {
2346 | csel PC, RB, PC, lo
2347 } else if (op == BC_ISGE) {
2348 | csel PC, RB, PC, hs
2349 } else if (op == BC_ISLE) {
2350 | csel PC, RB, PC, ls
2352 | csel PC, RB, PC, hi
2357 case BC_ISEQV: case BC_ISNEV:
2358 vk = op == BC_ISEQV;
2359 | // RA = src1, RC = src2, JMP with RC = target
2360 | ldr CARG1, [BASE, RA, lsl #3]
2361 | add RC, BASE, RC, lsl #3
2362 | ldrh RBw, [PC, # OFS_RD]
2365 | add RB, PC, RB, lsl #2
2366 | sub RB, RB, #0x20000
2367 | asr ITYPE, CARG3, #47
2368 | cmn ITYPE, #-LJ_TISNUM
2374 | // RC is not a number.
2375 | asr TMP0, CARG1, #47
2377 | // Check if RC or RA is a cdata.
2378 | cmn ITYPE, #-LJ_TCDATA
2379 | ccmn TMP0, #-LJ_TCDATA, #4, ne
2380 | beq ->vmeta_equal_cd
2384 | // Tag and value are equal.
2387 | mov PC, RB // Perform branch.
2392 |2: // Check if the tags are the same and it's a table or userdata.
2394 | ccmn ITYPE, #-LJ_TISTABUD, #2, eq
2398 | bhi ->BC_ISEQV_Z // Reuse code from opposite instruction.
2400 | // Different tables or userdatas. Need to check __eq metamethod.
2401 | // Field metatable must be at same offset for GCtab and GCudata!
2402 | and TAB:CARG2, CARG1, #LJ_GCVMASK
2403 | ldr TAB:TMP2, TAB:CARG2->metatable
2405 | cbz TAB:TMP2, <1 // No metatable?
2406 | ldrb TMP1w, TAB:TMP2->nomm
2407 | mov CARG4, #0 // ne = 0
2408 | tbnz TMP1w, #MM_eq, <1 // 'no __eq' flag set: done.
2410 | cbz TAB:TMP2, ->BC_ISEQV_Z // No metatable?
2411 | ldrb TMP1w, TAB:TMP2->nomm
2412 | mov CARG4, #1 // ne = 1.
2413 | tbnz TMP1w, #MM_eq, ->BC_ISEQV_Z // 'no __eq' flag set: done.
2418 case BC_ISEQS: case BC_ISNES:
2419 vk = op == BC_ISEQS;
2420 | // RA = src, RC = str_const (~), JMP with RC = target
2421 | ldr CARG1, [BASE, RA, lsl #3]
2423 | ldrh RBw, [PC, # OFS_RD]
2424 | ldr CARG2, [KBASE, RC, lsl #3]
2426 | movn TMP0, #~LJ_TSTR
2428 | asr ITYPE, CARG1, #47
2430 | add RB, PC, RB, lsl #2
2431 | add CARG2, CARG2, TMP0, lsl #47
2432 | sub RB, RB, #0x20000
2434 | cmn ITYPE, #-LJ_TCDATA
2435 | beq ->vmeta_equal_cd
2439 | csel PC, RB, PC, eq
2441 | csel PC, RB, PC, ne
2446 case BC_ISEQN: case BC_ISNEN:
2447 vk = op == BC_ISEQN;
2448 | // RA = src, RC = num_const (~), JMP with RC = target
2449 | ldr CARG1, [BASE, RA, lsl #3]
2450 | add RC, KBASE, RC, lsl #3
2451 | ldrh RBw, [PC, # OFS_RD]
2454 | add RB, PC, RB, lsl #2
2455 | sub RB, RB, #0x20000
2461 | checkint CARG1, >4
2462 | checkint CARG3, >6
2463 | cmp CARG1w, CARG3w
2466 | csel PC, RB, PC, eq
2470 | csel PC, RB, PC, ne
2481 | ldr FARG1, [BASE, RA, lsl #3]
2483 | cmp TISNUMhi, CARG3, lsr #32
2485 | // RA number, RC int.
2486 | scvtf FARG2, CARG3w
2488 | // RA number, RC number.
2492 |6: // RA int, RC number
2494 | scvtf FARG1, CARG1w
2500 | asr ITYPE, CARG1, #47
2501 | cmn ITYPE, #-LJ_TCDATA
2503 | b ->vmeta_equal_cd
2507 case BC_ISEQP: case BC_ISNEP:
2508 vk = op == BC_ISEQP;
2509 | // RA = src, RC = primitive_type (~), JMP with RC = target
2510 | ldr TMP0, [BASE, RA, lsl #3]
2511 | ldrh RBw, [PC, # OFS_RD]
2514 | add RB, PC, RB, lsl #2
2516 | asr ITYPE, TMP0, #47
2517 | cmn ITYPE, #-LJ_TCDATA
2518 | beq ->vmeta_equal_cd
2521 | cmn RC, TMP0, asr #47
2523 | sub RB, RB, #0x20000
2525 | csel PC, RB, PC, eq
2527 | csel PC, RB, PC, ne
2532 /* -- Unary test and copy ops ------------------------------------------- */
2534 case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
2535 | // RA = dst or unused, RC = src, JMP with RC = target
2536 | ldrh RBw, [PC, # OFS_RD]
2537 | ldr TMP0, [BASE, RC, lsl #3]
2540 | add RB, PC, RB, lsl #2
2542 | sub RB, RB, #0x20000
2543 if (op == BC_ISTC || op == BC_IST) {
2544 if (op == BC_ISTC) {
2545 | csel RA, RA, RC, lo
2547 | csel PC, RB, PC, lo
2549 if (op == BC_ISFC) {
2550 | csel RA, RA, RC, hs
2552 | csel PC, RB, PC, hs
2554 if (op == BC_ISTC || op == BC_ISFC) {
2555 | str TMP0, [BASE, RA, lsl #3]
2561 | // RA = src, RC = -type
2562 | ldr TMP0, [BASE, RA, lsl #3]
2563 | cmn RC, TMP0, asr #47
2564 | bne ->vmeta_istype
2568 | // RA = src, RC = -(TISNUM-1)
2569 | ldr TMP0, [BASE, RA]
2570 | checknum TMP0, ->vmeta_istype
2574 /* -- Unary ops --------------------------------------------------------- */
2577 | // RA = dst, RC = src
2578 | ldr TMP0, [BASE, RC, lsl #3]
2579 | str TMP0, [BASE, RA, lsl #3]
2583 | // RA = dst, RC = src
2584 | ldr TMP0, [BASE, RC, lsl #3]
2588 | csel TMP0, TMP1, TMP2, lo
2589 | str TMP0, [BASE, RA, lsl #3]
2593 | // RA = dst, RC = src
2594 | ldr TMP0, [BASE, RC, lsl #3]
2595 | asr ITYPE, TMP0, #47
2596 | cmn ITYPE, #-LJ_TISNUM
2598 | eor TMP0, TMP0, #U64x(80000000,00000000)
2601 | movz CARG3, #0x41e0, lsl #48 // 2^31.
2602 | add_TISNUM TMP0, TMP0
2603 | csel TMP0, TMP0, CARG3, vc
2605 | str TMP0, [BASE, RA, lsl #3]
2609 | // RA = dst, RC = src
2610 | ldr CARG1, [BASE, RC, lsl #3]
2611 | asr ITYPE, CARG1, #47
2612 | cmn ITYPE, #-LJ_TSTR
2613 | and CARG1, CARG1, #LJ_GCVMASK
2615 | ldr CARG1w, STR:CARG1->len
2617 | add_TISNUM CARG1, CARG1
2618 | str CARG1, [BASE, RA, lsl #3]
2622 | cmn ITYPE, #-LJ_TTAB
2625 | ldr TAB:CARG2, TAB:CARG1->metatable
2626 | cbnz TAB:CARG2, >9
2630 | bl extern lj_tab_len // (GCtab *t)
2631 | // Returns uint32_t (but less than 2^31).
2636 | ldrb TMP1w, TAB:CARG2->nomm
2637 | tbnz TMP1w, #MM_len, <3 // 'no __len' flag set: done.
2642 /* -- Binary ops -------------------------------------------------------- */
2644 |.macro ins_arithcheck_int, target
2645 | checkint CARG1, target
2646 | checkint CARG2, target
2649 |.macro ins_arithcheck_num, target
2650 | checknum CARG1, target
2651 | checknum CARG2, target
2654 |.macro ins_arithcheck_nzdiv, target
2655 | cbz CARG2w, target
2658 |.macro ins_arithhead
2659 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
2669 |.macro ins_arithload, reg1, reg2
2670 | // RA = dst, RB = src1, RC = src2 | num_const
2673 | ldr reg1, [BASE, RB, lsl #3]
2674 | ldr reg2, [KBASE, RC, lsl #3]
2677 | ldr reg1, [KBASE, RC, lsl #3]
2678 | ldr reg2, [BASE, RB, lsl #3]
2681 | ldr reg1, [BASE, RB, lsl #3]
2682 | ldr reg2, [BASE, RC, lsl #3]
2687 |.macro ins_arithfallback, ins
2690 | ins ->vmeta_arith_vn
2693 | ins ->vmeta_arith_nv
2696 | ins ->vmeta_arith_vv
2701 |.macro ins_arithmod, res, reg1, reg2
2702 | fdiv d2, reg1, reg2
2704 | // Cannot use fmsub, because FMA is not enabled by default.
2706 | fsub res, reg1, d2
2709 |.macro ins_arithdn, intins, fpins
2711 | ins_arithload CARG1, CARG2
2712 | ins_arithcheck_int >5
2713 |.if "intins" == "smull"
2714 | smull CARG1, CARG1w, CARG2w
2715 | cmp CARG1, CARG1, sxtw
2716 | mov CARG1w, CARG1w
2717 | ins_arithfallback bne
2718 |.elif "intins" == "ins_arithmodi"
2719 | ins_arithfallback ins_arithcheck_nzdiv
2722 | intins CARG1w, CARG1w, CARG2w
2723 | ins_arithfallback bvs
2725 | add_TISNUM CARG1, CARG1
2726 | str CARG1, [BASE, RA, lsl #3]
2731 | ins_arithload FARG1, FARG2
2732 | ins_arithfallback ins_arithcheck_num
2733 | fpins FARG1, FARG1, FARG2
2734 | str FARG1, [BASE, RA, lsl #3]
2738 |.macro ins_arithfp, fpins
2740 | ins_arithload CARG1, CARG2
2741 | ins_arithload FARG1, FARG2
2742 | ins_arithfallback ins_arithcheck_num
2743 |.if "fpins" == "fpow"
2746 | fpins FARG1, FARG1, FARG2
2748 | str FARG1, [BASE, RA, lsl #3]
2752 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
2753 | ins_arithdn adds, fadd
2755 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
2756 | ins_arithdn subs, fsub
2758 case BC_MULVN: case BC_MULNV: case BC_MULVV:
2759 | ins_arithdn smull, fmul
2761 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
2764 case BC_MODVN: case BC_MODNV: case BC_MODVV:
2765 | ins_arithdn ins_arithmodi, ins_arithmod
2768 | // NYI: (partial) integer arithmetic.
2775 | // RA = dst, RB = src_start, RC = src_end
2778 | add CARG2, BASE, RC, lsl #3
2780 | // RA = dst, CARG2 = top-1, CARG3 = left
2783 | bl extern lj_meta_cat // (lua_State *L, TValue *top, int left)
2784 | // Returns NULL (finished) or TValue * (metamethod).
2785 | ldrb RBw, [PC, #-4+OFS_RB]
2787 | cbnz CRET1, ->vmeta_binop
2788 | ldr TMP0, [BASE, RB, lsl #3]
2789 | str TMP0, [BASE, RA, lsl #3] // Copy result to RA.
2793 /* -- Constant ops ------------------------------------------------------ */
2796 | // RA = dst, RC = str_const (~)
2798 | ldr TMP0, [KBASE, RC, lsl #3]
2799 | movn TMP1, #~LJ_TSTR
2800 | add TMP0, TMP0, TMP1, lsl #47
2801 | str TMP0, [BASE, RA, lsl #3]
2806 | // RA = dst, RC = cdata_const (~)
2808 | ldr TMP0, [KBASE, RC, lsl #3]
2809 | movn TMP1, #~LJ_TCDATA
2810 | add TMP0, TMP0, TMP1, lsl #47
2811 | str TMP0, [BASE, RA, lsl #3]
2816 | // RA = dst, RC = int16_literal
2818 | add_TISNUM TMP0, RC
2819 | str TMP0, [BASE, RA, lsl #3]
2823 | // RA = dst, RC = num_const
2824 | ldr TMP0, [KBASE, RC, lsl #3]
2825 | str TMP0, [BASE, RA, lsl #3]
2829 | // RA = dst, RC = primitive_type (~)
2830 | mvn TMP0, RC, lsl #47
2831 | str TMP0, [BASE, RA, lsl #3]
2835 | // RA = base, RC = end
2836 | add RA, BASE, RA, lsl #3
2837 | add RC, BASE, RC, lsl #3
2838 | str TISNIL, [RA], #8
2841 | str TISNIL, [RA], #8
2846 /* -- Upvalue and function ops ------------------------------------------ */
2849 | // RA = dst, RC = uvnum
2850 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2851 | add RC, RC, #offsetof(GCfuncL, uvptr)/8
2852 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2853 | ldr UPVAL:CARG2, [LFUNC:CARG2, RC, lsl #3]
2854 | ldr CARG2, UPVAL:CARG2->v
2856 | str TMP0, [BASE, RA, lsl #3]
2860 | // RA = uvnum, RC = src
2861 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2862 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2863 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2864 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2865 | ldr CARG3, [BASE, RC, lsl #3]
2866 | ldr CARG2, UPVAL:CARG1->v
2867 | ldrb TMP2w, UPVAL:CARG1->marked
2868 | ldrb TMP0w, UPVAL:CARG1->closed
2869 | asr ITYPE, CARG3, #47
2870 | str CARG3, [CARG2]
2871 | add ITYPE, ITYPE, #-LJ_TISGCV
2872 | tst TMP2w, #LJ_GC_BLACK // isblack(uv)
2873 | ccmp TMP0w, #0, #4, ne // && uv->closed
2874 | ccmn ITYPE, #-(LJ_TNUMX - LJ_TISGCV), #0, ne // && tvisgcv(v)
2879 |2: // Check if new value is white.
2880 | and GCOBJ:CARG3, CARG3, #LJ_GCVMASK
2881 | ldrb TMP1w, GCOBJ:CARG3->gch.marked
2882 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2884 | // Crossed a write barrier. Move the barrier forward.
2886 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2890 | // RA = uvnum, RC = str_const (~)
2891 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2892 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2894 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2895 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2896 | ldr STR:CARG3, [KBASE, RC, lsl #3]
2897 | movn TMP0, #~LJ_TSTR
2898 | ldr CARG2, UPVAL:CARG1->v
2899 | ldrb TMP2w, UPVAL:CARG1->marked
2900 | add TMP0, STR:CARG3, TMP0, lsl #47
2901 | ldrb TMP1w, STR:CARG3->marked
2903 | tbnz TMP2w, #2, >2 // isblack(uv)
2907 |2: // Check if string is white and ensure upvalue is closed.
2908 | ldrb TMP0w, UPVAL:CARG1->closed
2909 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2910 | ccmp TMP0w, #0, #4, ne
2912 | // Crossed a write barrier. Move the barrier forward.
2914 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2918 | // RA = uvnum, RC = num_const
2919 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2920 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2921 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2922 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2923 | ldr TMP0, [KBASE, RC, lsl #3]
2924 | ldr CARG2, UPVAL:CARG2->v
2929 | // RA = uvnum, RC = primitive_type (~)
2930 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2931 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2932 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2933 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2934 | mvn TMP0, RC, lsl #47
2935 | ldr CARG2, UPVAL:CARG2->v
2941 | // RA = level, RC = target
2942 | ldr CARG3, L->openupval
2943 | add RC, PC, RC, lsl #2
2945 | sub PC, RC, #0x20000
2948 | add CARG2, BASE, RA, lsl #3
2949 | bl extern lj_func_closeuv // (lua_State *L, TValue *level)
2956 | // RA = dst, RC = proto_const (~) (holding function prototype)
2959 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2961 | ldr CARG2, [KBASE, RC, lsl #3]
2963 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2964 | // (lua_State *L, GCproto *pt, GCfuncL *parent)
2965 | bl extern lj_func_newL_gc
2966 | // Returns GCfuncL *.
2968 | movn TMP0, #~LJ_TFUNC
2969 | add CRET1, CRET1, TMP0, lsl #47
2970 | str CRET1, [BASE, RA, lsl #3]
2974 /* -- Table ops --------------------------------------------------------- */
2978 | // RA = dst, RC = (hbits|asize) | tab_const (~)
2979 | ldp CARG3, CARG4, GL->gc.total // Assumes threshold follows total.
2986 if (op == BC_TNEW) {
2987 | and CARG2, RC, #0x7ff
2988 | lsr CARG3, RC, #11
2991 | csel CARG2, CARG2, TMP0, ne
2992 | bl extern lj_tab_new // (lua_State *L, int32_t asize, uint32_t hbits)
2993 | // Returns GCtab *.
2996 | ldr CARG2, [KBASE, RC, lsl #3]
2997 | bl extern lj_tab_dup // (lua_State *L, Table *kt)
2998 | // Returns GCtab *.
3001 | movk CRET1, #(LJ_TTAB>>1)&0xffff, lsl #48
3002 | str CRET1, [BASE, RA, lsl #3]
3006 | bl extern lj_gc_step_fixtop // (lua_State *L)
3012 | // RA = dst, RC = str_const (~)
3014 | // RA = src, RC = str_const (~)
3015 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3017 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3018 | ldr TAB:CARG2, LFUNC:CARG1->env
3019 | ldr STR:RC, [KBASE, RC, lsl #3]
3020 if (op == BC_GGET) {
3030 | // RA = dst, RB = table, RC = key
3031 | ldr CARG2, [BASE, RB, lsl #3]
3032 | ldr TMP1, [BASE, RC, lsl #3]
3033 | checktab CARG2, ->vmeta_tgetv
3034 | checkint TMP1, >9 // Integer key?
3035 | ldr CARG3, TAB:CARG2->array
3036 | ldr CARG1w, TAB:CARG2->asize
3037 | add CARG3, CARG3, TMP1, uxtw #3
3038 | cmp TMP1w, CARG1w // In array part?
3044 | str TMP0, [BASE, RA, lsl #3]
3047 |5: // Check for __index if table value is nil.
3048 | ldr TAB:CARG1, TAB:CARG2->metatable
3049 | cbz TAB:CARG1, <1 // No metatable: done.
3050 | ldrb TMP1w, TAB:CARG1->nomm
3051 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
3055 | asr ITYPE, TMP1, #47
3056 | cmn ITYPE, #-LJ_TSTR // String key?
3058 | and STR:RC, TMP1, #LJ_GCVMASK
3064 | // RA = dst, RB = table, RC = str_const (~)
3065 | ldr CARG2, [BASE, RB, lsl #3]
3067 | ldr STR:RC, [KBASE, RC, lsl #3]
3068 | checktab CARG2, ->vmeta_tgets1
3070 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst
3071 | ldr TMP1w, TAB:CARG2->hmask
3072 | ldr TMP2w, STR:RC->sid
3073 | ldr NODE:CARG3, TAB:CARG2->node
3074 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
3075 | add TMP1, TMP1, TMP1, lsl #1
3076 | movn CARG4, #~LJ_TSTR
3077 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
3078 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
3080 | ldp TMP0, CARG1, NODE:CARG3->val
3081 | ldr NODE:CARG3, NODE:CARG3->next
3087 | str TMP0, [BASE, RA, lsl #3]
3090 |4: // Follow hash chain.
3091 | cbnz NODE:CARG3, <1
3092 | // End of hash chain: key not found, nil result.
3095 |5: // Check for __index if table value is nil.
3096 | ldr TAB:CARG1, TAB:CARG2->metatable
3097 | cbz TAB:CARG1, <3 // No metatable: done.
3098 | ldrb TMP1w, TAB:CARG1->nomm
3099 | tbnz TMP1w, #MM_index, <3 // 'no __index' flag set: done.
3105 | // RA = dst, RB = table, RC = index
3106 | ldr CARG2, [BASE, RB, lsl #3]
3107 | checktab CARG2, ->vmeta_tgetb
3108 | ldr CARG3, TAB:CARG2->array
3109 | ldr CARG1w, TAB:CARG2->asize
3110 | add CARG3, CARG3, RC, lsl #3
3111 | cmp RCw, CARG1w // In array part?
3117 | str TMP0, [BASE, RA, lsl #3]
3120 |5: // Check for __index if table value is nil.
3121 | ldr TAB:CARG1, TAB:CARG2->metatable
3122 | cbz TAB:CARG1, <1 // No metatable: done.
3123 | ldrb TMP1w, TAB:CARG1->nomm
3124 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
3130 | // RA = dst, RB = table, RC = key
3131 | ldr CARG1, [BASE, RB, lsl #3]
3132 | ldr TMP1, [BASE, RC, lsl #3]
3133 | and TAB:CARG1, CARG1, #LJ_GCVMASK
3134 | ldr CARG3, TAB:CARG1->array
3135 | ldr TMP2w, TAB:CARG1->asize
3136 | add CARG3, CARG3, TMP1w, uxtw #3
3137 | cmp TMP1w, TMP2w // In array part?
3141 | str TMP0, [BASE, RA, lsl #3]
3148 | // RA = src, RB = table, RC = key
3149 | ldr CARG2, [BASE, RB, lsl #3]
3150 | ldr TMP1, [BASE, RC, lsl #3]
3151 | checktab CARG2, ->vmeta_tsetv
3152 | checkint TMP1, >9 // Integer key?
3153 | ldr CARG3, TAB:CARG2->array
3154 | ldr CARG1w, TAB:CARG2->asize
3155 | add CARG3, CARG3, TMP1, uxtw #3
3156 | cmp TMP1w, CARG1w // In array part?
3159 | ldr TMP0, [BASE, RA, lsl #3]
3160 | ldrb TMP2w, TAB:CARG2->marked
3161 | cmp_nil TMP1 // Previous value is nil?
3165 | tbnz TMP2w, #2, >7 // isblack(table)
3169 |5: // Check for __newindex if previous value is nil.
3170 | ldr TAB:CARG1, TAB:CARG2->metatable
3171 | cbz TAB:CARG1, <1 // No metatable: done.
3172 | ldrb TMP1w, TAB:CARG1->nomm
3173 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
3176 |7: // Possible table write barrier for the value. Skip valiswhite check.
3177 | barrierback TAB:CARG2, TMP2w, TMP1
3181 | asr ITYPE, TMP1, #47
3182 | cmn ITYPE, #-LJ_TSTR // String key?
3184 | and STR:RC, TMP1, #LJ_GCVMASK
3190 | // RA = dst, RB = table, RC = str_const (~)
3191 | ldr CARG2, [BASE, RB, lsl #3]
3193 | ldr STR:RC, [KBASE, RC, lsl #3]
3194 | checktab CARG2, ->vmeta_tsets1
3196 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src
3197 | ldr TMP1w, TAB:CARG2->hmask
3198 | ldr TMP2w, STR:RC->sid
3199 | ldr NODE:CARG3, TAB:CARG2->node
3200 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
3201 | add TMP1, TMP1, TMP1, lsl #1
3202 | movn CARG4, #~LJ_TSTR
3203 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
3204 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
3205 | strb wzr, TAB:CARG2->nomm // Clear metamethod cache.
3207 | ldp TMP1, CARG1, NODE:CARG3->val
3208 | ldr NODE:TMP3, NODE:CARG3->next
3209 | ldrb TMP2w, TAB:CARG2->marked
3212 | ldr TMP0, [BASE, RA, lsl #3]
3213 | cmp_nil TMP1 // Previous value is nil?
3216 | str TMP0, NODE:CARG3->val
3217 | tbnz TMP2w, #2, >7 // isblack(table)
3221 |4: // Check for __newindex if previous value is nil.
3222 | ldr TAB:CARG1, TAB:CARG2->metatable
3223 | cbz TAB:CARG1, <2 // No metatable: done.
3224 | ldrb TMP1w, TAB:CARG1->nomm
3225 | tbnz TMP1w, #MM_newindex, <2 // 'no __newindex' flag set: done.
3228 |5: // Follow hash chain.
3229 | mov NODE:CARG3, NODE:TMP3
3230 | cbnz NODE:TMP3, <1
3231 | // End of hash chain: key not found, add a new one.
3233 | // But check for __newindex first.
3234 | ldr TAB:CARG1, TAB:CARG2->metatable
3235 | cbz TAB:CARG1, >6 // No metatable: continue.
3236 | ldrb TMP1w, TAB:CARG1->nomm
3237 | // 'no __newindex' flag NOT set: check.
3238 | tbz TMP1w, #MM_newindex, ->vmeta_tsets
3240 | movn TMP1, #~LJ_TSTR
3242 | add TMP0, STR:RC, TMP1, lsl #47
3246 | add CARG3, sp, TMPDofs
3247 | bl extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k)
3248 | // Returns TValue *.
3250 | ldr TMP0, [BASE, RA, lsl #3]
3252 | b <3 // No 2nd write barrier needed.
3254 |7: // Possible table write barrier for the value. Skip valiswhite check.
3255 | barrierback TAB:CARG2, TMP2w, TMP1
3261 | // RA = src, RB = table, RC = index
3262 | ldr CARG2, [BASE, RB, lsl #3]
3263 | checktab CARG2, ->vmeta_tsetb
3264 | ldr CARG3, TAB:CARG2->array
3265 | ldr CARG1w, TAB:CARG2->asize
3266 | add CARG3, CARG3, RC, lsl #3
3267 | cmp RCw, CARG1w // In array part?
3270 | ldr TMP0, [BASE, RA, lsl #3]
3271 | ldrb TMP2w, TAB:CARG2->marked
3272 | cmp_nil TMP1 // Previous value is nil?
3276 | tbnz TMP2w, #2, >7 // isblack(table)
3280 |5: // Check for __newindex if previous value is nil.
3281 | ldr TAB:CARG1, TAB:CARG2->metatable
3282 | cbz TAB:CARG1, <1 // No metatable: done.
3283 | ldrb TMP1w, TAB:CARG1->nomm
3284 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
3287 |7: // Possible table write barrier for the value. Skip valiswhite check.
3288 | barrierback TAB:CARG2, TMP2w, TMP1
3294 | // RA = src, RB = table, RC = key
3295 | ldr CARG2, [BASE, RB, lsl #3]
3296 | ldr TMP1, [BASE, RC, lsl #3]
3297 | and TAB:CARG2, CARG2, #LJ_GCVMASK
3298 | ldr CARG1, TAB:CARG2->array
3299 | ldrb TMP2w, TAB:CARG2->marked
3300 | ldr CARG4w, TAB:CARG2->asize
3301 | add CARG1, CARG1, TMP1, uxtw #3
3302 | tbnz TMP2w, #2, >7 // isblack(table)
3304 | cmp TMP1w, CARG4w // In array part?
3307 | ldr TMP0, [BASE, RA, lsl #3]
3311 |7: // Possible table write barrier for the value. Skip valiswhite check.
3312 | barrierback TAB:CARG2, TMP2w, TMP0
3317 | // RA = base (table at base-1), RC = num_const (start index)
3318 | add RA, BASE, RA, lsl #3
3320 | ldr RBw, SAVE_MULTRES
3321 | ldr TAB:CARG2, [RA, #-8] // Guaranteed to be a table.
3322 | ldr TMP1, [KBASE, RC, lsl #3] // Integer constant is in lo-word.
3324 | cbz RB, >4 // Nothing to copy?
3325 | and TAB:CARG2, CARG2, #LJ_GCVMASK
3326 | ldr CARG1w, TAB:CARG2->asize
3327 | add CARG3w, TMP1w, RBw, lsr #3
3328 | ldr CARG4, TAB:CARG2->array
3332 | add TMP1, CARG4, TMP1w, uxtw #3
3333 | ldrb TMP2w, TAB:CARG2->marked
3334 |3: // Copy result slots to table.
3335 | ldr TMP0, [RA], #8
3336 | str TMP0, [TMP1], #8
3339 | tbnz TMP2w, #2, >7 // isblack(table)
3343 |5: // Need to resize array part.
3347 | bl extern lj_tab_reasize // (lua_State *L, GCtab *t, int nasize)
3348 | // Must not reallocate the stack.
3351 |7: // Possible table write barrier for any value. Skip valiswhite check.
3352 | barrierback TAB:CARG2, TMP2w, TMP1
3356 /* -- Calls and vararg handling ----------------------------------------- */
3359 | // RA = base, (RB = nresults+1,) RC = extra_nargs
3360 | ldr TMP0w, SAVE_MULTRES
3361 | decode_RC8RD NARGS8:RC, RC
3362 | add NARGS8:RC, NARGS8:RC, TMP0
3366 | decode_RC8RD NARGS8:RC, RC
3367 | // RA = base, (RB = nresults+1,) RC = (nargs+1)*8
3369 | mov RB, BASE // Save old BASE for vmeta_call.
3370 | add BASE, BASE, RA, lsl #3
3371 | ldr CARG3, [BASE], #16
3372 | sub NARGS8:RC, NARGS8:RC, #8
3373 | checkfunc CARG3, ->vmeta_call
3378 | // RA = base, (RB = 0,) RC = extra_nargs
3379 | ldr TMP0w, SAVE_MULTRES
3380 | add NARGS8:RC, TMP0, RC, lsl #3
3384 | lsl NARGS8:RC, RC, #3
3385 | // RA = base, (RB = 0,) RC = (nargs+1)*8
3387 | add RA, BASE, RA, lsl #3
3388 | ldr TMP1, [RA], #16
3389 | sub NARGS8:RC, NARGS8:RC, #8
3390 | checktp CARG3, TMP1, LJ_TFUNC, ->vmeta_callt
3391 | ldr PC, [BASE, FRAME_PC]
3394 | ldrb TMP2w, LFUNC:CARG3->ffid
3395 | tst PC, #FRAME_TYPE
3398 | str TMP1, [BASE, FRAME_FUNC] // Copy function down, but keep PC.
3401 | ldr TMP0, [RA, RB]
3403 | cmp TMP1, NARGS8:RC
3404 | str TMP0, [BASE, RB]
3408 | cmp TMP2, #1 // (> FF_C) Calling a fast function?
3413 |5: // Tailcall to a fast function with a Lua frame below.
3414 | ldrb RAw, [PC, #-4+OFS_RA]
3415 | sub CARG1, BASE, RA, lsl #3
3416 | ldr LFUNC:CARG1, [CARG1, #-32]
3417 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3418 | ldr CARG1, LFUNC:CARG1->pc
3419 | ldr KBASE, [CARG1, #PC2PROTO(k)]
3422 |7: // Tailcall from a vararg function.
3423 | eor PC, PC, #FRAME_VARG
3424 | tst PC, #FRAME_TYPEP // Vararg frame below?
3425 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3427 | sub BASE, BASE, PC
3428 | ldr PC, [BASE, FRAME_PC]
3429 | tst PC, #FRAME_TYPE
3430 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3435 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3436 | add RA, BASE, RA, lsl #3
3437 | ldr CARG3, [RA, #-24]
3438 | mov RB, BASE // Save old BASE for vmeta_call.
3439 | ldp CARG1, CARG2, [RA, #-16]
3441 | mov NARGS8:RC, #16 // Iterators get 2 arguments.
3442 | str CARG3, [RA] // Copy callable.
3443 | stp CARG1, CARG2, [RA, #16] // Copy state and control var.
3444 | checkfunc CARG3, ->vmeta_call
3453 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3454 | add RA, BASE, RA, lsl #3
3455 | ldr TAB:RB, [RA, #-16]
3456 | ldrh TMP3w, [PC, # OFS_RD]
3457 | ldr CARG1w, [RA, #-8+LO] // Get index from control var.
3459 | add TMP3, PC, TMP3, lsl #2
3460 | and TAB:RB, RB, #LJ_GCVMASK
3461 | sub TMP3, TMP3, #0x20000
3462 | ldr TMP1w, TAB:RB->asize
3463 | ldr CARG2, TAB:RB->array
3464 |1: // Traverse array part.
3465 | subs RC, CARG1, TMP1
3466 | add CARG3, CARG2, CARG1, lsl #3
3467 | bhs >5 // Index points after array part?
3470 | cinc CARG1, CARG1, eq // Skip holes in array part.
3472 | add_TISNUM CARG1, CARG1
3473 | stp CARG1, TMP0, [RA]
3474 | add CARG1, CARG1, #1
3476 | str CARG1w, [RA, #-8+LO] // Update control var.
3481 |5: // Traverse hash part.
3482 | ldr TMP2w, TAB:RB->hmask
3483 | ldr NODE:RB, TAB:RB->node
3485 | add CARG1, RC, RC, lsl #1
3486 | cmp RC, TMP2 // End of iteration? Branch to ITERN+1.
3487 | add NODE:CARG3, NODE:RB, CARG1, lsl #3 // node = tab->node + idx*3*8
3489 | ldp TMP0, CARG1, NODE:CARG3->val
3492 | beq <6 // Skip holes in hash part.
3493 | stp CARG1, TMP0, [RA]
3494 | add CARG1, RC, TMP1
3499 | // RA = base, RC = target (points to ITERN)
3500 | add RA, BASE, RA, lsl #3
3501 | ldr CFUNC:CARG1, [RA, #-24]
3502 | add RC, PC, RC, lsl #2
3503 | ldp TAB:CARG3, CARG4, [RA, #-16]
3504 | sub RC, RC, #0x20000
3505 | checkfunc CFUNC:CARG1, >5
3506 | asr TMP0, TAB:CARG3, #47
3507 | ldrb TMP1w, CFUNC:CARG1->ffid
3509 | ccmn TMP0, #-LJ_TTAB, #0, eq
3510 | ccmp TMP1w, #FF_next_N, #0, eq
3512 | mov TMP0w, #0xfffe7fff // LJ_KEYINDEX
3513 | lsl TMP0, TMP0, #32
3514 | str TMP0, [RA, #-8] // Initialize control var.
3519 |5: // Despecialize bytecode if any of the checks fail.
3521 | ldrb TMP2w, [RC, # OFS_OP]
3524 | mov TMP1, #BC_ITERC
3525 | strb TMP0w, [PC, #-4+OFS_OP]
3527 | cmp TMP2w, #BC_ITERN
3530 | strb TMP1w, [RC, # OFS_OP]
3533 |6: // Unpatch JLOOP.
3534 | ldr RA, [GL, #GL_J(trace)]
3535 | ldrh TMP2w, [RC, # OFS_RD]
3536 | ldr TRACE:RA, [RA, TMP2, lsl #3]
3537 | ldr TMP2w, TRACE:RA->startins
3538 | bfxil TMP2w, TMP1w, #0, #8
3547 | // RA = base, RB = (nresults+1), RC = numparams
3548 | ldr TMP1, [BASE, FRAME_PC]
3549 | add TMP0, BASE, RC, lsl #3
3550 | add RC, BASE, RA, lsl #3 // RC = destination
3551 | add TMP0, TMP0, #FRAME_VARG
3552 | add TMP2, RC, RB, lsl #3
3553 | sub RA, TMP0, TMP1 // RA = vbase
3554 | // Note: RA may now be even _above_ BASE if nargs was < numparams.
3555 | sub TMP3, BASE, #16 // TMP3 = vtop
3557 | sub TMP2, TMP2, #16
3558 |1: // Copy vararg slots to destination slots.
3560 | ldr TMP0, [RA], #8
3561 | csinv TMP0, TMP0, xzr, lo // TISNIL = ~xzr
3563 | str TMP0, [RC], #8
3568 |5: // Copy all varargs.
3569 | ldr TMP0, L->maxstack
3570 | subs TMP2, TMP3, RA
3571 | csel RB, xzr, TMP2, le // MULTRES = (max(vtop-vbase,0)+1)*8
3573 | add TMP1, RC, TMP2
3574 | str RBw, SAVE_MULTRES
3575 | ble <2 // Nothing to copy.
3579 | ldr TMP0, [RA], #8
3580 | str TMP0, [RC], #8
3585 |7: // Grow stack for varargs.
3586 | lsr CARG2, TMP2, #3
3587 | stp BASE, RC, L->base
3589 | sub RA, RA, BASE // Need delta, because BASE may change.
3591 | bl extern lj_state_growstack // (lua_State *L, int n)
3592 | ldp BASE, RC, L->base
3594 | sub TMP3, BASE, #16
3598 /* -- Returns ----------------------------------------------------------- */
3601 | // RA = results, RC = extra results
3602 | ldr TMP0w, SAVE_MULTRES
3603 | ldr PC, [BASE, FRAME_PC]
3604 | add RA, BASE, RA, lsl #3
3605 | add RC, TMP0, RC, lsl #3
3610 | // RA = results, RC = nresults+1
3611 | ldr PC, [BASE, FRAME_PC]
3613 | add RA, BASE, RA, lsl #3
3615 | str RCw, SAVE_MULTRES
3617 | ands CARG1, PC, #FRAME_TYPE
3618 | eor CARG2, PC, #FRAME_VARG
3622 | // BASE = base, RA = resultptr, RC = (nresults+1)*8, PC = return
3623 | ldr INSw, [PC, #-4]
3625 | sub CARG3, BASE, #16
3628 | ldr TMP0, [RA], #8
3629 | add BASE, BASE, #8
3630 | sub TMP1, TMP1, #8
3631 | str TMP0, [BASE, #-24]
3635 | sub CARG4, CARG3, RA, lsl #3
3637 | ldr LFUNC:CARG1, [CARG4, FRAME_FUNC]
3639 | cmp RC, RB, lsl #3 // More results expected?
3641 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3643 | ldr CARG2, LFUNC:CARG1->pc
3644 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3647 |6: // Fill up results with nil.
3648 | add BASE, BASE, #8
3650 | str TISNIL, [BASE, #-24]
3653 |->BC_RETV1_Z: // Non-standard return case.
3654 | add RA, BASE, RA, lsl #3
3656 | tst CARG2, #FRAME_TYPEP
3658 | // Return from vararg function: relocate BASE down.
3659 | sub BASE, BASE, CARG2
3660 | ldr PC, [BASE, FRAME_PC]
3664 case BC_RET0: case BC_RET1:
3665 | // RA = results, RC = nresults+1
3666 | ldr PC, [BASE, FRAME_PC]
3668 | str RCw, SAVE_MULTRES
3669 | ands CARG1, PC, #FRAME_TYPE
3670 | eor CARG2, PC, #FRAME_VARG
3672 | ldr INSw, [PC, #-4]
3673 if (op == BC_RET1) {
3674 | ldr TMP0, [BASE, RA, lsl #3]
3676 | sub CARG4, BASE, #16
3678 | sub BASE, CARG4, RA, lsl #3
3679 if (op == BC_RET1) {
3680 | str TMP0, [CARG4], #8
3683 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3685 | cmp RC, RB, lsl #3
3687 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3688 | ldr CARG2, LFUNC:CARG1->pc
3689 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3692 |6: // Fill up results with nil.
3694 | str TISNIL, [CARG4], #8
3698 /* -- Loops and branches ------------------------------------------------ */
3700 |.define FOR_IDX, [RA]; .define FOR_TIDX, [RA, #4]
3701 |.define FOR_STOP, [RA, #8]; .define FOR_TSTOP, [RA, #12]
3702 |.define FOR_STEP, [RA, #16]; .define FOR_TSTEP, [RA, #20]
3703 |.define FOR_EXT, [RA, #24]; .define FOR_TEXT, [RA, #28]
3709 | // Fall through. Assumes BC_IFORL follows.
3719 | // RA = base, RC = target (after end of loop or start of loop)
3720 vk = (op == BC_IFORL || op == BC_JFORL);
3721 | add RA, BASE, RA, lsl #3
3722 | ldp CARG1, CARG2, FOR_IDX // CARG1 = IDX, CARG2 = STOP
3723 | ldr CARG3, FOR_STEP // CARG3 = STEP
3724 if (op != BC_JFORL) {
3725 | add RC, PC, RC, lsl #2
3726 | sub RC, RC, #0x20000
3728 | checkint CARG1, >5
3730 | checkint CARG2, ->vmeta_for
3731 | checkint CARG3, ->vmeta_for
3732 | tbnz CARG3w, #31, >4
3733 | cmp CARG1w, CARG2w
3735 | adds CARG1w, CARG1w, CARG3w
3737 | add_TISNUM TMP0, CARG1
3738 | tbnz CARG3w, #31, >4
3739 | cmp CARG1w, CARG2w
3742 if (op == BC_FORI) {
3743 | csel PC, RC, PC, gt
3744 } else if (op == BC_JFORI) {
3746 | ldrh RCw, [RC, #-4+OFS_RD]
3747 } else if (op == BC_IFORL) {
3748 | csel PC, RC, PC, le
3754 | str CARG1, FOR_EXT
3756 if (op == BC_JFORI || op == BC_JFORL) {
3762 |4: // Invert check for negative step.
3763 | cmp CARG2w, CARG1w
3767 | ldp d0, d1, FOR_IDX
3770 | checknum CARG2, ->vmeta_for
3771 | checknum CARG3, ->vmeta_for
3777 | tbnz CARG3, #63, >7
3784 if (op == BC_FORI) {
3785 | csel PC, RC, PC, hi
3786 } else if (op == BC_JFORI) {
3787 | ldrh RCw, [RC, #-4+OFS_RD]
3789 } else if (op == BC_IFORL) {
3790 | csel PC, RC, PC, ls
3796 |7: // Invert check for negative step.
3805 | // Fall through. Assumes BC_IITERL follows.
3813 | // RA = base, RC = target
3814 | ldr CARG1, [BASE, RA, lsl #3]
3815 | add TMP1, BASE, RA, lsl #3
3817 | beq >1 // Stop if iterator returned nil.
3818 if (op == BC_JITERL) {
3819 | str CARG1, [TMP1, #-8]
3822 | add TMP0, PC, RC, lsl #2 // Otherwise save control var + branch.
3823 | sub PC, TMP0, #0x20000
3824 | str CARG1, [TMP1, #-8]
3831 | // RA = base, RC = target (loop extent)
3832 | // Note: RA/RC is only used by trace recorder to determine scope/extent
3833 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
3837 | // Fall through. Assumes BC_ILOOP follows.
3841 | // RA = base, RC = target (loop extent)
3847 | // RA = base (ignored), RC = traceno
3848 | ldr CARG1, [GL, #GL_J(trace)]
3849 | st_vmstate wzr // Traces on ARM64 don't store the trace #, so use 0.
3850 | ldr TRACE:RC, [CARG1, RC, lsl #3]
3852 | ldr RA, TRACE:RC->mcauth
3854 | ldr RA, TRACE:RC->mcode
3856 | str BASE, GL->jit_base
3857 | str L, GL->tmpbuf.L
3858 | sub sp, sp, #16 // See SPS_FIXED. Avoids sp adjust in every root trace.
3868 | // RA = base (only used by trace recorder), RC = target
3869 | add RC, PC, RC, lsl #2
3870 | sub PC, RC, #0x20000
3874 /* -- Function headers -------------------------------------------------- */
3880 case BC_FUNCV: /* NYI: compiled vararg functions. */
3881 | // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
3889 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3890 | ldr CARG1, L->maxstack
3891 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3892 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3894 | bhi ->vm_growstack_l
3896 | cmp NARGS8:RC, TMP1, lsl #3 // Check for missing parameters.
3898 if (op == BC_JFUNCF) {
3905 |3: // Clear missing parameters.
3906 | str TISNIL, [BASE, NARGS8:RC]
3907 | add NARGS8:RC, NARGS8:RC, #8
3915 | NYI // NYI: compiled vararg functions
3916 break; /* NYI: compiled vararg functions. */
3919 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3920 | ldr CARG1, L->maxstack
3921 | movn TMP0, #~LJ_TFUNC
3922 | add TMP2, BASE, RC
3923 | add LFUNC:CARG3, CARG3, TMP0, lsl #47
3925 | sub CARG1, CARG1, #8
3926 | add TMP0, RC, #16+FRAME_VARG
3927 | str LFUNC:CARG3, [TMP2], #8 // Store (tagged) copy of LFUNC.
3928 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3930 | str TMP0, [TMP2], #8 // Store delta + FRAME_VARG.
3931 | bhs ->vm_growstack_l
3933 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3938 | cmp RA, RC // Less args than parameters?
3941 | sub TMP1, TMP1, #1
3942 | str TISNIL, [RA], #8 // Clear old fixarg slot (help the GC).
3943 | str TMP0, [TMP2], #8
3949 | sub TMP1, TMP1, #1
3950 | str TISNIL, [TMP2], #8
3957 | // BASE = new base, RA = BASE+framesize*8, CARG3 = CFUNC, RC = nargs*8
3958 if (op == BC_FUNCC) {
3959 | ldr CARG4, CFUNC:CARG3->f
3961 | ldr CARG4, GL->wrapf
3963 | add CARG2, RA, NARGS8:RC
3964 | ldr CARG1, L->maxstack
3965 | add RC, BASE, NARGS8:RC
3967 | stp BASE, RC, L->base
3968 if (op == BC_FUNCCW) {
3969 | ldr CARG2, CFUNC:CARG3->f
3971 | mv_vmstate TMP0w, C
3973 | bhi ->vm_growstack_c // Need to grow stack.
3975 | blr_auth CARG4 // (lua_State *L [, lua_CFunction f])
3976 | // Returns nresults.
3977 | ldp BASE, TMP1, L->base
3979 | sbfiz RC, CRET1, #3, #32
3980 | st_vmstate ST_INTERP
3981 | ldr PC, [BASE, FRAME_PC]
3982 | sub RA, TMP1, RC // RA = L->top - nresults*8
3986 /* ---------------------------------------------------------------------- */
3989 fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
3995 static int build_backend(BuildCtx *ctx)
3999 dasm_growpc(Dst, BC__MAX);
4001 build_subroutines(ctx);
4004 for (op = 0; op < BC__MAX; op++)
4005 build_ins(ctx, (BCOp)op, op);
4010 /* Emit pseudo frame-info for all assembler functions. */
4011 static void emit_asm_debug(BuildCtx *ctx)
4013 int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
4015 switch (ctx->mode) {
4017 fprintf(ctx->fp, "\t.section .debug_frame,\"\",%%progbits\n");
4020 "\t.long .LECIE0-.LSCIE0\n"
4022 "\t.long 0xffffffff\n"
4027 "\t.byte 30\n" /* Return address is in lr. */
4028 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4033 "\t.long .LEFDE0-.LASFDE0\n"
4035 "\t.long .Lframe0\n"
4038 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4039 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4041 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4042 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4043 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4044 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4045 64+i, i+(3+(28-19+1)-8));
4052 "\t.long .LEFDE1-.LASFDE1\n"
4054 "\t.long .Lframe0\n"
4055 "\t.quad lj_vm_ffi_call\n"
4057 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4058 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4059 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4060 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4062 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
4065 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",%%progbits\n");
4068 "\t.long .LECIE1-.LSCIE1\n"
4072 "\t.string \"zPR\"\n"
4075 "\t.byte 30\n" /* Return address is in lr. */
4076 "\t.uleb128 6\n" /* augmentation length */
4077 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4078 "\t.long lj_err_unwind_dwarf-.\n"
4079 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4080 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4085 "\t.long .LEFDE2-.LASFDE2\n"
4087 "\t.long .LASFDE2-.Lframe1\n"
4088 "\t.long .Lbegin-.\n"
4090 "\t.uleb128 0\n" /* augmentation length */
4091 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4092 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4094 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4095 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4096 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4097 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4098 64+i, i+(3+(28-19+1)-8));
4105 "\t.long .LECIE2-.LSCIE2\n"
4109 "\t.string \"zR\"\n"
4112 "\t.byte 30\n" /* Return address is in lr. */
4113 "\t.uleb128 1\n" /* augmentation length */
4114 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4115 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4120 "\t.long .LEFDE3-.LASFDE3\n"
4122 "\t.long .LASFDE3-.Lframe2\n"
4123 "\t.long lj_vm_ffi_call-.\n"
4125 "\t.uleb128 0\n" /* augmentation length */
4126 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4127 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4128 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4129 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4131 ".LEFDE3:\n\n", (int)ctx->codesz - fcofs);
4136 case BUILD_machasm: {
4141 fprintf(ctx->fp, "\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support\n");
4144 "\t.set L$set$x,LECIEX-LSCIEX\n"
4149 "\t.ascii \"zPR\\0\"\n"
4152 "\t.byte 30\n" /* Return address is in lr. */
4153 "\t.uleb128 6\n" /* augmentation length */
4154 "\t.byte 0x9b\n" /* indirect|pcrel|sdata4 */
4155 "\t.long _lj_err_unwind_dwarf@GOT-.\n"
4156 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4157 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4160 for (j = 0; j < ctx->nsym; j++) {
4161 const char *name = ctx->sym[j].name;
4162 int32_t size = ctx->sym[j+1].ofs - ctx->sym[j].ofs;
4163 if (size == 0) continue;
4165 if (!strcmp(name, "_lj_vm_ffi_call")) { fcsize = size; continue; }
4169 "\t.set L$set$%d,LEFDE%d-LASFDE%d\n"
4170 "\t.long L$set$%d\n"
4172 "\t.long LASFDE%d-EH_frame1\n"
4175 "\t.uleb128 0\n" /* augmentation length */
4176 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4177 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4178 j, j, j, j, j, j, j, name, size);
4179 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4180 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4181 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4182 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4183 64+i, i+(3+(28-19+1)-8));
4192 "\t.set L$set$y,LECIEY-LSCIEY\n"
4197 "\t.ascii \"zR\\0\"\n"
4200 "\t.byte 30\n" /* Return address is in lr. */
4201 "\t.uleb128 1\n" /* augmentation length */
4202 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4203 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4208 "\t.set L$set$yy,LEFDEY-LASFDEY\n"
4209 "\t.long L$set$yy\n"
4211 "\t.long LASFDEY-EH_frame2\n"
4212 "\t.long _lj_vm_ffi_call-.\n"
4214 "\t.uleb128 0\n" /* augmentation length */
4215 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4216 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4217 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4218 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4220 "LEFDEY:\n\n", fcsize);
4223 fprintf(ctx->fp, ".subsections_via_symbols\n");