src/runtime/arm64-assem.S

   1 #define LANGUAGE_ASSEMBLY
   2
   3 #include "globals.h"
   4 #include "lispregs.h"
   5 #include "sbcl.h"
   6
   7 #include "genesis/closure.h"
   8 #include "genesis/funcallable-instance.h"
   9 #include "genesis/fdefn.h"
  10 #include "genesis/static-symbols.h"
  11 #include "genesis/simple-fun.h"
  12 #include "genesis/symbol.h"
  13 #ifdef LISP_FEATURE_SB_THREAD
  14 #include "genesis/thread.h"
  15
  16 // For LDP/STP to work right
  17 #if THREAD_CONTROL_STACK_POINTER_OFFSET - THREAD_CONTROL_FRAME_POINTER_OFFSET != 8
  18 #error "THREAD_CONTROL_FRAME_POINTER_OFFSET - THREAD_CONTROL_STACK_POINTER_OFFSET not sequential"
  19 #endif
  20
  21 #endif
  22
  23 #define LOAD_STATIC_SYMBOL_VALUE(value,sym) \
  24         mov reg_TMP,((sym)-NIL+SYMBOL_VALUE_OFFSET) ;\
  25         ldr value,[reg_NULL, reg_TMP]
  26
  27 #define STORE_STATIC_SYMBOL_VALUE(value,sym) \
  28         mov reg_TMP,((sym)-NIL+SYMBOL_VALUE_OFFSET) ;\
  29         str value,[reg_NULL, reg_TMP]
  30 #ifdef LISP_FEATURE_SB_THREAD
  31         #define ENTER_PA \
  32         str reg_wNULL,[reg_THREAD,THREAD_PSEUDO_ATOMIC_BITS_OFFSET]
  33 #define LEAVE_PA \
  34         str wzr,[reg_THREAD,THREAD_PSEUDO_ATOMIC_BITS_OFFSET]       ;\
  35         ldr reg_wTMP,[reg_THREAD,THREAD_PSEUDO_ATOMIC_BITS_OFFSET+4] ;\
  36         cbz reg_wTMP,1f                                              ;\
  37         brk trap_PendingInterrupt                                   ;\
  38 1:
  39 #else
  40 #define ENTER_PA \
  41         STORE_STATIC_SYMBOL_VALUE(reg_CFP,PSEUDO_ATOMIC_ATOMIC)
  42
  43 #define LEAVE_PA \
  44         STORE_STATIC_SYMBOL_VALUE(reg_NULL,PSEUDO_ATOMIC_ATOMIC) ;\
  45         LOAD_STATIC_SYMBOL_VALUE(reg_TMP,PSEUDO_ATOMIC_INTERRUPTED) ;\
  46         cbz reg_TMP, 1f     ; \
  47         brk trap_PendingInterrupt         ; \
  48 1:
  49 #endif
  50         .align
  51         .global call_into_lisp
  52         .type   call_into_lisp, %function
  53 call_into_lisp:
  54         // At this point, we have:
  55         // X0 - function
  56         // X1 - pointer to args
  57         // X2 - number of args (unboxed)
  58         // There will be no more than three args, so we don't need to
  59         // worry about parameters to be passed on the stack.
  60
  61         // X19-X28 are callee-saved registers.
  62
  63         stp     x19,x20, [sp,#-160]!
  64         stp     x21,x22, [sp,#16]
  65         stp     x23,x24, [sp,#32]
  66         stp     x25,x26, [sp,#48]
  67         stp     x27,x28, [sp,#64]
  68         stp     x29,x30, [sp,#80] // save the return address in x30 aka LR
  69
  70         stp     d8,d9, [sp,#96]
  71         stp     d10,d11, [sp,#112]
  72         stp     d12,d13, [sp,#128]
  73         stp     d14,d15, [sp,#144]
  74
  75         // Start by finding NIL.
  76         ldr     reg_NULL, =NIL
  77
  78         // Set up NARGS.
  79         lsl     reg_NARGS, x2, #N_FIXNUM_TAG_BITS
  80
  81         // Move args pointer out of the way of the args to be loaded.
  82         mov     reg_R9, x1
  83
  84         // Move the function to its passing location.
  85         mov     reg_LEXENV, x0
  86
  87 #ifdef LISP_FEATURE_SB_THREAD
  88         ldr     x0, specials
  89         bl      pthread_getspecific
  90         mov     reg_THREAD, x0
  91 #else
  92         mov     reg_R10, #0
  93 #endif
  94         // Clear the boxed registers that don't already have something
  95         // in them.
  96         mov     reg_R2, #0
  97         mov     reg_R3, #0
  98         mov     reg_R4, #0
  99         mov     reg_R5, #0
 100         mov     reg_R6, #0
 101         mov     reg_R7, #0
 102         mov     reg_R8, #0
 103
 104         mov     reg_CODE, #0
 105
 106         // Find the lisp stack and frame pointers.  We're allocating a
 107         // new lisp stack frame, so load the stack pointer into CFP.
 108 #ifdef LISP_FEATURE_SB_THREAD
 109         ldp     reg_OCFP,reg_CFP, [reg_THREAD, THREAD_CONTROL_FRAME_POINTER_OFFSET]
 110 #else
 111         ldr     reg_OCFP, =current_control_frame_pointer
 112         ldr     reg_CFP, =current_control_stack_pointer
 113         ldr     reg_OCFP, [reg_OCFP]
 114         ldr     reg_CFP, [reg_CFP]
 115 #endif
 116         ENTER_PA
 117
 118
 119 #ifndef LISP_FEATURE_GENCGC
 120         // Copy the current allocation pointer into the symbol.
 121         ldr     reg_NL3, =dynamic_space_free_pointer
 122         ldr     reg_NL3, [reg_NL3]
 123         STORE_STATIC_SYMBOL_VALUE(reg_NL3, ALLOCATION_POINTER)
 124 #endif
 125
 126         // Clear FFCA, so the runtime knows that we're "in lisp".
 127 #ifdef LISP_FEATURE_SB_THREAD
 128         str     xzr,[reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 129 #else
 130         ldr     reg_NL3, =foreign_function_call_active
 131         str     xzr, [reg_NL3]
 132 #endif
 133         // We need to set up the lisp stack pointer and the basics of
 134         // our stack frame while we're still in P-A.  Any sooner and
 135         // our stack frame can be clobbered by a stray interrupt, any
 136         // later and we can end up with a half-configured stack frame
 137         // when we catch a stray interrupt.
 138
 139         // Allocate our frame and set up the Lisp stack pointer
 140         add     reg_CSP, reg_CFP, #16
 141
 142         // Set up the "frame link"
 143         str     reg_OCFP, [reg_CFP]
 144
 145         // Set up the return address
 146         ldr     reg_NL3, =.lra
 147         str     reg_NL3, [reg_CFP, #8]
 148
 149         LEAVE_PA
 150
 151         // Load our function args.
 152         cbz reg_NARGS, no_args
 153         cmp reg_NARGS, #2
 154         beq two_args
 155         bmi one_arg
 156 three_args:
 157         ldr     reg_R2, [reg_R9, #16]
 158 two_args:
 159         ldr     reg_R1, [reg_R9, #8]
 160 one_arg:
 161         ldr     reg_R0, [reg_R9]
 162 no_args:
 163
 164         // Load the closure-fun (or simple-fun-self), in case we're
 165         // trying to call a closure.
 166         ldr     reg_CODE, [reg_LEXENV, #CLOSURE_FUN_OFFSET]
 167
 168         // And, finally, call into Lisp!
 169         add     reg_LR, reg_CODE, #SIMPLE_FUN_CODE_OFFSET
 170         br      reg_LR
 171
 172         .align 4
 173         .equ    .lra, .+OTHER_POINTER_LOWTAG
 174         .dword  RETURN_PC_HEADER_WIDETAG
 175
 176         // Correct stack pointer for return processing.
 177         csel reg_CSP, reg_OCFP, reg_CSP, eq
 178
 179         // Return value
 180         mov     x0, reg_R0
 181
 182         ENTER_PA
 183
 184 // Save the lisp stack and frame pointers.
 185 #ifdef LISP_FEATURE_SB_THREAD
 186         stp     reg_CFP,reg_CSP, [reg_THREAD, THREAD_CONTROL_FRAME_POINTER_OFFSET]
 187         str     reg_CSP, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 188 #else
 189         ldr     reg_NFP, =current_control_frame_pointer
 190         str     reg_CFP, [reg_NFP]
 191         ldr     reg_OCFP, =current_control_stack_pointer
 192         str     reg_CSP, [reg_OCFP]
 193
 194         // Set FFCA, so the runtime knows that we're not "in lisp".
 195         ldr     reg_OCFP, =foreign_function_call_active
 196         str     reg_OCFP, [reg_OCFP]
 197 #endif
 198
 199 #ifndef LISP_FEATURE_GENCGC
 200         // Copy the current allocation pointer out from the symbol.
 201         ldr     reg_OCFP, =dynamic_space_free_pointer
 202         LOAD_STATIC_SYMBOL_VALUE(reg_NFP, ALLOCATION_POINTER)
 203         str     reg_NFP, [reg_OCFP]
 204 #endif
 205
 206         LEAVE_PA
 207
 208         // Restore saved registers.
 209
 210         ldp     d14,d15, [sp,#144]
 211         ldp     d12,d13, [sp,#128]
 212         ldp     d10,d11, [sp,#112]
 213         ldp     d8,d9, [sp,#96]
 214
 215
 216         ldp     x29,x30, [sp,#80]
 217         ldp     x27,x28, [sp,#64]
 218         ldp     x25,x26, [sp,#48]
 219         ldp     x23,x24, [sp,#32]
 220         ldp     x21,x22, [sp,#16]
 221         ldp     x19,x20, [sp],#160
 222
 223         ret
 224         .size   call_into_lisp, .-call_into_lisp
 225
 226
 227         .align
 228         .global call_into_c
 229         .type   call_into_c, %function
 230 call_into_c:
 231         // At this point, we have:
 232         // R8 -- C function to call.
 233         // LR -- Return address within the code component.
 234         // X0-X7 arguments
 235         // All other C arguments are already stashed on the C stack.
 236
 237         // We need to convert our return address to a GC-safe format,
 238         // build a stack frame to count for the "foreign" frame,
 239         // switch to C mode, move the register arguments to the
 240         // correct locations, call the C function, move the result to
 241         // the correct location, switch back to Lisp mode, tear down
 242         // our stack frame, restore the return address, and return to
 243         // our caller.
 244
 245         sub     reg_NARGS, reg_LR, reg_CODE
 246         add     reg_NFP, reg_NARGS, #OTHER_POINTER_LOWTAG
 247
 248         // Build a Lisp stack frame.  We need to stash our frame link,
 249         // the code component, and our return offset.  Frame link goes
 250         // in slot 0 (OCFP-SAVE-OFFSET), the offset (a FIXNUM) goes in
 251         // slot 1 (LRA-SAVE-OFFSET), and reg_CODE goes in slot 2.  The
 252         // debugger knows about this layout (see COMPUTE-CALLING-FRAME
 253         // in SYS:SRC;CODE;DEBUG-INT.LISP).
 254         add     reg_CSP, reg_CSP, #4*8
 255         stp     reg_CFP, reg_NFP, [reg_CSP, #-4*8]
 256         str     reg_CODE, [reg_CSP, #-2*8]
 257
 258         ENTER_PA
 259
 260         // Save the lisp stack and frame pointers.
 261
 262 #ifdef LISP_FEATURE_SB_THREAD
 263         sub     reg_TMP, reg_CSP, #4*8
 264         stp     reg_TMP,reg_CSP, [reg_THREAD, THREAD_CONTROL_FRAME_POINTER_OFFSET]
 265         str     reg_CSP, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 266 #else
 267         ldr     reg_NFP, =current_control_stack_pointer
 268         str     reg_CSP, [reg_NFP]
 269         ldr     reg_NFP, =current_control_frame_pointer
 270         sub     reg_TMP, reg_CSP, #4*8
 271         str     reg_TMP, [reg_NFP]
 272
 273         // Set FFCA, so the runtime knows that we're not "in lisp".
 274         ldr     reg_OCFP, =foreign_function_call_active
 275         str     reg_OCFP, [reg_OCFP]
 276 #endif
 277
 278 #ifndef LISP_FEATURE_GENCGC
 279         // Copy the current allocation pointer out from the symbol.
 280         ldr     reg_NL9, =dynamic_space_free_pointer
 281         LOAD_STATIC_SYMBOL_VALUE(reg_NFP, ALLOCATION_POINTER)
 282         str     reg_NFP, [reg_NL9]
 283 #endif
 284
 285         LEAVE_PA
 286
 287         // And call the C function.
 288         //
 289         // R8 is important for undefined_alien_function.
 290         blr      reg_R8
 291
 292         // We're back.  Our main tasks are to move the C return value
 293         // to where Lisp expects it, and to re-establish the Lisp
 294         // environment.
 295
 296         // Blank the boxed registers.
 297         mov     reg_R0, #0
 298         mov     reg_R1, #0
 299         mov     reg_R2, #0
 300         mov     reg_R3, #0
 301         mov     reg_R4, #0
 302         mov     reg_R5, #0
 303         mov     reg_R6, #0
 304         mov     reg_R7, #0
 305         mov     reg_R8, #0
 306         mov     reg_R9, #0
 307 #ifndef LISP_FEATURE_SB_THREAD
 308         mov     reg_R10, #0
 309 #endif
 310         mov     reg_LEXENV, #0
 311         mov     reg_CODE, #0
 312
 313         ENTER_PA
 314
 315         // Restore the Lisp stack and frame pointers, but store the
 316         // control frame pointer in reg_NFP (saving a register move
 317         // later).
 318 #ifdef LISP_FEATURE_SB_THREAD
 319         str     xzr, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 320
 321         ldp     reg_NFP,reg_CSP, [reg_THREAD, THREAD_CONTROL_FRAME_POINTER_OFFSET]
 322 #else
 323         // Clear FFCA, so the runtime knows that we're "in lisp".
 324         str     xzr, [reg_OCFP]
 325
 326 #ifndef LISP_FEATURE_GENCGC
 327         // Copy the current allocation pointer into the symbol.
 328         ldr     reg_NL9, =dynamic_space_free_pointer
 329         ldr     reg_NL9, [reg_NL9]
 330         STORE_STATIC_SYMBOL_VALUE(reg_OCFP, ALLOCATION_POINTER)
 331 #endif
 332
 333         ldr     reg_NFP, =current_control_stack_pointer
 334         ldr     reg_CSP, [reg_NFP]
 335         ldr     reg_NFP, =current_control_frame_pointer
 336         ldr     reg_NFP, [reg_NFP]
 337 #endif
 338         LEAVE_PA
 339
 340         // Restore our caller state from our stack frame.
 341         ldr     reg_CODE, [reg_NFP, #16]
 342         ldr     reg_CFP, [reg_NFP]
 343         mov     reg_CSP, reg_NFP
 344
 345         // Return
 346         add     reg_LR, reg_NARGS, reg_CODE // reg_NARGS has the offset from reg_CODE
 347         ret
 348
 349         .size   call_into_c, .-call_into_c
 350 \f
 351         // FIXME-ARM: The following is random garbage, to make
 352         // code/debug-int compile. To get the debugger working, this
 353         // needs to be implemented.
 354         .align
 355         .global fun_end_breakpoint_guts
 356         .type   fun_end_breakpoint_guts, %object
 357 fun_end_breakpoint_guts:
 358         .global fun_end_breakpoint_trap
 359         .type   fun_end_breakpoint_trap, %function
 360 fun_end_breakpoint_trap:
 361         b      fun_end_breakpoint_trap
 362         .global fun_end_breakpoint_end
 363 fun_end_breakpoint_end:
 364
 365 #ifdef LISP_FEATURE_GENCGC
 366         .align
 367         .global alloc_tramp
 368         .type   alloc_tramp, %function
 369 alloc_tramp:
 370         stp     reg_NL0,reg_NL1, [sp,#-80]!
 371         stp     reg_NL2,reg_NL3, [sp,#16]
 372         stp     reg_NL4,reg_NL5, [sp,#32]
 373         stp     reg_NL6,reg_NL7, [sp,#48]
 374         stp     reg_NL8,reg_NL9, [sp,#64]
 375
 376         mov     x0, reg_TMP // size
 377
 378 #ifdef LISP_FEATURE_SB_THREAD
 379         str     reg_CSP, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 380 #else
 381         ldr     reg_TMP, =foreign_function_call_active
 382         str     reg_TMP, [reg_TMP]
 383 #endif
 384         // Create a new frame
 385         add     reg_CSP,reg_CSP, #32+96
 386         stp     reg_CFP,reg_NULL,[reg_CSP, #-128]
 387         stp     reg_CODE,reg_LR,[reg_CSP, #-112]
 388
 389         stp     reg_R0,reg_R1, [reg_CSP,#-96]
 390         stp     reg_R2,reg_R3, [reg_CSP,#-80]
 391         stp     reg_R4,reg_R5, [reg_CSP,#-64]
 392         stp     reg_R6,reg_R7, [reg_CSP,#-48]
 393         stp     reg_R8,reg_R9, [reg_CSP,#-32]
 394         stp     x20,reg_LEXENV, [reg_CSP,#-16]
 395
 396         stp     q0, q1, [sp,#-512]!
 397         stp     q2,q3, [sp,#32]
 398         stp     q4,q5, [sp,#64]
 399         stp     q6,q7, [sp,#96]
 400         stp     q8,q9, [sp,#128]
 401         stp     q10,q11, [sp,#160]
 402         stp     q12,q13, [sp,#192]
 403         stp     q14,q15, [sp,#224]
 404         stp     q16,q17, [sp,#256]
 405         stp     q18,q19, [sp,#288]
 406         stp     q20,q21, [sp,#320]
 407         stp     q22,q23, [sp,#352]
 408         stp     q24,q25, [sp,#384]
 409         stp     q26,q27, [sp,#416]
 410         stp     q28,q29, [sp,#448]
 411         stp     q30,q31, [sp,#480]
 412
 413         bl      alloc
 414
 415         ldp     q30,q31, [sp,#480]
 416         ldp     q28,q29, [sp,#448]
 417         ldp     q26,q27, [sp,#416]
 418         ldp     q24,q25, [sp,#384]
 419         ldp     q22,q23, [sp,#352]
 420         ldp     q20,q21, [sp,#320]
 421         ldp     q18,q19, [sp,#288]
 422         ldp     q16,q17, [sp,#256]
 423         ldp     q14,q15, [sp,#224]
 424         ldp     q12,q13, [sp,#192]
 425         ldp     q10,q11, [sp,#160]
 426         ldp     q8,q9, [sp,#128]
 427         ldp     q6,q7, [sp,#96]
 428         ldp     q4,q5, [sp,#64]
 429         ldp     q2,q3, [sp,#32]
 430         ldp     q0, q1, [sp],#512
 431
 432         ldp     x20, reg_LEXENV, [reg_CSP, #-16]
 433         ldp     reg_R8,reg_R9, [reg_CSP,#-32]
 434         ldp     reg_R6,reg_R7, [reg_CSP,#-48]
 435         ldp     reg_R4,reg_R5, [reg_CSP,#-64]
 436         ldp     reg_R2,reg_R3, [reg_CSP,#-80]
 437         ldp     reg_R0,reg_R1, [reg_CSP,#-96]
 438
 439         ldr     reg_LR,[reg_CSP,#-104]
 440
 441         sub     reg_CSP, reg_CSP, #32+96 // deallocate the frame
 442
 443         // foreign_function_call_active
 444 #ifdef LISP_FEATURE_SB_THREAD
 445         str     xzr, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 446 #else
 447         str     xzr, [reg_TMP]
 448 #endif
 449
 450         mov     reg_TMP, x0
 451
 452         ldp     reg_NL8,reg_NL9, [sp,#64]
 453         ldp     reg_NL6,reg_NL7, [sp,#48]
 454         ldp     reg_NL4,reg_NL5, [sp,#32]
 455         ldp     reg_NL2,reg_NL3, [sp,#16]
 456         ldp     reg_NL0,reg_NL1, [sp],#80
 457
 458         ret
 459
 460         .align
 461         .global do_pending_interrupt
 462         .type   do_pending_interrupt, %function
 463 do_pending_interrupt:
 464         brk trap_PendingInterrupt
 465         ret
 466         #endif