src/runtime/arm64-assem.S

   1 #define LANGUAGE_ASSEMBLY
   2
   3 #include "globals.h"
   4 #include "lispregs.h"
   5 #include "sbcl.h"
   6
   7 #include "genesis/closure.h"
   8 #include "genesis/funcallable-instance.h"
   9 #include "genesis/fdefn.h"
  10 #include "genesis/static-symbols.h"
  11 #include "genesis/simple-fun.h"
  12 #include "genesis/symbol.h"
  13 #ifdef LISP_FEATURE_SB_THREAD
  14 #include "genesis/thread.h"
  15
  16 // For LDP/STP to work right
  17 #if THREAD_CONTROL_STACK_POINTER_OFFSET - THREAD_CONTROL_FRAME_POINTER_OFFSET != 8
  18 #error "THREAD_CONTROL_FRAME_POINTER_OFFSET - THREAD_CONTROL_STACK_POINTER_OFFSET not sequential"
  19 #endif
  20
  21 #endif
  22
  23 #define LOAD_STATIC_SYMBOL_VALUE(value,sym) \
  24         mov reg_TMP,((sym)-NIL+SYMBOL_VALUE_OFFSET) ;\
  25         ldr value,[reg_NULL, reg_TMP]
  26
  27 #define STORE_STATIC_SYMBOL_VALUE(value,sym) \
  28         mov reg_TMP,((sym)-NIL+SYMBOL_VALUE_OFFSET) ;\
  29         str value,[reg_NULL, reg_TMP]
  30 #ifdef LISP_FEATURE_SB_THREAD
  31         #define ENTER_PA \
  32         str reg_wNULL,[reg_THREAD,THREAD_PSEUDO_ATOMIC_BITS_OFFSET]
  33 #define LEAVE_PA \
  34         str wzr,[reg_THREAD,THREAD_PSEUDO_ATOMIC_BITS_OFFSET]       ;\
  35         ldr reg_wTMP,[reg_THREAD,THREAD_PSEUDO_ATOMIC_BITS_OFFSET+4] ;\
  36         cbz reg_wTMP,1f                                              ;\
  37         brk trap_PendingInterrupt                                   ;\
  38 1:
  39 #else
  40 #define ENTER_PA \
  41         STORE_STATIC_SYMBOL_VALUE(reg_CFP,PSEUDO_ATOMIC_ATOMIC)
  42
  43 #define LEAVE_PA \
  44         STORE_STATIC_SYMBOL_VALUE(reg_NULL,PSEUDO_ATOMIC_ATOMIC) ;\
  45         LOAD_STATIC_SYMBOL_VALUE(reg_TMP,PSEUDO_ATOMIC_INTERRUPTED) ;\
  46         cbz     reg_TMP, 1f     ; \
  47         blr     reg_TMP         ; \
  48 1:
  49 #endif
  50         .align
  51         .global call_into_lisp
  52         .type   call_into_lisp, %function
  53 call_into_lisp:
  54         // At this point, we have:
  55         // X0 - function
  56         // X1 - pointer to args
  57         // X2 - number of args (unboxed)
  58         // There will be no more than three args, so we don't need to
  59         // worry about parameters to be passed on the stack.
  60
  61         // X19-X28 are callee-saved registers.
  62
  63         stp     x19,x20, [sp,#-160]!
  64         stp     x21,x22, [sp,#16]
  65         stp     x23,x24, [sp,#32]
  66         stp     x25,x26, [sp,#48]
  67         stp     x27,x28, [sp,#64]
  68         stp     x29,x30, [sp,#80] // save the return address in x30 aka LR
  69
  70         stp     d8,d9, [sp,#96]
  71         stp     d10,d11, [sp,#112]
  72         stp     d12,d13, [sp,#128]
  73         stp     d14,d15, [sp,#144]
  74
  75         // Start by finding NIL.
  76         ldr     reg_NULL, .known_nil
  77
  78         // Set up NARGS.
  79         lsl     reg_NARGS, x2, #N_FIXNUM_TAG_BITS
  80
  81         // Move args pointer out of the way of the args to be loaded.
  82         mov     reg_R9, x1
  83
  84         // Move the function to its passing location.
  85         mov     reg_LEXENV, x0
  86
  87 #ifdef LISP_FEATURE_SB_THREAD
  88         ldr     x0, specials
  89         bl      pthread_getspecific
  90         mov     reg_THREAD, x0
  91 #else
  92         mov     reg_R10, #0
  93 #endif
  94         // Clear the boxed registers that don't already have something
  95         // in them.
  96         mov     reg_R2, #0
  97         mov     reg_R3, #0
  98         mov     reg_R4, #0
  99         mov     reg_R5, #0
 100         mov     reg_R6, #0
 101         mov     reg_R7, #0
 102         mov     reg_R8, #0
 103
 104         mov     reg_CODE, #0
 105
 106         // Find the lisp stack and frame pointers.  We're allocating a
 107         // new lisp stack frame, so load the stack pointer into CFP.
 108 #ifdef LISP_FEATURE_SB_THREAD
 109         ldp     reg_OCFP,reg_CFP, [reg_THREAD, THREAD_CONTROL_FRAME_POINTER_OFFSET]
 110 #else
 111         ldr     reg_OCFP, =current_control_frame_pointer
 112         ldr     reg_CFP, =current_control_stack_pointer
 113         ldr     reg_OCFP, [reg_OCFP]
 114         ldr     reg_CFP, [reg_CFP]
 115 #endif
 116         ENTER_PA
 117
 118
 119 #ifndef LISP_FEATURE_GENCGC
 120         // Copy the current allocation pointer into the symbol.
 121         ldr     reg_NL3, =dynamic_space_free_pointer
 122         ldr     reg_NL3, [reg_NL3]
 123         STORE_STATIC_SYMBOL_VALUE(reg_NL3, ALLOCATION_POINTER)
 124 #endif
 125
 126         // Clear FFCA, so the runtime knows that we're "in lisp".
 127 #ifdef LISP_FEATURE_SB_THREAD
 128         str     xzr,[reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 129 #else
 130         ldr     reg_NL3, =foreign_function_call_active
 131         str     xzr, [reg_NL3]
 132 #endif
 133         // We need to set up the lisp stack pointer and the basics of
 134         // our stack frame while we're still in P-A.  Any sooner and
 135         // our stack frame can be clobbered by a stray interrupt, any
 136         // later and we can end up with a half-configured stack frame
 137         // when we catch a stray interrupt.
 138
 139         // Allocate our frame and set up the Lisp stack pointer
 140         add     reg_CSP, reg_CFP, #16
 141
 142         // Set up the "frame link"
 143         str     reg_OCFP, [reg_CFP]
 144
 145         // Set up the return address
 146         ldr     reg_NL3, =.lra
 147         str     reg_NL3, [reg_CFP, #8]
 148
 149         LEAVE_PA
 150
 151         // Load our function args.
 152         cbz reg_NARGS, no_args
 153         cmp reg_NARGS, #2
 154         beq two_args
 155         bmi one_arg
 156 three_args:
 157         ldr     reg_R2, [reg_R9, #16]
 158 two_args:
 159         ldr     reg_R1, [reg_R9, #8]
 160 one_arg:
 161         ldr     reg_R0, [reg_R9]
 162 no_args:
 163
 164         // Load the closure-fun (or simple-fun-self), in case we're
 165         // trying to call a closure.
 166         ldr     reg_CODE, [reg_LEXENV, #CLOSURE_FUN_OFFSET]
 167
 168         // And, finally, call into Lisp!
 169         add     reg_LR, reg_CODE, #SIMPLE_FUN_CODE_OFFSET
 170         br      reg_LR
 171
 172         .align 4
 173         .equ    .lra, .+OTHER_POINTER_LOWTAG
 174         .dword  RETURN_PC_HEADER_WIDETAG
 175
 176         // Correct stack pointer for return processing.
 177         csel reg_CSP, reg_OCFP, reg_CSP, eq
 178
 179         // Return value
 180         mov     x0, reg_R0
 181
 182         ENTER_PA
 183
 184 // Save the lisp stack and frame pointers.
 185 #ifdef LISP_FEATURE_SB_THREAD
 186         stp     reg_CFP,reg_CSP, [reg_THREAD, THREAD_CONTROL_FRAME_POINTER_OFFSET]
 187         str     reg_CSP, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 188 #else
 189         ldr     reg_NFP, =current_control_frame_pointer
 190         str     reg_CFP, [reg_NFP]
 191         ldr     reg_OCFP, =current_control_stack_pointer
 192         str     reg_CSP, [reg_OCFP]
 193
 194         // Set FFCA, so the runtime knows that we're not "in lisp".
 195         ldr     reg_OCFP, =foreign_function_call_active
 196         str     reg_OCFP, [reg_OCFP]
 197 #endif
 198
 199 #ifndef LISP_FEATURE_GENCGC
 200         // Copy the current allocation pointer out from the symbol.
 201         ldr     reg_OCFP, =dynamic_space_free_pointer
 202         LOAD_STATIC_SYMBOL_VALUE(reg_NFP, ALLOCATION_POINTER)
 203         str     reg_NFP, [reg_OCFP]
 204 #endif
 205
 206         LEAVE_PA
 207
 208         // Restore saved registers.
 209
 210         ldp     d14,d15, [sp,#144]
 211         ldp     d12,d13, [sp,#128]
 212         ldp     d10,d11, [sp,#112]
 213         ldp     d8,d9, [sp,#96]
 214
 215
 216         ldp     x29,x30, [sp,#80]
 217         ldp     x27,x28, [sp,#64]
 218         ldp     x25,x26, [sp,#48]
 219         ldp     x23,x24, [sp,#32]
 220         ldp     x21,x22, [sp,#16]
 221         ldp     x19,x20, [sp],#160
 222
 223         ret
 224         .size   call_into_lisp, .-call_into_lisp
 225
 226
 227         .align
 228         .global call_into_c
 229         .type   call_into_c, %function
 230 call_into_c:
 231         // At this point, we have:
 232         // R8 -- C function to call.
 233         // LR -- Return address within the code component.
 234         // X0-X7 arguments
 235         // All other C arguments are already stashed on the C stack.
 236
 237         // We need to convert our return address to a GC-safe format,
 238         // build a stack frame to count for the "foreign" frame,
 239         // switch to C mode, move the register arguments to the
 240         // correct locations, call the C function, move the result to
 241         // the correct location, switch back to Lisp mode, tear down
 242         // our stack frame, restore the return address, and return to
 243         // our caller.
 244
 245         sub     reg_NARGS, reg_LR, reg_CODE
 246         add     reg_NFP, reg_NARGS, #OTHER_POINTER_LOWTAG
 247
 248         // Build a Lisp stack frame.  We need to stash our frame link,
 249         // the code component, and our return offset.  Frame link goes
 250         // in slot 0 (OCFP-SAVE-OFFSET), the offset (a FIXNUM) goes in
 251         // slot 1 (LRA-SAVE-OFFSET), and reg_CODE goes in slot 2.  The
 252         // debugger knows about this layout (see COMPUTE-CALLING-FRAME
 253         // in SYS:SRC;CODE;DEBUG-INT.LISP).
 254         add     reg_CSP, reg_CSP, #4*8
 255         stp     reg_CFP, reg_NFP, [reg_CSP, #-4*8]
 256         str     reg_CODE, [reg_CSP, #-2*8]
 257
 258         ENTER_PA
 259
 260         // Save the lisp stack and frame pointers.
 261
 262 #ifdef LISP_FEATURE_SB_THREAD
 263         sub     reg_TMP, reg_CSP, #4*8
 264         stp     reg_TMP,reg_CSP, [reg_THREAD, THREAD_CONTROL_FRAME_POINTER_OFFSET]
 265         str     reg_CSP, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 266 #else
 267         ldr     reg_NFP, =current_control_stack_pointer
 268         str     reg_CSP, [reg_NFP]
 269         ldr     reg_NFP, =current_control_frame_pointer
 270         sub     reg_TMP, reg_CSP, #4*8
 271         str     reg_TMP, [reg_NFP]
 272
 273         // Set FFCA, so the runtime knows that we're not "in lisp".
 274         ldr     reg_OCFP, =foreign_function_call_active
 275         str     reg_OCFP, [reg_OCFP]
 276 #endif
 277
 278 #ifndef LISP_FEATURE_GENCGC
 279         // Copy the current allocation pointer out from the symbol.
 280         ldr     reg_NL9, =dynamic_space_free_pointer
 281         LOAD_STATIC_SYMBOL_VALUE(reg_NFP, ALLOCATION_POINTER)
 282         str     reg_NFP, [reg_NL9]
 283 #endif
 284
 285         LEAVE_PA
 286
 287         // And call the C function.
 288         //
 289         // R8 is important for undefined_alien_function.
 290         blr      reg_R8
 291
 292         // We're back.  Our main tasks are to move the C return value
 293         // to where Lisp expects it, and to re-establish the Lisp
 294         // environment.
 295
 296         // Blank the boxed registers.
 297         mov     reg_R0, #0
 298         mov     reg_R1, #0
 299         mov     reg_R2, #0
 300         mov     reg_R3, #0
 301         mov     reg_R4, #0
 302         mov     reg_R5, #0
 303         mov     reg_R6, #0
 304         mov     reg_R7, #0
 305         mov     reg_R8, #0
 306         mov     reg_R9, #0
 307 #ifndef LISP_FEATURE_SB_THREAD
 308         mov     reg_R10, #0
 309 #endif
 310         mov     reg_LEXENV, #0
 311         mov     reg_CODE, #0
 312
 313         ENTER_PA
 314
 315         // Restore the Lisp stack and frame pointers, but store the
 316         // control frame pointer in reg_NFP (saving a register move
 317         // later).
 318 #ifdef LISP_FEATURE_SB_THREAD
 319         str     xzr, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 320
 321         ldp     reg_NFP,reg_CSP, [reg_THREAD, THREAD_CONTROL_FRAME_POINTER_OFFSET]
 322 #else
 323         // Clear FFCA, so the runtime knows that we're "in lisp".
 324         str     xzr, [reg_OCFP]
 325
 326 #ifndef LISP_FEATURE_GENCGC
 327         // Copy the current allocation pointer into the symbol.
 328         ldr     reg_NL9, =dynamic_space_free_pointer
 329         ldr     reg_NL9, [reg_NL9]
 330         STORE_STATIC_SYMBOL_VALUE(reg_OCFP, ALLOCATION_POINTER)
 331 #endif
 332
 333         ldr     reg_NFP, =current_control_stack_pointer
 334         ldr     reg_CSP, [reg_NFP]
 335         ldr     reg_NFP, =current_control_frame_pointer
 336         ldr     reg_NFP, [reg_NFP]
 337 #endif
 338         LEAVE_PA
 339
 340         // Restore our caller state from our stack frame.
 341         ldr     reg_CODE, [reg_NFP, #16]
 342         ldr     reg_CFP, [reg_NFP]
 343         mov     reg_CSP, reg_NFP
 344
 345         // Return
 346         add     reg_LR, reg_NARGS, reg_CODE // reg_NARGS has the offset from reg_CODE
 347         ret
 348
 349         .size   call_into_c, .-call_into_c
 350
 351         .align  4
 352         .global undefined_tramp
 353         .type   undefined_tramp, %object
 354         .equ    undefined_tramp_header, . + FUN_POINTER_LOWTAG
 355         .dword  SIMPLE_FUN_HEADER_WIDETAG
 356         .dword  undefined_tramp_header
 357 .known_nil:
 358         .dword  NIL
 359         .dword  NIL
 360         .dword  NIL
 361         .dword  NIL
 362         .dword  NIL
 363 undefined_tramp:
 364
 365         // As in ppc-assem.S, point reg_CODE to the header with a
 366         // function lowtag
 367         adr     reg_CODE, undefined_tramp_header
 368
 369
 370         brk UNDEFINED_FUN_ERROR << 8 | trap_Error
 371
 372         .byte  .error_args_end - . - 1
 373         // Encode LEXENV
 374         // produced with
 375         // (let ((result (make-array 10 :fill-pointer 0)))
 376         //  (sb-c:write-var-integer
 377         //   (sb-c:make-sc-offset sb-vm:descriptor-reg-sc-number sb-vm::lexenv-offset)
 378         //   result)
 379         //  result)
 380         .byte   254
 381         .byte   69
 382         .byte   5
 383 .error_args_end:
 384
 385 \f
 386         .align  4
 387         .global undefined_alien_function
 388         .type   undefined_alien_function, %object
 389         .equ    undefined_alien_function_header, . + FUN_POINTER_LOWTAG
 390         .dword  SIMPLE_FUN_HEADER_WIDETAG
 391         .dword  undefined_alien_function_header
 392         .dword  NIL
 393         .dword  NIL
 394         .dword  NIL
 395         .dword  NIL
 396         .dword  NIL
 397 undefined_alien_function:
 398         adr     reg_CODE, undefined_alien_function_header
 399
 400         brk     UNDEFINED_ALIEN_FUN_ERROR << 8 | trap_Error
 401
 402         // Error arguments for an undefined function.
 403         .byte   3
 404         // Encode unsigned R8, which comes from call_into_c
 405         .byte   0xFE
 406         .byte   0x92
 407         .byte   0x04
 408 \f
 409
 410         .align  4
 411         .global closure_tramp
 412         .type   closure_tramp, %object
 413         .dword  SIMPLE_FUN_HEADER_WIDETAG
 414         .dword  closure_tramp - SIMPLE_FUN_CODE_OFFSET
 415         .dword  NIL
 416         .dword  NIL
 417         .dword  NIL
 418         .dword  NIL
 419         .dword  NIL
 420 closure_tramp:
 421         ldr     reg_LEXENV, [reg_LEXENV, #FDEFN_FUN_OFFSET]
 422         ldr     reg_CODE, [reg_LEXENV, #CLOSURE_FUN_OFFSET]
 423         add     reg_LR, reg_CODE, #SIMPLE_FUN_CODE_OFFSET
 424         br      reg_LR
 425
 426         .align  4
 427         .global funcallable_instance_tramp
 428         .type   funcallable_instance_tramp, %object
 429         .equ    funcallable_instance_tramp, .+ FUN_POINTER_LOWTAG
 430         .dword  SIMPLE_FUN_HEADER_WIDETAG
 431         .dword  funcallable_instance_tramp
 432         .dword  NIL
 433         .dword  NIL
 434         .dword  NIL
 435         .dword  NIL
 436         .dword  NIL
 437         ldr     reg_LEXENV, [reg_LEXENV, #FUNCALLABLE_INSTANCE_FUNCTION_OFFSET]
 438         ldr     reg_CODE, [reg_LEXENV, #CLOSURE_FUN_OFFSET]
 439         add     reg_LR, reg_CODE, #SIMPLE_FUN_CODE_OFFSET
 440         br      reg_LR
 441
 442         // FIXME-ARM: The following is random garbage, to make
 443         // code/debug-int compile. To get the debugger working, this
 444         // needs to be implemented.
 445         .align
 446         .global fun_end_breakpoint_guts
 447         .type   fun_end_breakpoint_guts, %object
 448 fun_end_breakpoint_guts:
 449         .global fun_end_breakpoint_trap
 450         .type   fun_end_breakpoint_trap, %function
 451 fun_end_breakpoint_trap:
 452         b      fun_end_breakpoint_trap
 453         .global fun_end_breakpoint_end
 454 fun_end_breakpoint_end:
 455
 456 #ifdef LISP_FEATURE_GENCGC
 457         .align
 458         .global alloc_tramp
 459         .type   alloc_tramp, %function
 460 alloc_tramp:
 461         stp     reg_NL0,reg_NL1, [sp,#-80]!
 462         stp     reg_NL2,reg_NL3, [sp,#16]
 463         stp     reg_NL4,reg_NL5, [sp,#32]
 464         stp     reg_NL6,reg_NL7, [sp,#48]
 465         stp     reg_NL8,reg_NL9, [sp,#64]
 466
 467         mov     x0, reg_TMP // size
 468
 469 #ifdef LISP_FEATURE_SB_THREAD
 470         str     reg_CSP, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 471 #else
 472         ldr     reg_TMP, =foreign_function_call_active
 473         str     reg_TMP, [reg_TMP]
 474 #endif
 475         // Create a new frame
 476         add     reg_CSP,reg_CSP, #32+96
 477         stp     reg_CFP,reg_NULL,[reg_CSP, #-128]
 478         stp     reg_CODE,reg_LR,[reg_CSP, #-112]
 479
 480         stp     reg_R0,reg_R1, [reg_CSP,#-96]
 481         stp     reg_R2,reg_R3, [reg_CSP,#-80]
 482         stp     reg_R4,reg_R5, [reg_CSP,#-64]
 483         stp     reg_R6,reg_R7, [reg_CSP,#-48]
 484         stp     reg_R8,reg_R9, [reg_CSP,#-32]
 485         stp     x20,reg_LEXENV, [reg_CSP,#-16]
 486
 487         stp     q0, q1, [sp,#-512]!
 488         stp     q2,q3, [sp,#32]
 489         stp     q4,q5, [sp,#64]
 490         stp     q6,q7, [sp,#96]
 491         stp     q8,q9, [sp,#128]
 492         stp     q10,q11, [sp,#160]
 493         stp     q12,q13, [sp,#192]
 494         stp     q14,q15, [sp,#224]
 495         stp     q16,q17, [sp,#256]
 496         stp     q18,q19, [sp,#288]
 497         stp     q20,q21, [sp,#320]
 498         stp     q22,q23, [sp,#352]
 499         stp     q24,q25, [sp,#384]
 500         stp     q26,q27, [sp,#416]
 501         stp     q28,q29, [sp,#448]
 502         stp     q30,q31, [sp,#480]
 503
 504         bl      alloc
 505
 506         ldp     q30,q31, [sp,#480]
 507         ldp     q28,q29, [sp,#448]
 508         ldp     q26,q27, [sp,#416]
 509         ldp     q24,q25, [sp,#384]
 510         ldp     q22,q23, [sp,#352]
 511         ldp     q20,q21, [sp,#320]
 512         ldp     q18,q19, [sp,#288]
 513         ldp     q16,q17, [sp,#256]
 514         ldp     q14,q15, [sp,#224]
 515         ldp     q12,q13, [sp,#192]
 516         ldp     q10,q11, [sp,#160]
 517         ldp     q8,q9, [sp,#128]
 518         ldp     q6,q7, [sp,#96]
 519         ldp     q4,q5, [sp,#64]
 520         ldp     q2,q3, [sp,#32]
 521         ldp     q0, q1, [sp],#512
 522
 523         ldp     x20, reg_LEXENV, [reg_CSP, #-16]
 524         ldp     reg_R8,reg_R9, [reg_CSP,#-32]
 525         ldp     reg_R6,reg_R7, [reg_CSP,#-48]
 526         ldp     reg_R4,reg_R5, [reg_CSP,#-64]
 527         ldp     reg_R2,reg_R3, [reg_CSP,#-80]
 528         ldp     reg_R0,reg_R1, [reg_CSP,#-96]
 529
 530         ldr     reg_LR,[reg_CSP,#-104]
 531
 532         sub     reg_CSP, reg_CSP, #32+96 // deallocate the frame
 533
 534         // foreign_function_call_active
 535 #ifdef LISP_FEATURE_SB_THREAD
 536         str     xzr, [reg_THREAD, THREAD_FOREIGN_FUNCTION_CALL_ACTIVE_OFFSET]
 537 #else
 538         str     xzr, [reg_TMP]
 539 #endif
 540
 541         mov     reg_TMP, x0
 542
 543         ldp     reg_NL8,reg_NL9, [sp,#64]
 544         ldp     reg_NL6,reg_NL7, [sp,#48]
 545         ldp     reg_NL4,reg_NL5, [sp,#32]
 546         ldp     reg_NL2,reg_NL3, [sp,#16]
 547         ldp     reg_NL0,reg_NL1, [sp],#80
 548
 549         ret
 550
 551         .align
 552         .global fpu_save
 553         .type   fpu_save, %function
 554 fpu_save:
 555         stp     q0, q1, [x0]
 556         stp     q2,q3, [x0,#32]
 557         stp     q4,q5, [x0,#64]
 558         stp     q6,q7, [x0,#96]
 559         stp     q8,q9, [x0,#128]
 560         stp     q10,q11, [x0,#160]
 561         stp     q12,q13, [x0,#192]
 562         stp     q14,q15, [x0,#224]
 563         stp     q16,q17, [x0,#256]
 564         stp     q18,q19, [x0,#288]
 565         stp     q20,q21, [x0,#320]
 566         stp     q22,q23, [x0,#352]
 567         stp     q24,q25, [x0,#384]
 568         stp     q26,q27, [x0,#416]
 569         stp     q28,q29, [x0,#448]
 570         stp     q30,q31, [x0,#480]
 571         ret
 572
 573         .align
 574         .global fpu_restore
 575         .type   fpu_restore, %function
 576 fpu_restore:
 577         ldp     q0, q1, [x0]
 578         ldp     q2,q3, [x0,#32]
 579         ldp     q4,q5, [x0,#64]
 580         ldp     q6,q7, [x0,#96]
 581         ldp     q8,q9, [x0,#128]
 582         ldp     q10,q11, [x0,#160]
 583         ldp     q12,q13, [x0,#192]
 584         ldp     q14,q15, [x0,#224]
 585         ldp     q16,q17, [x0,#256]
 586         ldp     q18,q19, [x0,#288]
 587         ldp     q20,q21, [x0,#320]
 588         ldp     q22,q23, [x0,#352]
 589         ldp     q24,q25, [x0,#384]
 590         ldp     q26,q27, [x0,#416]
 591         ldp     q28,q29, [x0,#448]
 592         ldp     q30,q31, [x0,#480]
 593         ret
 594
 595         .align
 596         .global do_pending_interrupt
 597         .type   do_pending_interrupt, %function
 598 do_pending_interrupt:
 599         brk trap_PendingInterrupt
 600         ret
 601         #endif