2 # Copyright (c) 2005 Jakub Vana
3 # Copyright (c) 2005 Jakub Jermar
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions
10 # - Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 # - Redistributions in binary form must reproduce the above copyright
13 # notice, this list of conditions and the following disclaimer in the
14 # documentation and/or other materials provided with the distribution.
15 # - The name of the author may not be used to endorse or promote products
16 # derived from this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #include <arch/stack.h>
31 #include <arch/register.h>
32 #include <arch/mm/page.h>
35 #define FRS_TO_SAVE 30
36 #define STACK_ITEMS (19 + FRS_TO_SAVE*2)
37 #define STACK_FRAME_SIZE ALIGN_UP((STACK_ITEMS*STACK_ITEM_SIZE) + STACK_SCRATCH_AREA_SIZE, STACK_ALIGNMENT)
39 #if (STACK_ITEMS % 2 == 0)
40 # define STACK_FRAME_BIAS 8
42 # define STACK_FRAME_BIAS 16
45 /** Partitioning of bank 0 registers. */
50 #define R_KSTACK_BSP r22 /* keep in sync with before_thread_runs_arch() */
51 #define R_KSTACK r23 /* keep in sync with before_thread_runs_arch() */
53 /** Heavyweight interrupt handler
55 * This macro roughly follows steps from 1 to 19 described in
56 * Intel Itanium Architecture Software Developer's Manual, Chapter 3.4.2.
58 * HEAVYWEIGHT_HANDLER macro must cram into 16 bundles (48 instructions).
59 * This goal is achieved by using procedure calls after RSE becomes operational.
61 * Some steps are skipped (enabling and disabling interrupts).
63 * @param offs Offset from the beginning of IVT.
64 * @param handler Interrupt handler address.
66 .macro HEAVYWEIGHT_HANDLER offs, handler=universal_handler
69 movl R_HANDLER = \handler ;;
70 br heavyweight_handler
73 .global heavyweight_handler
75 /* 1. copy interrupt registers into bank 0 */
78 * Note that r24-r31 from bank 0 can be used only as long as PSR.ic = 0.
81 /* Set up FPU as in interrupted context. */
84 mov r26 = PSR_DFH_MASK
85 mov r27 = ~PSR_DFH_MASK ;;
99 /* 2. preserve predicate register into bank 0 */
102 /* 3. switch to kernel memory stack */
104 shr.u r31 = r12, VRN_SHIFT ;;
106 shr.u r30 = r30, PSR_CPL_SHIFT ;;
107 and r30 = PSR_CPL_MASK_SHIFTED, r30 ;;
110 * Set p3 to true if the interrupted context executed in kernel mode.
111 * Set p4 to false if the interrupted context didn't execute in kernel mode.
113 cmp.eq p3, p4 = r30, r0 ;;
114 cmp.eq p1, p2 = r30, r0 ;; /* remember IPSR setting in p1 and p2 */
117 * Set p3 to true if the stack register references kernel address space.
118 * Set p4 to false if the stack register doesn't reference kernel address space.
120 (p3) cmp.eq p3, p4 = VRN_KERNEL, r31 ;;
123 * Now, p4 is true iff the stack needs to be switched to kernel stack.
126 (p4) mov r12 = R_KSTACK ;;
128 add r31 = -STACK_FRAME_BIAS, r12 ;;
129 add r12 = -STACK_FRAME_SIZE, r12
131 /* 4. save registers in bank 0 into memory stack */
134 * If this is break_instruction handler,
135 * copy input parameters to stack.
137 mov R_TMP = 0x2c00 ;;
138 cmp.eq p6,p5 = R_OFFS, R_TMP ;;
141 * From now on, if this is break_instruction handler, p6 is true and p5 is false.
142 * Otherwise p6 is false and p5 is true.
143 * Note that p5 is a preserved predicate register and we make use of it.
146 (p6) st8 [r31] = r36, -8 ;; /* save in4 */
147 (p6) st8 [r31] = r35, -8 ;; /* save in3 */
148 (p6) st8 [r31] = r34, -8 ;; /* save in2 */
149 (p6) st8 [r31] = r33, -8 ;; /* save in1 */
150 (p6) st8 [r31] = r32, -8 ;; /* save in0 */
151 (p5) add r31 = -40, r31 ;;
153 st8 [r31] = r30, -8 ;; /* save old stack pointer */
155 st8 [r31] = r29, -8 ;; /* save predicate registers */
157 st8 [r31] = r24, -8 ;; /* save cr.iip */
158 st8 [r31] = r25, -8 ;; /* save cr.ipsr */
159 st8 [r31] = r26, -8 ;; /* save cr.iipa */
160 st8 [r31] = r27, -8 ;; /* save cr.isr */
161 st8 [r31] = r28, -8 ;; /* save cr.ifa */
163 /* 5. RSE switch from interrupted context */
169 st8 [r31] = r24, -8 ;; /* save ar.rsc */
170 st8 [r31] = r25, -8 ;; /* save ar.pfs */
171 st8 [r31] = r26, -8 /* save ar.ifs */
173 and r24 = ~(RSC_PL_MASK), r24 ;;
174 and r30 = ~(RSC_MODE_MASK), r24 ;;
175 mov ar.rsc = r30 ;; /* update RSE state */
178 mov r28 = ar.bspstore ;;
181 * Inspect BSPSTORE to figure out whether it is necessary to switch to kernel BSPSTORE.
183 (p1) shr.u r30 = r28, VRN_SHIFT ;;
184 (p1) cmp.eq p1, p2 = VRN_KERNEL, r30 ;;
187 * If BSPSTORE needs to be switched, p1 is false and p2 is true.
190 (p2) mov r30 = R_KSTACK_BSP ;;
191 (p2) mov ar.bspstore = r30 ;;
195 st8 [r31] = r27, -8 ;; /* save ar.rnat */
196 st8 [r31] = r30, -8 ;; /* save new value written to ar.bspstore */
197 st8 [r31] = r28, -8 ;; /* save ar.bspstore */
198 st8 [r31] = r29, -8 /* save ar.bsp */
200 mov ar.rsc = r24 /* restore RSE's setting + kernel privileges */
202 /* steps 6 - 15 are done by heavyweight_handler_inner() */
203 mov R_RET = b0 /* save b0 belonging to interrupted context */
204 br.call.sptk.many b0 = heavyweight_handler_inner
205 0: mov b0 = R_RET /* restore b0 belonging to the interrupted context */
207 /* 16. RSE switch to interrupted context */
208 cover /* allocate zerro size frame (step 1 (from Intel Docs)) */
210 add r31 = (STACK_SCRATCH_AREA_SIZE+(FRS_TO_SAVE*2*8)), r12 ;;
212 ld8 r30 = [r31], +8 ;; /* load ar.bsp */
213 ld8 r29 = [r31], +8 ;; /* load ar.bspstore */
214 ld8 r28 = [r31], +8 ;; /* load ar.bspstore_new */
215 sub r27 = r30 , r28 ;; /* calculate loadrs (step 2) */
220 or r24 = r30 , r27 ;;
221 mov ar.rsc = r24 ;; /* place RSE in enforced lazy mode */
223 loadrs /* (step 3) */
225 ld8 r27 = [r31], +8 ;; /* load ar.rnat */
226 ld8 r26 = [r31], +8 ;; /* load cr.ifs */
227 ld8 r25 = [r31], +8 ;; /* load ar.pfs */
228 ld8 r24 = [r31], +8 ;; /* load ar.rsc */
230 mov ar.bspstore = r29 ;; /* (step 4) */
231 mov ar.rnat = r27 /* (step 5) */
233 mov ar.pfs = r25 /* (step 6) */
236 mov ar.rsc = r24 /* (step 7) */
238 /* 17. restore interruption state from memory stack */
239 ld8 r28 = [r31], +8 ;; /* load cr.ifa */
240 ld8 r27 = [r31], +8 ;; /* load cr.isr */
241 ld8 r26 = [r31], +8 ;; /* load cr.iipa */
242 ld8 r25 = [r31], +8 ;; /* load cr.ipsr */
243 ld8 r24 = [r31], +8 ;; /* load cr.iip */
250 /* Set up FPU as in exception. */
252 mov r26 = PSR_DFH_MASK
253 mov r27 = ~PSR_DFH_MASK ;;
255 and r24 = r24, r26 ;;
259 /* 18. restore predicate registers from memory stack */
260 ld8 r29 = [r31], +8 ;; /* load predicate registers */
263 /* 19. return from interruption */
264 ld8 r12 = [r31] /* load stack pointer */
267 .global heavyweight_handler_inner
268 heavyweight_handler_inner:
270 * From this point, the rest of the interrupted context
271 * will be preserved in stacked registers and backing store.
273 alloc loc0 = ar.pfs, 0, 48, 2, 0 ;;
275 /* bank 0 is going to be shadowed, copy essential data from there */
276 mov loc1 = R_RET /* b0 belonging to interrupted context */
280 add out1 = STACK_SCRATCH_AREA_SIZE, r12
282 /* 6. switch to bank 1 and reenable PSR.ic */
287 /* 7. preserve branch and application registers */
304 /* 8. preserve general and floating-point registers */
312 (p5) mov loc24 = r8 /* only if not in break_instruction handler */
316 /* skip r12 (stack pointer) */
337 add r24 = 96 + STACK_SCRATCH_AREA_SIZE, r12
338 add r25 = 112 + STACK_SCRATCH_AREA_SIZE, r12
339 add r26 = 0 + STACK_SCRATCH_AREA_SIZE, r12
340 add r27 = 16 + STACK_SCRATCH_AREA_SIZE, r12
341 add r28 = 32 + STACK_SCRATCH_AREA_SIZE, r12
342 add r29 = 48 + STACK_SCRATCH_AREA_SIZE, r12
343 add r30 = 64 + STACK_SCRATCH_AREA_SIZE, r12
344 add r31 = 80 + STACK_SCRATCH_AREA_SIZE, r12 ;;
346 stf.spill [r26] = f2, 0x80
347 stf.spill [r27] = f3, 0x80
348 stf.spill [r28] = f4, 0x80
349 stf.spill [r29] = f5, 0x80
350 stf.spill [r30] = f6, 0x80
351 stf.spill [r31] = f7, 0x80 ;;
353 stf.spill [r24] = f8, 0x80
354 stf.spill [r25] = f9, 0x80
355 stf.spill [r26] = f10, 0x80
356 stf.spill [r27] = f11, 0x80
357 stf.spill [r28] = f12, 0x80
358 stf.spill [r29] = f13, 0x80
359 stf.spill [r30] = f14, 0x80
360 stf.spill [r31] = f15, 0x80 ;;
362 stf.spill [r24] = f16, 0x80
363 stf.spill [r25] = f17, 0x80
364 stf.spill [r26] = f18, 0x80
365 stf.spill [r27] = f19, 0x80
366 stf.spill [r28] = f20, 0x80
367 stf.spill [r29] = f21, 0x80
368 stf.spill [r30] = f22, 0x80
369 stf.spill [r31] = f23, 0x80 ;;
371 stf.spill [r24] = f24, 0x80
372 stf.spill [r25] = f25, 0x80
373 stf.spill [r26] = f26, 0x80
374 stf.spill [r27] = f27, 0x80
375 stf.spill [r28] = f28, 0x80
376 stf.spill [r29] = f29, 0x80
377 stf.spill [r30] = f30, 0x80
378 stf.spill [r31] = f31, 0x80 ;;
380 mov loc47 = ar.fpsr /* preserve floating point status register */
382 /* 9. skipped (will not enable interrupts) */
389 /* 10. call handler */
390 movl r1 = _hardcoded_load_address
393 br.call.sptk.many b0 = b1
395 /* 11. return from handler */
398 /* 12. skipped (will not disable interrupts) */
405 /* 13. restore general and floating-point registers */
406 add r24 = 96 + STACK_SCRATCH_AREA_SIZE, r12
407 add r25 = 112 + STACK_SCRATCH_AREA_SIZE, r12
408 add r26 = 0 + STACK_SCRATCH_AREA_SIZE, r12
409 add r27 = 16 + STACK_SCRATCH_AREA_SIZE, r12
410 add r28 = 32 + STACK_SCRATCH_AREA_SIZE, r12
411 add r29 = 48 + STACK_SCRATCH_AREA_SIZE, r12
412 add r30 = 64 + STACK_SCRATCH_AREA_SIZE, r12
413 add r31 = 80 + STACK_SCRATCH_AREA_SIZE, r12 ;;
415 ldf.fill f2 = [r26], 0x80
416 ldf.fill f3 = [r27], 0x80
417 ldf.fill f4 = [r28], 0x80
418 ldf.fill f5 = [r29], 0x80
419 ldf.fill f6 = [r30], 0x80
420 ldf.fill f7 = [r31], 0x80 ;;
422 ldf.fill f8 = [r24], 0x80
423 ldf.fill f9 = [r25], 0x80
424 ldf.fill f10 = [r26], 0x80
425 ldf.fill f11 = [r27], 0x80
426 ldf.fill f12 = [r28], 0x80
427 ldf.fill f13 = [r29], 0x80
428 ldf.fill f14 = [r30], 0x80
429 ldf.fill f15 = [r31], 0x80 ;;
431 ldf.fill f16 = [r24], 0x80
432 ldf.fill f17 = [r25], 0x80
433 ldf.fill f18 = [r26], 0x80
434 ldf.fill f19 = [r27], 0x80
435 ldf.fill f20 = [r28], 0x80
436 ldf.fill f21 = [r29], 0x80
437 ldf.fill f22 = [r30], 0x80
438 ldf.fill f23 = [r31], 0x80 ;;
440 ldf.fill f24 = [r24], 0x80
441 ldf.fill f25 = [r25], 0x80
442 ldf.fill f26 = [r26], 0x80
443 ldf.fill f27 = [r27], 0x80
444 ldf.fill f28 = [r28], 0x80
445 ldf.fill f29 = [r29], 0x80
446 ldf.fill f30 = [r30], 0x80
447 ldf.fill f31 = [r31], 0x80 ;;
456 (p5) mov r8 = loc24 /* only if not in break_instruction handler */
460 /* skip r12 (stack pointer) */
481 mov ar.fpsr = loc47 /* restore floating point status register */
483 /* 14. restore branch and application registers */
500 /* 15. disable PSR.ic and switch to bank 0 */
512 HEAVYWEIGHT_HANDLER 0x0000
513 HEAVYWEIGHT_HANDLER 0x0400
514 HEAVYWEIGHT_HANDLER 0x0800
515 HEAVYWEIGHT_HANDLER 0x0c00 alternate_instruction_tlb_fault
516 HEAVYWEIGHT_HANDLER 0x1000 alternate_data_tlb_fault
517 HEAVYWEIGHT_HANDLER 0x1400 data_nested_tlb_fault
518 HEAVYWEIGHT_HANDLER 0x1800
519 HEAVYWEIGHT_HANDLER 0x1c00
520 HEAVYWEIGHT_HANDLER 0x2000 data_dirty_bit_fault
521 HEAVYWEIGHT_HANDLER 0x2400 instruction_access_bit_fault
522 HEAVYWEIGHT_HANDLER 0x2800 data_access_bit_fault
523 HEAVYWEIGHT_HANDLER 0x2c00 break_instruction
524 HEAVYWEIGHT_HANDLER 0x3000 external_interrupt /* For external interrupt, heavyweight handler is used. */
525 HEAVYWEIGHT_HANDLER 0x3400
526 HEAVYWEIGHT_HANDLER 0x3800
527 HEAVYWEIGHT_HANDLER 0x3c00
528 HEAVYWEIGHT_HANDLER 0x4000
529 HEAVYWEIGHT_HANDLER 0x4400
530 HEAVYWEIGHT_HANDLER 0x4800
531 HEAVYWEIGHT_HANDLER 0x4c00
533 HEAVYWEIGHT_HANDLER 0x5000 page_not_present
534 HEAVYWEIGHT_HANDLER 0x5100
535 HEAVYWEIGHT_HANDLER 0x5200
536 HEAVYWEIGHT_HANDLER 0x5300
537 HEAVYWEIGHT_HANDLER 0x5400 general_exception
538 HEAVYWEIGHT_HANDLER 0x5500 disabled_fp_register
539 HEAVYWEIGHT_HANDLER 0x5600
540 HEAVYWEIGHT_HANDLER 0x5700
541 HEAVYWEIGHT_HANDLER 0x5800
542 HEAVYWEIGHT_HANDLER 0x5900
543 HEAVYWEIGHT_HANDLER 0x5a00
544 HEAVYWEIGHT_HANDLER 0x5b00
545 HEAVYWEIGHT_HANDLER 0x5c00
546 HEAVYWEIGHT_HANDLER 0x5d00
547 HEAVYWEIGHT_HANDLER 0x5e00
548 HEAVYWEIGHT_HANDLER 0x5f00
550 HEAVYWEIGHT_HANDLER 0x6000
551 HEAVYWEIGHT_HANDLER 0x6100
552 HEAVYWEIGHT_HANDLER 0x6200
553 HEAVYWEIGHT_HANDLER 0x6300
554 HEAVYWEIGHT_HANDLER 0x6400
555 HEAVYWEIGHT_HANDLER 0x6500
556 HEAVYWEIGHT_HANDLER 0x6600
557 HEAVYWEIGHT_HANDLER 0x6700
558 HEAVYWEIGHT_HANDLER 0x6800
559 HEAVYWEIGHT_HANDLER 0x6900
560 HEAVYWEIGHT_HANDLER 0x6a00
561 HEAVYWEIGHT_HANDLER 0x6b00
562 HEAVYWEIGHT_HANDLER 0x6c00
563 HEAVYWEIGHT_HANDLER 0x6d00
564 HEAVYWEIGHT_HANDLER 0x6e00
565 HEAVYWEIGHT_HANDLER 0x6f00
567 HEAVYWEIGHT_HANDLER 0x7000
568 HEAVYWEIGHT_HANDLER 0x7100
569 HEAVYWEIGHT_HANDLER 0x7200
570 HEAVYWEIGHT_HANDLER 0x7300
571 HEAVYWEIGHT_HANDLER 0x7400
572 HEAVYWEIGHT_HANDLER 0x7500
573 HEAVYWEIGHT_HANDLER 0x7600
574 HEAVYWEIGHT_HANDLER 0x7700
575 HEAVYWEIGHT_HANDLER 0x7800
576 HEAVYWEIGHT_HANDLER 0x7900
577 HEAVYWEIGHT_HANDLER 0x7a00
578 HEAVYWEIGHT_HANDLER 0x7b00
579 HEAVYWEIGHT_HANDLER 0x7c00
580 HEAVYWEIGHT_HANDLER 0x7d00
581 HEAVYWEIGHT_HANDLER 0x7e00
582 HEAVYWEIGHT_HANDLER 0x7f00