[tomato.git] / release / src-rt-6.x.4708 / linux / linux-2.6.36 / arch / powerpc / kernel / entry_64.S
| blob | 11d0679532e94e38831c71d3f4e1f468c2297387 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <asm/unistd.h>
23 #include <asm/processor.h>
24 #include <asm/page.h>
25 #include <asm/mmu.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputable.h>
30 #include <asm/firmware.h>
31 #include <asm/bug.h>
32 #include <asm/ptrace.h>
33 #include <asm/irqflags.h>
34 #include <asm/ftrace.h>
36 /*
37 * System calls.
38 */
39 .section ".toc","aw"
40 .SYS_CALL_TABLE:
41 .tc .sys_call_table[TC],.sys_call_table
43 /* This value is used to mark exception frames on the stack. */
44 exception_marker:
45 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
47 .section ".text"
48 .align 7
50 #undef SHOW_SYSCALLS
52 .globl system_call_common
53 system_call_common:
54 andi. r10,r12,MSR_PR
55 mr r10,r1
56 addi r1,r1,-INT_FRAME_SIZE
57 beq- 1f
58 ld r1,PACAKSAVE(r13)
59 1: std r10,0(r1)
60 std r11,_NIP(r1)
61 std r12,_MSR(r1)
62 std r0,GPR0(r1)
63 std r10,GPR1(r1)
64 ACCOUNT_CPU_USER_ENTRY(r10, r11)
65 /*
66 * This "crclr so" clears CR0.SO, which is the error indication on
67 * return from this system call. There must be no cmp instruction
68 * between it and the "mfcr r9" below, otherwise if XER.SO is set,
69 * CR0.SO will get set, causing all system calls to appear to fail.
70 */
71 crclr so
72 std r2,GPR2(r1)
73 std r3,GPR3(r1)
74 std r4,GPR4(r1)
75 std r5,GPR5(r1)
76 std r6,GPR6(r1)
77 std r7,GPR7(r1)
78 std r8,GPR8(r1)
79 li r11,0
80 std r11,GPR9(r1)
81 std r11,GPR10(r1)
82 std r11,GPR11(r1)
83 std r11,GPR12(r1)
84 std r9,GPR13(r1)
85 mfcr r9
86 mflr r10
87 li r11,0xc01
88 std r9,_CCR(r1)
89 std r10,_LINK(r1)
90 std r11,_TRAP(r1)
91 mfxer r9
92 mfctr r10
93 std r9,_XER(r1)
94 std r10,_CTR(r1)
95 std r3,ORIG_GPR3(r1)
96 ld r2,PACATOC(r13)
97 addi r9,r1,STACK_FRAME_OVERHEAD
98 ld r11,exception_marker@toc(r2)
99 std r11,-16(r9) /* "regshere" marker */
100 #ifdef CONFIG_TRACE_IRQFLAGS
101 bl .trace_hardirqs_on
102 REST_GPR(0,r1)
103 REST_4GPRS(3,r1)
104 REST_2GPRS(7,r1)
105 addi r9,r1,STACK_FRAME_OVERHEAD
106 ld r12,_MSR(r1)
107 #endif /* CONFIG_TRACE_IRQFLAGS */
108 li r10,1
109 stb r10,PACASOFTIRQEN(r13)
110 stb r10,PACAHARDIRQEN(r13)
111 std r10,SOFTE(r1)
112 #ifdef CONFIG_PPC_ISERIES
113 BEGIN_FW_FTR_SECTION
114 /* Hack for handling interrupts when soft-enabling on iSeries */
115 cmpdi cr1,r0,0x5555 /* syscall 0x5555 */
116 andi. r10,r12,MSR_PR /* from kernel */
117 crand 4*cr0+eq,4*cr1+eq,4*cr0+eq
118 bne 2f
119 b hardware_interrupt_entry
120 2:
121 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
122 #endif /* CONFIG_PPC_ISERIES */
124 /* Hard enable interrupts */
125 #ifdef CONFIG_PPC_BOOK3E
126 wrteei 1
127 #else
128 mfmsr r11
129 ori r11,r11,MSR_EE
130 mtmsrd r11,1
131 #endif /* CONFIG_PPC_BOOK3E */
133 #ifdef SHOW_SYSCALLS
134 bl .do_show_syscall
135 REST_GPR(0,r1)
136 REST_4GPRS(3,r1)
137 REST_2GPRS(7,r1)
138 addi r9,r1,STACK_FRAME_OVERHEAD
139 #endif
140 clrrdi r11,r1,THREAD_SHIFT
141 ld r10,TI_FLAGS(r11)
142 andi. r11,r10,_TIF_SYSCALL_T_OR_A
143 bne- syscall_dotrace
144 syscall_dotrace_cont:
145 cmpldi 0,r0,NR_syscalls
146 bge- syscall_enosys
148 system_call: /* label this so stack traces look sane */
149 /*
150 * Need to vector to 32 Bit or default sys_call_table here,
151 * based on caller's run-mode / personality.
152 */
153 ld r11,.SYS_CALL_TABLE@toc(2)
154 andi. r10,r10,_TIF_32BIT
155 beq 15f
156 addi r11,r11,8 /* use 32-bit syscall entries */
157 clrldi r3,r3,32
158 clrldi r4,r4,32
159 clrldi r5,r5,32
160 clrldi r6,r6,32
161 clrldi r7,r7,32
162 clrldi r8,r8,32
163 15:
164 slwi r0,r0,4
165 ldx r10,r11,r0 /* Fetch system call handler [ptr] */
166 mtctr r10
167 bctrl /* Call handler */
169 syscall_exit:
170 std r3,RESULT(r1)
171 #ifdef SHOW_SYSCALLS
172 bl .do_show_syscall_exit
173 ld r3,RESULT(r1)
174 #endif
175 clrrdi r12,r1,THREAD_SHIFT
177 ld r8,_MSR(r1)
178 #ifdef CONFIG_PPC_BOOK3S
179 /* No MSR:RI on BookE */
180 andi. r10,r8,MSR_RI
181 beq- unrecov_restore
182 #endif
184 /* Disable interrupts so current_thread_info()->flags can't change,
185 * and so that we don't get interrupted after loading SRR0/1.
186 */
187 #ifdef CONFIG_PPC_BOOK3E
188 wrteei 0
189 #else
190 mfmsr r10
191 rldicl r10,r10,48,1
192 rotldi r10,r10,16
193 mtmsrd r10,1
194 #endif /* CONFIG_PPC_BOOK3E */
196 ld r9,TI_FLAGS(r12)
197 li r11,-_LAST_ERRNO
198 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
199 bne- syscall_exit_work
200 cmpld r3,r11
201 ld r5,_CCR(r1)
202 bge- syscall_error
203 syscall_error_cont:
204 ld r7,_NIP(r1)
205 stdcx. r0,0,r1 /* to clear the reservation */
206 andi. r6,r8,MSR_PR
207 ld r4,_LINK(r1)
208 /*
209 * Clear RI before restoring r13. If we are returning to
210 * userspace and we take an exception after restoring r13,
211 * we end up corrupting the userspace r13 value.
212 */
213 #ifdef CONFIG_PPC_BOOK3S
214 /* No MSR:RI on BookE */
215 li r12,MSR_RI
216 andc r11,r10,r12
217 mtmsrd r11,1 /* clear MSR.RI */
218 #endif /* CONFIG_PPC_BOOK3S */
220 beq- 1f
221 ACCOUNT_CPU_USER_EXIT(r11, r12)
222 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
223 1: ld r2,GPR2(r1)
224 ld r1,GPR1(r1)
225 mtlr r4
226 mtcr r5
227 mtspr SPRN_SRR0,r7
228 mtspr SPRN_SRR1,r8
229 RFI
230 b . /* prevent speculative execution */
232 syscall_error:
233 oris r5,r5,0x1000 /* Set SO bit in CR */
234 neg r3,r3
235 std r5,_CCR(r1)
236 b syscall_error_cont
238 /* Traced system call support */
239 syscall_dotrace:
240 bl .save_nvgprs
241 addi r3,r1,STACK_FRAME_OVERHEAD
242 bl .do_syscall_trace_enter
243 /*
244 * Restore argument registers possibly just changed.
245 * We use the return value of do_syscall_trace_enter
246 * for the call number to look up in the table (r0).
247 */
248 mr r0,r3
249 ld r3,GPR3(r1)
250 ld r4,GPR4(r1)
251 ld r5,GPR5(r1)
252 ld r6,GPR6(r1)
253 ld r7,GPR7(r1)
254 ld r8,GPR8(r1)
255 addi r9,r1,STACK_FRAME_OVERHEAD
256 clrrdi r10,r1,THREAD_SHIFT
257 ld r10,TI_FLAGS(r10)
258 b syscall_dotrace_cont
260 syscall_enosys:
261 li r3,-ENOSYS
262 b syscall_exit
264 syscall_exit_work:
265 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
266 If TIF_NOERROR is set, just save r3 as it is. */
268 andi. r0,r9,_TIF_RESTOREALL
269 beq+ 0f
270 REST_NVGPRS(r1)
271 b 2f
272 0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
273 blt+ 1f
274 andi. r0,r9,_TIF_NOERROR
275 bne- 1f
276 ld r5,_CCR(r1)
277 neg r3,r3
278 oris r5,r5,0x1000 /* Set SO bit in CR */
279 std r5,_CCR(r1)
280 1: std r3,GPR3(r1)
281 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
282 beq 4f
284 /* Clear per-syscall TIF flags if any are set. */
286 li r11,_TIF_PERSYSCALL_MASK
287 addi r12,r12,TI_FLAGS
288 3: ldarx r10,0,r12
289 andc r10,r10,r11
290 stdcx. r10,0,r12
291 bne- 3b
292 subi r12,r12,TI_FLAGS
294 4: /* Anything else left to do? */
295 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
296 beq .ret_from_except_lite
298 /* Re-enable interrupts */
299 #ifdef CONFIG_PPC_BOOK3E
300 wrteei 1
301 #else
302 mfmsr r10
303 ori r10,r10,MSR_EE
304 mtmsrd r10,1
305 #endif /* CONFIG_PPC_BOOK3E */
307 bl .save_nvgprs
308 addi r3,r1,STACK_FRAME_OVERHEAD
309 bl .do_syscall_trace_leave
310 b .ret_from_except
312 /* Save non-volatile GPRs, if not already saved. */
313 _GLOBAL(save_nvgprs)
314 ld r11,_TRAP(r1)
315 andi. r0,r11,1
316 beqlr-
317 SAVE_NVGPRS(r1)
318 clrrdi r0,r11,1
319 std r0,_TRAP(r1)
320 blr
323 /*
324 * The sigsuspend and rt_sigsuspend system calls can call do_signal
325 * and thus put the process into the stopped state where we might
326 * want to examine its user state with ptrace. Therefore we need
327 * to save all the nonvolatile registers (r14 - r31) before calling
328 * the C code. Similarly, fork, vfork and clone need the full
329 * register state on the stack so that it can be copied to the child.
330 */
332 _GLOBAL(ppc_fork)
333 bl .save_nvgprs
334 bl .sys_fork
335 b syscall_exit
337 _GLOBAL(ppc_vfork)
338 bl .save_nvgprs
339 bl .sys_vfork
340 b syscall_exit
342 _GLOBAL(ppc_clone)
343 bl .save_nvgprs
344 bl .sys_clone
345 b syscall_exit
347 _GLOBAL(ppc32_swapcontext)
348 bl .save_nvgprs
349 bl .compat_sys_swapcontext
350 b syscall_exit
352 _GLOBAL(ppc64_swapcontext)
353 bl .save_nvgprs
354 bl .sys_swapcontext
355 b syscall_exit
357 _GLOBAL(ret_from_fork)
358 bl .schedule_tail
359 REST_NVGPRS(r1)
360 li r3,0
361 b syscall_exit
363 /*
364 * This routine switches between two different tasks. The process
365 * state of one is saved on its kernel stack. Then the state
366 * of the other is restored from its kernel stack. The memory
367 * management hardware is updated to the second process's state.
368 * Finally, we can return to the second process, via ret_from_except.
369 * On entry, r3 points to the THREAD for the current task, r4
370 * points to the THREAD for the new task.
371 *
372 * Note: there are two ways to get to the "going out" portion
373 * of this code; either by coming in via the entry (_switch)
374 * or via "fork" which must set up an environment equivalent
375 * to the "_switch" path. If you change this you'll have to change
376 * the fork code also.
377 *
378 * The code which creates the new task context is in 'copy_thread'
379 * in arch/powerpc/kernel/process.c
380 */
381 .align 7
382 _GLOBAL(_switch)
383 mflr r0
384 std r0,16(r1)
385 stdu r1,-SWITCH_FRAME_SIZE(r1)
386 /* r3-r13 are caller saved -- Cort */
387 SAVE_8GPRS(14, r1)
388 SAVE_10GPRS(22, r1)
389 mflr r20 /* Return to switch caller */
390 mfmsr r22
391 li r0, MSR_FP
392 #ifdef CONFIG_VSX
393 BEGIN_FTR_SECTION
394 oris r0,r0,MSR_VSX@h /* Disable VSX */
395 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
396 #endif /* CONFIG_VSX */
397 #ifdef CONFIG_ALTIVEC
398 BEGIN_FTR_SECTION
399 oris r0,r0,MSR_VEC@h /* Disable altivec */
400 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
401 std r24,THREAD_VRSAVE(r3)
402 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
403 #endif /* CONFIG_ALTIVEC */
404 and. r0,r0,r22
405 beq+ 1f
406 andc r22,r22,r0
407 MTMSRD(r22)
408 isync
409 1: std r20,_NIP(r1)
410 mfcr r23
411 std r23,_CCR(r1)
412 std r1,KSP(r3) /* Set old stack pointer */
414 #ifdef CONFIG_SMP
415 /* We need a sync somewhere here to make sure that if the
416 * previous task gets rescheduled on another CPU, it sees all
417 * stores it has performed on this one.
418 */
419 sync
420 #endif /* CONFIG_SMP */
422 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
423 std r6,PACACURRENT(r13) /* Set new 'current' */
425 ld r8,KSP(r4) /* new stack pointer */
426 #ifdef CONFIG_PPC_BOOK3S
427 BEGIN_FTR_SECTION
428 BEGIN_FTR_SECTION_NESTED(95)
429 clrrdi r6,r8,28 /* get its ESID */
430 clrrdi r9,r1,28 /* get current sp ESID */
431 FTR_SECTION_ELSE_NESTED(95)
432 clrrdi r6,r8,40 /* get its 1T ESID */
433 clrrdi r9,r1,40 /* get current sp 1T ESID */
434 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_1T_SEGMENT, 95)
435 FTR_SECTION_ELSE
436 b 2f
437 ALT_FTR_SECTION_END_IFSET(CPU_FTR_SLB)
438 clrldi. r0,r6,2 /* is new ESID c00000000? */
439 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
440 cror eq,4*cr1+eq,eq
441 beq 2f /* if yes, don't slbie it */
443 /* Bolt in the new stack SLB entry */
444 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
445 oris r0,r6,(SLB_ESID_V)@h
446 ori r0,r0,(SLB_NUM_BOLTED-1)@l
447 BEGIN_FTR_SECTION
448 li r9,MMU_SEGSIZE_1T /* insert B field */
449 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
450 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
451 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
453 /* Update the last bolted SLB. No write barriers are needed
454 * here, provided we only update the current CPU's SLB shadow
455 * buffer.
456 */
457 ld r9,PACA_SLBSHADOWPTR(r13)
458 li r12,0
459 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
460 std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
461 std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
463 /* No need to check for CPU_FTR_NO_SLBIE_B here, since when
464 * we have 1TB segments, the only CPUs known to have the errata
465 * only support less than 1TB of system memory and we'll never
466 * actually hit this code path.
467 */
469 slbie r6
470 slbie r6
471 slbmte r7,r0
472 isync
473 2:
474 #endif /* !CONFIG_PPC_BOOK3S */
476 clrrdi r7,r8,THREAD_SHIFT /* base of new stack */
477 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
478 because we don't need to leave the 288-byte ABI gap at the
479 top of the kernel stack. */
480 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
482 mr r1,r8 /* start using new stack pointer */
483 std r7,PACAKSAVE(r13)
485 ld r6,_CCR(r1)
486 mtcrf 0xFF,r6
488 #ifdef CONFIG_ALTIVEC
489 BEGIN_FTR_SECTION
490 ld r0,THREAD_VRSAVE(r4)
491 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
492 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
493 #endif /* CONFIG_ALTIVEC */
495 /* r3-r13 are destroyed -- Cort */
496 REST_8GPRS(14, r1)
497 REST_10GPRS(22, r1)
499 /* convert old thread to its task_struct for return value */
500 addi r3,r3,-THREAD
501 ld r7,_NIP(r1) /* Return to _switch caller in new task */
502 mtlr r7
503 addi r1,r1,SWITCH_FRAME_SIZE
504 blr
506 .align 7
507 _GLOBAL(ret_from_except)
508 ld r11,_TRAP(r1)
509 andi. r0,r11,1
510 bne .ret_from_except_lite
511 REST_NVGPRS(r1)
513 _GLOBAL(ret_from_except_lite)
514 /*
515 * Disable interrupts so that current_thread_info()->flags
516 * can't change between when we test it and when we return
517 * from the interrupt.
518 */
519 #ifdef CONFIG_PPC_BOOK3E
520 wrteei 0
521 #else
522 mfmsr r10 /* Get current interrupt state */
523 rldicl r9,r10,48,1 /* clear MSR_EE */
524 rotldi r9,r9,16
525 mtmsrd r9,1 /* Update machine state */
526 #endif /* CONFIG_PPC_BOOK3E */
528 #ifdef CONFIG_PREEMPT
529 clrrdi r9,r1,THREAD_SHIFT /* current_thread_info() */
530 li r0,_TIF_NEED_RESCHED /* bits to check */
531 ld r3,_MSR(r1)
532 ld r4,TI_FLAGS(r9)
533 /* Move MSR_PR bit in r3 to _TIF_SIGPENDING position in r0 */
534 rlwimi r0,r3,32+TIF_SIGPENDING-MSR_PR_LG,_TIF_SIGPENDING
535 and. r0,r4,r0 /* check NEED_RESCHED and maybe SIGPENDING */
536 bne do_work
538 #else /* !CONFIG_PREEMPT */
539 ld r3,_MSR(r1) /* Returning to user mode? */
540 andi. r3,r3,MSR_PR
541 beq restore /* if not, just restore regs and return */
543 /* Check current_thread_info()->flags */
544 clrrdi r9,r1,THREAD_SHIFT
545 ld r4,TI_FLAGS(r9)
546 andi. r0,r4,_TIF_USER_WORK_MASK
547 bne do_work
548 #endif
550 restore:
551 BEGIN_FW_FTR_SECTION
552 ld r5,SOFTE(r1)
553 FW_FTR_SECTION_ELSE
554 b .Liseries_check_pending_irqs
555 ALT_FW_FTR_SECTION_END_IFCLR(FW_FEATURE_ISERIES)
556 2:
557 TRACE_AND_RESTORE_IRQ(r5);
559 /* extract EE bit and use it to restore paca->hard_enabled */
560 ld r3,_MSR(r1)
561 rldicl r4,r3,49,63 /* r0 = (r3 >> 15) & 1 */
562 stb r4,PACAHARDIRQEN(r13)
564 #ifdef CONFIG_PPC_BOOK3E
565 b .exception_return_book3e
566 #else
567 ld r4,_CTR(r1)
568 ld r0,_LINK(r1)
569 mtctr r4
570 mtlr r0
571 ld r4,_XER(r1)
572 mtspr SPRN_XER,r4
574 REST_8GPRS(5, r1)
576 andi. r0,r3,MSR_RI
577 beq- unrecov_restore
579 stdcx. r0,0,r1 /* to clear the reservation */
581 /*
582 * Clear RI before restoring r13. If we are returning to
583 * userspace and we take an exception after restoring r13,
584 * we end up corrupting the userspace r13 value.
585 */
586 mfmsr r4
587 andc r4,r4,r0 /* r0 contains MSR_RI here */
588 mtmsrd r4,1
590 /*
591 * r13 is our per cpu area, only restore it if we are returning to
592 * userspace
593 */
594 andi. r0,r3,MSR_PR
595 beq 1f
596 ACCOUNT_CPU_USER_EXIT(r2, r4)
597 REST_GPR(13, r1)
598 1:
599 mtspr SPRN_SRR1,r3
601 ld r2,_CCR(r1)
602 mtcrf 0xFF,r2
603 ld r2,_NIP(r1)
604 mtspr SPRN_SRR0,r2
606 ld r0,GPR0(r1)
607 ld r2,GPR2(r1)
608 ld r3,GPR3(r1)
609 ld r4,GPR4(r1)
610 ld r1,GPR1(r1)
612 rfid
613 b . /* prevent speculative execution */
615 #endif /* CONFIG_PPC_BOOK3E */
617 .Liseries_check_pending_irqs:
618 #ifdef CONFIG_PPC_ISERIES
619 ld r5,SOFTE(r1)
620 cmpdi 0,r5,0
621 beq 2b
622 /* Check for pending interrupts (iSeries) */
623 ld r3,PACALPPACAPTR(r13)
624 ld r3,LPPACAANYINT(r3)
625 cmpdi r3,0
626 beq+ 2b /* skip do_IRQ if no interrupts */
628 li r3,0
629 stb r3,PACASOFTIRQEN(r13) /* ensure we are soft-disabled */
630 #ifdef CONFIG_TRACE_IRQFLAGS
631 bl .trace_hardirqs_off
632 mfmsr r10
633 #endif
634 ori r10,r10,MSR_EE
635 mtmsrd r10 /* hard-enable again */
636 addi r3,r1,STACK_FRAME_OVERHEAD
637 bl .do_IRQ
638 b .ret_from_except_lite /* loop back and handle more */
639 #endif
641 do_work:
642 #ifdef CONFIG_PREEMPT
643 andi. r0,r3,MSR_PR /* Returning to user mode? */
644 bne user_work
645 /* Check that preempt_count() == 0 and interrupts are enabled */
646 lwz r8,TI_PREEMPT(r9)
647 cmpwi cr1,r8,0
648 ld r0,SOFTE(r1)
649 cmpdi r0,0
650 crandc eq,cr1*4+eq,eq
651 bne restore
653 /* Here we are preempting the current task.
654 *
655 * Ensure interrupts are soft-disabled. We also properly mark
656 * the PACA to reflect the fact that they are hard-disabled
657 * and trace the change
658 */
659 li r0,0
660 stb r0,PACASOFTIRQEN(r13)
661 stb r0,PACAHARDIRQEN(r13)
662 TRACE_DISABLE_INTS
664 /* Call the scheduler with soft IRQs off */
665 1: bl .preempt_schedule_irq
667 /* Hard-disable interrupts again (and update PACA) */
668 #ifdef CONFIG_PPC_BOOK3E
669 wrteei 0
670 #else
671 mfmsr r10
672 rldicl r10,r10,48,1
673 rotldi r10,r10,16
674 mtmsrd r10,1
675 #endif /* CONFIG_PPC_BOOK3E */
676 li r0,0
677 stb r0,PACAHARDIRQEN(r13)
679 /* Re-test flags and eventually loop */
680 clrrdi r9,r1,THREAD_SHIFT
681 ld r4,TI_FLAGS(r9)
682 andi. r0,r4,_TIF_NEED_RESCHED
683 bne 1b
684 b restore
686 user_work:
687 #endif /* CONFIG_PREEMPT */
689 /* Enable interrupts */
690 #ifdef CONFIG_PPC_BOOK3E
691 wrteei 1
692 #else
693 ori r10,r10,MSR_EE
694 mtmsrd r10,1
695 #endif /* CONFIG_PPC_BOOK3E */
697 andi. r0,r4,_TIF_NEED_RESCHED
698 beq 1f
699 bl .schedule
700 b .ret_from_except_lite
702 1: bl .save_nvgprs
703 addi r3,r1,STACK_FRAME_OVERHEAD
704 bl .do_signal
705 b .ret_from_except
707 unrecov_restore:
708 addi r3,r1,STACK_FRAME_OVERHEAD
709 bl .unrecoverable_exception
710 b unrecov_restore
712 #ifdef CONFIG_PPC_RTAS
713 /*
714 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
715 * called with the MMU off.
716 *
717 * In addition, we need to be in 32b mode, at least for now.
718 *
719 * Note: r3 is an input parameter to rtas, so don't trash it...
720 */
721 _GLOBAL(enter_rtas)
722 mflr r0
723 std r0,16(r1)
724 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
726 /* Because RTAS is running in 32b mode, it clobbers the high order half
727 * of all registers that it saves. We therefore save those registers
728 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
729 */
730 SAVE_GPR(2, r1) /* Save the TOC */
731 SAVE_GPR(13, r1) /* Save paca */
732 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
733 SAVE_10GPRS(22, r1) /* ditto */
735 mfcr r4
736 std r4,_CCR(r1)
737 mfctr r5
738 std r5,_CTR(r1)
739 mfspr r6,SPRN_XER
740 std r6,_XER(r1)
741 mfdar r7
742 std r7,_DAR(r1)
743 mfdsisr r8
744 std r8,_DSISR(r1)
746 li r0,0
747 mtcr r0
749 #ifdef CONFIG_BUG
750 /* There is no way it is acceptable to get here with interrupts enabled,
751 * check it with the asm equivalent of WARN_ON
752 */
753 lbz r0,PACASOFTIRQEN(r13)
754 1: tdnei r0,0
755 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
756 #endif
758 /* Hard-disable interrupts */
759 mfmsr r6
760 rldicl r7,r6,48,1
761 rotldi r7,r7,16
762 mtmsrd r7,1
764 /* Unfortunately, the stack pointer and the MSR are also clobbered,
765 * so they are saved in the PACA which allows us to restore
766 * our original state after RTAS returns.
767 */
768 std r1,PACAR1(r13)
769 std r6,PACASAVEDMSR(r13)
771 /* Setup our real return addr */
772 LOAD_REG_ADDR(r4,.rtas_return_loc)
773 clrldi r4,r4,2 /* convert to realmode address */
774 mtlr r4
776 li r0,0
777 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
778 andc r0,r6,r0
780 li r9,1
781 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
782 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI
783 andc r6,r0,r9
784 sync /* disable interrupts so SRR0/1 */
785 mtmsrd r0 /* don't get trashed */
787 LOAD_REG_ADDR(r4, rtas)
788 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
789 ld r4,RTASBASE(r4) /* get the rtas->base value */
791 mtspr SPRN_SRR0,r5
792 mtspr SPRN_SRR1,r6
793 rfid
794 b . /* prevent speculative execution */
796 _STATIC(rtas_return_loc)
797 /* relocation is off at this point */
798 mfspr r4,SPRN_SPRG_PACA /* Get PACA */
799 clrldi r4,r4,2 /* convert to realmode address */
801 bcl 20,31,$+4
802 0: mflr r3
803 ld r3,(1f-0b)(r3) /* get &.rtas_restore_regs */
805 mfmsr r6
806 li r0,MSR_RI
807 andc r6,r6,r0
808 sync
809 mtmsrd r6
811 ld r1,PACAR1(r4) /* Restore our SP */
812 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
814 mtspr SPRN_SRR0,r3
815 mtspr SPRN_SRR1,r4
816 rfid
817 b . /* prevent speculative execution */
819 .align 3
820 1: .llong .rtas_restore_regs
822 _STATIC(rtas_restore_regs)
823 /* relocation is on at this point */
824 REST_GPR(2, r1) /* Restore the TOC */
825 REST_GPR(13, r1) /* Restore paca */
826 REST_8GPRS(14, r1) /* Restore the non-volatiles */
827 REST_10GPRS(22, r1) /* ditto */
829 mfspr r13,SPRN_SPRG_PACA
831 ld r4,_CCR(r1)
832 mtcr r4
833 ld r5,_CTR(r1)
834 mtctr r5
835 ld r6,_XER(r1)
836 mtspr SPRN_XER,r6
837 ld r7,_DAR(r1)
838 mtdar r7
839 ld r8,_DSISR(r1)
840 mtdsisr r8
842 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
843 ld r0,16(r1) /* get return address */
845 mtlr r0
846 blr /* return to caller */
848 #endif /* CONFIG_PPC_RTAS */
850 _GLOBAL(enter_prom)
851 mflr r0
852 std r0,16(r1)
853 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
855 /* Because PROM is running in 32b mode, it clobbers the high order half
856 * of all registers that it saves. We therefore save those registers
857 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
858 */
859 SAVE_GPR(2, r1)
860 SAVE_GPR(13, r1)
861 SAVE_8GPRS(14, r1)
862 SAVE_10GPRS(22, r1)
863 mfcr r10
864 mfmsr r11
865 std r10,_CCR(r1)
866 std r11,_MSR(r1)
868 /* Get the PROM entrypoint */
869 mtlr r4
871 /* Switch MSR to 32 bits mode
872 */
873 #ifdef CONFIG_PPC_BOOK3E
874 rlwinm r11,r11,0,1,31
875 mtmsr r11
876 #else /* CONFIG_PPC_BOOK3E */
877 mfmsr r11
878 li r12,1
879 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
880 andc r11,r11,r12
881 li r12,1
882 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
883 andc r11,r11,r12
884 mtmsrd r11
885 #endif /* CONFIG_PPC_BOOK3E */
886 isync
888 /* Enter PROM here... */
889 blrl
891 /* Just make sure that r1 top 32 bits didn't get
892 * corrupt by OF
893 */
894 rldicl r1,r1,0,32
896 /* Restore the MSR (back to 64 bits) */
897 ld r0,_MSR(r1)
898 MTMSRD(r0)
899 isync
901 /* Restore other registers */
902 REST_GPR(2, r1)
903 REST_GPR(13, r1)
904 REST_8GPRS(14, r1)
905 REST_10GPRS(22, r1)
906 ld r4,_CCR(r1)
907 mtcr r4
909 addi r1,r1,PROM_FRAME_SIZE
910 ld r0,16(r1)
911 mtlr r0
912 blr
914 #ifdef CONFIG_FUNCTION_TRACER
915 #ifdef CONFIG_DYNAMIC_FTRACE
916 _GLOBAL(mcount)
917 _GLOBAL(_mcount)
918 blr
920 _GLOBAL(ftrace_caller)
921 /* Taken from output of objdump from lib64/glibc */
922 mflr r3
923 ld r11, 0(r1)
924 stdu r1, -112(r1)
925 std r3, 128(r1)
926 ld r4, 16(r11)
927 subi r3, r3, MCOUNT_INSN_SIZE
928 .globl ftrace_call
929 ftrace_call:
930 bl ftrace_stub
931 nop
932 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
933 .globl ftrace_graph_call
934 ftrace_graph_call:
935 b ftrace_graph_stub
936 _GLOBAL(ftrace_graph_stub)
937 #endif
938 ld r0, 128(r1)
939 mtlr r0
940 addi r1, r1, 112
941 _GLOBAL(ftrace_stub)
942 blr
943 #else
944 _GLOBAL(mcount)
945 blr
947 _GLOBAL(_mcount)
948 /* Taken from output of objdump from lib64/glibc */
949 mflr r3
950 ld r11, 0(r1)
951 stdu r1, -112(r1)
952 std r3, 128(r1)
953 ld r4, 16(r11)
955 subi r3, r3, MCOUNT_INSN_SIZE
956 LOAD_REG_ADDR(r5,ftrace_trace_function)
957 ld r5,0(r5)
958 ld r5,0(r5)
959 mtctr r5
960 bctrl
961 nop
964 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
965 b ftrace_graph_caller
966 #endif
967 ld r0, 128(r1)
968 mtlr r0
969 addi r1, r1, 112
970 _GLOBAL(ftrace_stub)
971 blr
973 #endif /* CONFIG_DYNAMIC_FTRACE */
975 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
976 _GLOBAL(ftrace_graph_caller)
977 /* load r4 with local address */
978 ld r4, 128(r1)
979 subi r4, r4, MCOUNT_INSN_SIZE
981 /* get the parent address */
982 ld r11, 112(r1)
983 addi r3, r11, 16
985 bl .prepare_ftrace_return
986 nop
988 ld r0, 128(r1)
989 mtlr r0
990 addi r1, r1, 112
991 blr
993 _GLOBAL(return_to_handler)
994 /* need to save return values */
995 std r4, -24(r1)
996 std r3, -16(r1)
997 std r31, -8(r1)
998 mr r31, r1
999 stdu r1, -112(r1)
1001 bl .ftrace_return_to_handler
1002 nop
1004 /* return value has real return address */
1005 mtlr r3
1007 ld r1, 0(r1)
1008 ld r4, -24(r1)
1009 ld r3, -16(r1)
1010 ld r31, -8(r1)
1012 /* Jump back to real return address */
1013 blr
1015 _GLOBAL(mod_return_to_handler)
1016 /* need to save return values */
1017 std r4, -32(r1)
1018 std r3, -24(r1)
1019 /* save TOC */
1020 std r2, -16(r1)
1021 std r31, -8(r1)
1022 mr r31, r1
1023 stdu r1, -112(r1)
1025 /*
1026 * We are in a module using the module's TOC.
1027 * Switch to our TOC to run inside the core kernel.
1028 */
1029 ld r2, PACATOC(r13)
1031 bl .ftrace_return_to_handler
1032 nop
1034 /* return value has real return address */
1035 mtlr r3
1037 ld r1, 0(r1)
1038 ld r4, -32(r1)
1039 ld r3, -24(r1)
1040 ld r2, -16(r1)
1041 ld r31, -8(r1)
1043 /* Jump back to real return address */
1044 blr
1045 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1046 #endif /* CONFIG_FUNCTION_TRACER */