| blob | 2b42362a85b292d4091a3ebfbafc75eb6c8dd467 |
1 /*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 */
9 /*
10 * entry.S contains the system-call and fault low-level handling routines.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
14 *
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
17 *
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
23 *
24 * Some macro usage:
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
38 */
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
47 #include <asm/msr.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
51 #include <asm/page.h>
52 #include <asm/irqflags.h>
53 #include <asm/paravirt.h>
54 #include <asm/ftrace.h>
56 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
57 #include <linux/elf-em.h>
58 #define AUDIT_ARCH_X86_64 (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
59 #define __AUDIT_ARCH_64BIT 0x80000000
60 #define __AUDIT_ARCH_LE 0x40000000
62 .code64
64 #ifdef CONFIG_FUNCTION_TRACER
65 #ifdef CONFIG_DYNAMIC_FTRACE
66 ENTRY(mcount)
67 retq
68 END(mcount)
70 ENTRY(ftrace_caller)
72 /* taken from glibc */
73 subq $0x38, %rsp
74 movq %rax, (%rsp)
75 movq %rcx, 8(%rsp)
76 movq %rdx, 16(%rsp)
77 movq %rsi, 24(%rsp)
78 movq %rdi, 32(%rsp)
79 movq %r8, 40(%rsp)
80 movq %r9, 48(%rsp)
82 movq 0x38(%rsp), %rdi
83 movq 8(%rbp), %rsi
84 subq $MCOUNT_INSN_SIZE, %rdi
86 .globl ftrace_call
87 ftrace_call:
88 call ftrace_stub
90 movq 48(%rsp), %r9
91 movq 40(%rsp), %r8
92 movq 32(%rsp), %rdi
93 movq 24(%rsp), %rsi
94 movq 16(%rsp), %rdx
95 movq 8(%rsp), %rcx
96 movq (%rsp), %rax
97 addq $0x38, %rsp
99 .globl ftrace_stub
100 ftrace_stub:
101 retq
102 END(ftrace_caller)
104 #else /* ! CONFIG_DYNAMIC_FTRACE */
105 ENTRY(mcount)
106 cmpq $ftrace_stub, ftrace_trace_function
107 jnz trace
108 .globl ftrace_stub
109 ftrace_stub:
110 retq
112 trace:
113 /* taken from glibc */
114 subq $0x38, %rsp
115 movq %rax, (%rsp)
116 movq %rcx, 8(%rsp)
117 movq %rdx, 16(%rsp)
118 movq %rsi, 24(%rsp)
119 movq %rdi, 32(%rsp)
120 movq %r8, 40(%rsp)
121 movq %r9, 48(%rsp)
123 movq 0x38(%rsp), %rdi
124 movq 8(%rbp), %rsi
125 subq $MCOUNT_INSN_SIZE, %rdi
127 call *ftrace_trace_function
129 movq 48(%rsp), %r9
130 movq 40(%rsp), %r8
131 movq 32(%rsp), %rdi
132 movq 24(%rsp), %rsi
133 movq 16(%rsp), %rdx
134 movq 8(%rsp), %rcx
135 movq (%rsp), %rax
136 addq $0x38, %rsp
138 jmp ftrace_stub
139 END(mcount)
140 #endif /* CONFIG_DYNAMIC_FTRACE */
141 #endif /* CONFIG_FUNCTION_TRACER */
143 #ifndef CONFIG_PREEMPT
144 #define retint_kernel retint_restore_args
145 #endif
147 #ifdef CONFIG_PARAVIRT
148 ENTRY(native_usergs_sysret64)
149 swapgs
150 sysretq
151 #endif /* CONFIG_PARAVIRT */
154 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
155 #ifdef CONFIG_TRACE_IRQFLAGS
156 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
157 jnc 1f
158 TRACE_IRQS_ON
159 1:
160 #endif
161 .endm
163 /*
164 * C code is not supposed to know about undefined top of stack. Every time
165 * a C function with an pt_regs argument is called from the SYSCALL based
166 * fast path FIXUP_TOP_OF_STACK is needed.
167 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
168 * manipulation.
169 */
171 /* %rsp:at FRAMEEND */
172 .macro FIXUP_TOP_OF_STACK tmp
173 movq %gs:pda_oldrsp,\tmp
174 movq \tmp,RSP(%rsp)
175 movq $__USER_DS,SS(%rsp)
176 movq $__USER_CS,CS(%rsp)
177 movq $-1,RCX(%rsp)
178 movq R11(%rsp),\tmp /* get eflags */
179 movq \tmp,EFLAGS(%rsp)
180 .endm
182 .macro RESTORE_TOP_OF_STACK tmp,offset=0
183 movq RSP-\offset(%rsp),\tmp
184 movq \tmp,%gs:pda_oldrsp
185 movq EFLAGS-\offset(%rsp),\tmp
186 movq \tmp,R11-\offset(%rsp)
187 .endm
189 .macro FAKE_STACK_FRAME child_rip
190 /* push in order ss, rsp, eflags, cs, rip */
191 xorl %eax, %eax
192 pushq $__KERNEL_DS /* ss */
193 CFI_ADJUST_CFA_OFFSET 8
194 /*CFI_REL_OFFSET ss,0*/
195 pushq %rax /* rsp */
196 CFI_ADJUST_CFA_OFFSET 8
197 CFI_REL_OFFSET rsp,0
198 pushq $(1<<9) /* eflags - interrupts on */
199 CFI_ADJUST_CFA_OFFSET 8
200 /*CFI_REL_OFFSET rflags,0*/
201 pushq $__KERNEL_CS /* cs */
202 CFI_ADJUST_CFA_OFFSET 8
203 /*CFI_REL_OFFSET cs,0*/
204 pushq \child_rip /* rip */
205 CFI_ADJUST_CFA_OFFSET 8
206 CFI_REL_OFFSET rip,0
207 pushq %rax /* orig rax */
208 CFI_ADJUST_CFA_OFFSET 8
209 .endm
211 .macro UNFAKE_STACK_FRAME
212 addq $8*6, %rsp
213 CFI_ADJUST_CFA_OFFSET -(6*8)
214 .endm
216 .macro CFI_DEFAULT_STACK start=1
217 .if \start
218 CFI_STARTPROC simple
219 CFI_SIGNAL_FRAME
220 CFI_DEF_CFA rsp,SS+8
221 .else
222 CFI_DEF_CFA_OFFSET SS+8
223 .endif
224 CFI_REL_OFFSET r15,R15
225 CFI_REL_OFFSET r14,R14
226 CFI_REL_OFFSET r13,R13
227 CFI_REL_OFFSET r12,R12
228 CFI_REL_OFFSET rbp,RBP
229 CFI_REL_OFFSET rbx,RBX
230 CFI_REL_OFFSET r11,R11
231 CFI_REL_OFFSET r10,R10
232 CFI_REL_OFFSET r9,R9
233 CFI_REL_OFFSET r8,R8
234 CFI_REL_OFFSET rax,RAX
235 CFI_REL_OFFSET rcx,RCX
236 CFI_REL_OFFSET rdx,RDX
237 CFI_REL_OFFSET rsi,RSI
238 CFI_REL_OFFSET rdi,RDI
239 CFI_REL_OFFSET rip,RIP
240 /*CFI_REL_OFFSET cs,CS*/
241 /*CFI_REL_OFFSET rflags,EFLAGS*/
242 CFI_REL_OFFSET rsp,RSP
243 /*CFI_REL_OFFSET ss,SS*/
244 .endm
245 /*
246 * A newly forked process directly context switches into this.
247 */
248 /* rdi: prev */
249 ENTRY(ret_from_fork)
250 CFI_DEFAULT_STACK
251 push kernel_eflags(%rip)
252 CFI_ADJUST_CFA_OFFSET 8
253 popf # reset kernel eflags
254 CFI_ADJUST_CFA_OFFSET -8
255 call schedule_tail
256 GET_THREAD_INFO(%rcx)
257 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
258 jnz rff_trace
259 rff_action:
260 RESTORE_REST
261 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
262 je int_ret_from_sys_call
263 testl $_TIF_IA32,TI_flags(%rcx)
264 jnz int_ret_from_sys_call
265 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
266 jmp ret_from_sys_call
267 rff_trace:
268 movq %rsp,%rdi
269 call syscall_trace_leave
270 GET_THREAD_INFO(%rcx)
271 jmp rff_action
272 CFI_ENDPROC
273 END(ret_from_fork)
275 /*
276 * System call entry. Upto 6 arguments in registers are supported.
277 *
278 * SYSCALL does not save anything on the stack and does not change the
279 * stack pointer.
280 */
282 /*
283 * Register setup:
284 * rax system call number
285 * rdi arg0
286 * rcx return address for syscall/sysret, C arg3
287 * rsi arg1
288 * rdx arg2
289 * r10 arg3 (--> moved to rcx for C)
290 * r8 arg4
291 * r9 arg5
292 * r11 eflags for syscall/sysret, temporary for C
293 * r12-r15,rbp,rbx saved by C code, not touched.
294 *
295 * Interrupts are off on entry.
296 * Only called from user space.
297 *
298 * XXX if we had a free scratch register we could save the RSP into the stack frame
299 * and report it properly in ps. Unfortunately we haven't.
300 *
301 * When user can change the frames always force IRET. That is because
302 * it deals with uncanonical addresses better. SYSRET has trouble
303 * with them due to bugs in both AMD and Intel CPUs.
304 */
306 ENTRY(system_call)
307 CFI_STARTPROC simple
308 CFI_SIGNAL_FRAME
309 CFI_DEF_CFA rsp,PDA_STACKOFFSET
310 CFI_REGISTER rip,rcx
311 /*CFI_REGISTER rflags,r11*/
312 SWAPGS_UNSAFE_STACK
313 /*
314 * A hypervisor implementation might want to use a label
315 * after the swapgs, so that it can do the swapgs
316 * for the guest and jump here on syscall.
317 */
318 ENTRY(system_call_after_swapgs)
320 movq %rsp,%gs:pda_oldrsp
321 movq %gs:pda_kernelstack,%rsp
322 /*
323 * No need to follow this irqs off/on section - it's straight
324 * and short:
325 */
326 ENABLE_INTERRUPTS(CLBR_NONE)
327 SAVE_ARGS 8,1
328 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
329 movq %rcx,RIP-ARGOFFSET(%rsp)
330 CFI_REL_OFFSET rip,RIP-ARGOFFSET
331 GET_THREAD_INFO(%rcx)
332 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
333 jnz tracesys
334 system_call_fastpath:
335 cmpq $__NR_syscall_max,%rax
336 ja badsys
337 movq %r10,%rcx
338 call *sys_call_table(,%rax,8) # XXX: rip relative
339 movq %rax,RAX-ARGOFFSET(%rsp)
340 /*
341 * Syscall return path ending with SYSRET (fast path)
342 * Has incomplete stack frame and undefined top of stack.
343 */
344 ret_from_sys_call:
345 movl $_TIF_ALLWORK_MASK,%edi
346 /* edi: flagmask */
347 sysret_check:
348 LOCKDEP_SYS_EXIT
349 GET_THREAD_INFO(%rcx)
350 DISABLE_INTERRUPTS(CLBR_NONE)
351 TRACE_IRQS_OFF
352 movl TI_flags(%rcx),%edx
353 andl %edi,%edx
354 jnz sysret_careful
355 CFI_REMEMBER_STATE
356 /*
357 * sysretq will re-enable interrupts:
358 */
359 TRACE_IRQS_ON
360 movq RIP-ARGOFFSET(%rsp),%rcx
361 CFI_REGISTER rip,rcx
362 RESTORE_ARGS 0,-ARG_SKIP,1
363 /*CFI_REGISTER rflags,r11*/
364 movq %gs:pda_oldrsp, %rsp
365 USERGS_SYSRET64
367 CFI_RESTORE_STATE
368 /* Handle reschedules */
369 /* edx: work, edi: workmask */
370 sysret_careful:
371 bt $TIF_NEED_RESCHED,%edx
372 jnc sysret_signal
373 TRACE_IRQS_ON
374 ENABLE_INTERRUPTS(CLBR_NONE)
375 pushq %rdi
376 CFI_ADJUST_CFA_OFFSET 8
377 call schedule
378 popq %rdi
379 CFI_ADJUST_CFA_OFFSET -8
380 jmp sysret_check
382 /* Handle a signal */
383 sysret_signal:
384 TRACE_IRQS_ON
385 ENABLE_INTERRUPTS(CLBR_NONE)
386 #ifdef CONFIG_AUDITSYSCALL
387 bt $TIF_SYSCALL_AUDIT,%edx
388 jc sysret_audit
389 #endif
390 /* edx: work flags (arg3) */
391 leaq do_notify_resume(%rip),%rax
392 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
393 xorl %esi,%esi # oldset -> arg2
394 call ptregscall_common
395 movl $_TIF_WORK_MASK,%edi
396 /* Use IRET because user could have changed frame. This
397 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
398 DISABLE_INTERRUPTS(CLBR_NONE)
399 TRACE_IRQS_OFF
400 jmp int_with_check
402 badsys:
403 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
404 jmp ret_from_sys_call
406 #ifdef CONFIG_AUDITSYSCALL
407 /*
408 * Fast path for syscall audit without full syscall trace.
409 * We just call audit_syscall_entry() directly, and then
410 * jump back to the normal fast path.
411 */
412 auditsys:
413 movq %r10,%r9 /* 6th arg: 4th syscall arg */
414 movq %rdx,%r8 /* 5th arg: 3rd syscall arg */
415 movq %rsi,%rcx /* 4th arg: 2nd syscall arg */
416 movq %rdi,%rdx /* 3rd arg: 1st syscall arg */
417 movq %rax,%rsi /* 2nd arg: syscall number */
418 movl $AUDIT_ARCH_X86_64,%edi /* 1st arg: audit arch */
419 call audit_syscall_entry
420 LOAD_ARGS 0 /* reload call-clobbered registers */
421 jmp system_call_fastpath
423 /*
424 * Return fast path for syscall audit. Call audit_syscall_exit()
425 * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT
426 * masked off.
427 */
428 sysret_audit:
429 movq %rax,%rsi /* second arg, syscall return value */
430 cmpq $0,%rax /* is it < 0? */
431 setl %al /* 1 if so, 0 if not */
432 movzbl %al,%edi /* zero-extend that into %edi */
433 inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
434 call audit_syscall_exit
435 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
436 jmp sysret_check
437 #endif /* CONFIG_AUDITSYSCALL */
439 /* Do syscall tracing */
440 tracesys:
441 #ifdef CONFIG_AUDITSYSCALL
442 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
443 jz auditsys
444 #endif
445 SAVE_REST
446 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
447 FIXUP_TOP_OF_STACK %rdi
448 movq %rsp,%rdi
449 call syscall_trace_enter
450 /*
451 * Reload arg registers from stack in case ptrace changed them.
452 * We don't reload %rax because syscall_trace_enter() returned
453 * the value it wants us to use in the table lookup.
454 */
455 LOAD_ARGS ARGOFFSET, 1
456 RESTORE_REST
457 cmpq $__NR_syscall_max,%rax
458 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
459 movq %r10,%rcx /* fixup for C */
460 call *sys_call_table(,%rax,8)
461 movq %rax,RAX-ARGOFFSET(%rsp)
462 /* Use IRET because user could have changed frame */
464 /*
465 * Syscall return path ending with IRET.
466 * Has correct top of stack, but partial stack frame.
467 */
468 .globl int_ret_from_sys_call
469 .globl int_with_check
470 int_ret_from_sys_call:
471 DISABLE_INTERRUPTS(CLBR_NONE)
472 TRACE_IRQS_OFF
473 testl $3,CS-ARGOFFSET(%rsp)
474 je retint_restore_args
475 movl $_TIF_ALLWORK_MASK,%edi
476 /* edi: mask to check */
477 int_with_check:
478 LOCKDEP_SYS_EXIT_IRQ
479 GET_THREAD_INFO(%rcx)
480 movl TI_flags(%rcx),%edx
481 andl %edi,%edx
482 jnz int_careful
483 andl $~TS_COMPAT,TI_status(%rcx)
484 jmp retint_swapgs
486 /* Either reschedule or signal or syscall exit tracking needed. */
487 /* First do a reschedule test. */
488 /* edx: work, edi: workmask */
489 int_careful:
490 bt $TIF_NEED_RESCHED,%edx
491 jnc int_very_careful
492 TRACE_IRQS_ON
493 ENABLE_INTERRUPTS(CLBR_NONE)
494 pushq %rdi
495 CFI_ADJUST_CFA_OFFSET 8
496 call schedule
497 popq %rdi
498 CFI_ADJUST_CFA_OFFSET -8
499 DISABLE_INTERRUPTS(CLBR_NONE)
500 TRACE_IRQS_OFF
501 jmp int_with_check
503 /* handle signals and tracing -- both require a full stack frame */
504 int_very_careful:
505 TRACE_IRQS_ON
506 ENABLE_INTERRUPTS(CLBR_NONE)
507 SAVE_REST
508 /* Check for syscall exit trace */
509 testl $_TIF_WORK_SYSCALL_EXIT,%edx
510 jz int_signal
511 pushq %rdi
512 CFI_ADJUST_CFA_OFFSET 8
513 leaq 8(%rsp),%rdi # &ptregs -> arg1
514 call syscall_trace_leave
515 popq %rdi
516 CFI_ADJUST_CFA_OFFSET -8
517 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
518 jmp int_restore_rest
520 int_signal:
521 testl $_TIF_DO_NOTIFY_MASK,%edx
522 jz 1f
523 movq %rsp,%rdi # &ptregs -> arg1
524 xorl %esi,%esi # oldset -> arg2
525 call do_notify_resume
526 1: movl $_TIF_WORK_MASK,%edi
527 int_restore_rest:
528 RESTORE_REST
529 DISABLE_INTERRUPTS(CLBR_NONE)
530 TRACE_IRQS_OFF
531 jmp int_with_check
532 CFI_ENDPROC
533 END(system_call)
535 /*
536 * Certain special system calls that need to save a complete full stack frame.
537 */
539 .macro PTREGSCALL label,func,arg
540 .globl \label
541 \label:
542 leaq \func(%rip),%rax
543 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
544 jmp ptregscall_common
545 END(\label)
546 .endm
548 CFI_STARTPROC
550 PTREGSCALL stub_clone, sys_clone, %r8
551 PTREGSCALL stub_fork, sys_fork, %rdi
552 PTREGSCALL stub_vfork, sys_vfork, %rdi
553 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
554 PTREGSCALL stub_iopl, sys_iopl, %rsi
556 ENTRY(ptregscall_common)
557 popq %r11
558 CFI_ADJUST_CFA_OFFSET -8
559 CFI_REGISTER rip, r11
560 SAVE_REST
561 movq %r11, %r15
562 CFI_REGISTER rip, r15
563 FIXUP_TOP_OF_STACK %r11
564 call *%rax
565 RESTORE_TOP_OF_STACK %r11
566 movq %r15, %r11
567 CFI_REGISTER rip, r11
568 RESTORE_REST
569 pushq %r11
570 CFI_ADJUST_CFA_OFFSET 8
571 CFI_REL_OFFSET rip, 0
572 ret
573 CFI_ENDPROC
574 END(ptregscall_common)
576 ENTRY(stub_execve)
577 CFI_STARTPROC
578 popq %r11
579 CFI_ADJUST_CFA_OFFSET -8
580 CFI_REGISTER rip, r11
581 SAVE_REST
582 FIXUP_TOP_OF_STACK %r11
583 movq %rsp, %rcx
584 call sys_execve
585 RESTORE_TOP_OF_STACK %r11
586 movq %rax,RAX(%rsp)
587 RESTORE_REST
588 jmp int_ret_from_sys_call
589 CFI_ENDPROC
590 END(stub_execve)
592 /*
593 * sigreturn is special because it needs to restore all registers on return.
594 * This cannot be done with SYSRET, so use the IRET return path instead.
595 */
596 ENTRY(stub_rt_sigreturn)
597 CFI_STARTPROC
598 addq $8, %rsp
599 CFI_ADJUST_CFA_OFFSET -8
600 SAVE_REST
601 movq %rsp,%rdi
602 FIXUP_TOP_OF_STACK %r11
603 call sys_rt_sigreturn
604 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
605 RESTORE_REST
606 jmp int_ret_from_sys_call
607 CFI_ENDPROC
608 END(stub_rt_sigreturn)
610 /*
611 * initial frame state for interrupts and exceptions
612 */
613 .macro _frame ref
614 CFI_STARTPROC simple
615 CFI_SIGNAL_FRAME
616 CFI_DEF_CFA rsp,SS+8-\ref
617 /*CFI_REL_OFFSET ss,SS-\ref*/
618 CFI_REL_OFFSET rsp,RSP-\ref
619 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
620 /*CFI_REL_OFFSET cs,CS-\ref*/
621 CFI_REL_OFFSET rip,RIP-\ref
622 .endm
624 /* initial frame state for interrupts (and exceptions without error code) */
625 #define INTR_FRAME _frame RIP
626 /* initial frame state for exceptions with error code (and interrupts with
627 vector already pushed) */
628 #define XCPT_FRAME _frame ORIG_RAX
630 /*
631 * Build the entry stubs and pointer table with some assembler magic.
632 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
633 * single cache line on all modern x86 implementations.
634 */
635 .section .init.rodata,"a"
636 ENTRY(interrupt)
637 .text
638 .p2align 5
639 .p2align CONFIG_X86_L1_CACHE_SHIFT
640 ENTRY(irq_entries_start)
641 INTR_FRAME
642 vector=FIRST_EXTERNAL_VECTOR
643 .rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
644 .balign 32
645 .rept 7
646 .if vector < NR_VECTORS
647 .if vector <> FIRST_EXTERNAL_VECTOR
648 CFI_ADJUST_CFA_OFFSET -8
649 .endif
650 1: pushq $(~vector+0x80) /* Note: always in signed byte range */
651 CFI_ADJUST_CFA_OFFSET 8
652 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
653 jmp 2f
654 .endif
655 .previous
656 .quad 1b
657 .text
658 vector=vector+1
659 .endif
660 .endr
661 2: jmp common_interrupt
662 .endr
663 CFI_ENDPROC
664 END(irq_entries_start)
666 .previous
667 END(interrupt)
668 .previous
670 /*
671 * Interrupt entry/exit.
672 *
673 * Interrupt entry points save only callee clobbered registers in fast path.
674 *
675 * Entry runs with interrupts off.
676 */
678 /* 0(%rsp): ~(interrupt number)+0x80 */
679 .macro interrupt func
680 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
681 cld
682 SAVE_ARGS
683 leaq -ARGOFFSET(%rsp),%rdi /* arg1 for handler */
684 pushq %rbp
685 /*
686 * Save rbp twice: One is for marking the stack frame, as usual, and the
687 * other, to fill pt_regs properly. This is because bx comes right
688 * before the last saved register in that structure, and not bp. If the
689 * base pointer were in the place bx is today, this would not be needed.
690 */
691 movq %rbp, -8(%rsp)
692 CFI_ADJUST_CFA_OFFSET 8
693 CFI_REL_OFFSET rbp, 0
694 movq %rsp,%rbp
695 CFI_DEF_CFA_REGISTER rbp
696 testl $3,CS(%rdi)
697 je 1f
698 SWAPGS
699 /* irqcount is used to check if a CPU is already on an interrupt
700 stack or not. While this is essentially redundant with preempt_count
701 it is a little cheaper to use a separate counter in the PDA
702 (short of moving irq_enter into assembly, which would be too
703 much work) */
704 1: incl %gs:pda_irqcount
705 cmoveq %gs:pda_irqstackptr,%rsp
706 push %rbp # backlink for old unwinder
707 /*
708 * We entered an interrupt context - irqs are off:
709 */
710 TRACE_IRQS_OFF
711 call \func
712 .endm
714 .p2align CONFIG_X86_L1_CACHE_SHIFT
715 common_interrupt:
716 XCPT_FRAME
717 interrupt do_IRQ
718 /* 0(%rsp): oldrsp-ARGOFFSET */
719 ret_from_intr:
720 DISABLE_INTERRUPTS(CLBR_NONE)
721 TRACE_IRQS_OFF
722 decl %gs:pda_irqcount
723 leaveq
724 CFI_DEF_CFA_REGISTER rsp
725 CFI_ADJUST_CFA_OFFSET -8
726 exit_intr:
727 GET_THREAD_INFO(%rcx)
728 testl $3,CS-ARGOFFSET(%rsp)
729 je retint_kernel
731 /* Interrupt came from user space */
732 /*
733 * Has a correct top of stack, but a partial stack frame
734 * %rcx: thread info. Interrupts off.
735 */
736 retint_with_reschedule:
737 movl $_TIF_WORK_MASK,%edi
738 retint_check:
739 LOCKDEP_SYS_EXIT_IRQ
740 movl TI_flags(%rcx),%edx
741 andl %edi,%edx
742 CFI_REMEMBER_STATE
743 jnz retint_careful
745 retint_swapgs: /* return to user-space */
746 /*
747 * The iretq could re-enable interrupts:
748 */
749 DISABLE_INTERRUPTS(CLBR_ANY)
750 TRACE_IRQS_IRETQ
751 SWAPGS
752 jmp restore_args
754 retint_restore_args: /* return to kernel space */
755 DISABLE_INTERRUPTS(CLBR_ANY)
756 /*
757 * The iretq could re-enable interrupts:
758 */
759 TRACE_IRQS_IRETQ
760 restore_args:
761 RESTORE_ARGS 0,8,0
763 irq_return:
764 INTERRUPT_RETURN
766 .section __ex_table, "a"
767 .quad irq_return, bad_iret
768 .previous
770 #ifdef CONFIG_PARAVIRT
771 ENTRY(native_iret)
772 iretq
774 .section __ex_table,"a"
775 .quad native_iret, bad_iret
776 .previous
777 #endif
779 .section .fixup,"ax"
780 bad_iret:
781 /*
782 * The iret traps when the %cs or %ss being restored is bogus.
783 * We've lost the original trap vector and error code.
784 * #GPF is the most likely one to get for an invalid selector.
785 * So pretend we completed the iret and took the #GPF in user mode.
786 *
787 * We are now running with the kernel GS after exception recovery.
788 * But error_entry expects us to have user GS to match the user %cs,
789 * so swap back.
790 */
791 pushq $0
793 SWAPGS
794 jmp general_protection
796 .previous
798 /* edi: workmask, edx: work */
799 retint_careful:
800 CFI_RESTORE_STATE
801 bt $TIF_NEED_RESCHED,%edx
802 jnc retint_signal
803 TRACE_IRQS_ON
804 ENABLE_INTERRUPTS(CLBR_NONE)
805 pushq %rdi
806 CFI_ADJUST_CFA_OFFSET 8
807 call schedule
808 popq %rdi
809 CFI_ADJUST_CFA_OFFSET -8
810 GET_THREAD_INFO(%rcx)
811 DISABLE_INTERRUPTS(CLBR_NONE)
812 TRACE_IRQS_OFF
813 jmp retint_check
815 retint_signal:
816 testl $_TIF_DO_NOTIFY_MASK,%edx
817 jz retint_swapgs
818 TRACE_IRQS_ON
819 ENABLE_INTERRUPTS(CLBR_NONE)
820 SAVE_REST
821 movq $-1,ORIG_RAX(%rsp)
822 xorl %esi,%esi # oldset
823 movq %rsp,%rdi # &pt_regs
824 call do_notify_resume
825 RESTORE_REST
826 DISABLE_INTERRUPTS(CLBR_NONE)
827 TRACE_IRQS_OFF
828 GET_THREAD_INFO(%rcx)
829 jmp retint_with_reschedule
831 #ifdef CONFIG_PREEMPT
832 /* Returning to kernel space. Check if we need preemption */
833 /* rcx: threadinfo. interrupts off. */
834 ENTRY(retint_kernel)
835 cmpl $0,TI_preempt_count(%rcx)
836 jnz retint_restore_args
837 bt $TIF_NEED_RESCHED,TI_flags(%rcx)
838 jnc retint_restore_args
839 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
840 jnc retint_restore_args
841 call preempt_schedule_irq
842 jmp exit_intr
843 #endif
845 CFI_ENDPROC
846 END(common_interrupt)
848 /*
849 * APIC interrupts.
850 */
851 .macro apicinterrupt num,func
852 INTR_FRAME
853 pushq $~(\num)
854 CFI_ADJUST_CFA_OFFSET 8
855 interrupt \func
856 jmp ret_from_intr
857 CFI_ENDPROC
858 .endm
860 ENTRY(thermal_interrupt)
861 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
862 END(thermal_interrupt)
864 ENTRY(threshold_interrupt)
865 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
866 END(threshold_interrupt)
868 #ifdef CONFIG_SMP
869 ENTRY(reschedule_interrupt)
870 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
871 END(reschedule_interrupt)
873 .macro INVALIDATE_ENTRY num
874 ENTRY(invalidate_interrupt\num)
875 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
876 END(invalidate_interrupt\num)
877 .endm
879 INVALIDATE_ENTRY 0
880 INVALIDATE_ENTRY 1
881 INVALIDATE_ENTRY 2
882 INVALIDATE_ENTRY 3
883 INVALIDATE_ENTRY 4
884 INVALIDATE_ENTRY 5
885 INVALIDATE_ENTRY 6
886 INVALIDATE_ENTRY 7
888 ENTRY(call_function_interrupt)
889 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
890 END(call_function_interrupt)
891 ENTRY(call_function_single_interrupt)
892 apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
893 END(call_function_single_interrupt)
894 ENTRY(irq_move_cleanup_interrupt)
895 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
896 END(irq_move_cleanup_interrupt)
897 #endif
899 ENTRY(apic_timer_interrupt)
900 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
901 END(apic_timer_interrupt)
903 ENTRY(uv_bau_message_intr1)
904 apicinterrupt 220,uv_bau_message_interrupt
905 END(uv_bau_message_intr1)
907 ENTRY(error_interrupt)
908 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
909 END(error_interrupt)
911 ENTRY(spurious_interrupt)
912 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
913 END(spurious_interrupt)
915 /*
916 * Exception entry points.
917 */
918 .macro zeroentry sym
919 INTR_FRAME
920 PARAVIRT_ADJUST_EXCEPTION_FRAME
921 pushq $0 /* push error code/oldrax */
922 CFI_ADJUST_CFA_OFFSET 8
923 pushq %rax /* push real oldrax to the rdi slot */
924 CFI_ADJUST_CFA_OFFSET 8
925 CFI_REL_OFFSET rax,0
926 leaq \sym(%rip),%rax
927 jmp error_entry
928 CFI_ENDPROC
929 .endm
931 .macro errorentry sym
932 XCPT_FRAME
933 PARAVIRT_ADJUST_EXCEPTION_FRAME
934 pushq %rax
935 CFI_ADJUST_CFA_OFFSET 8
936 CFI_REL_OFFSET rax,0
937 leaq \sym(%rip),%rax
938 jmp error_entry
939 CFI_ENDPROC
940 .endm
942 /* error code is on the stack already */
943 /* handle NMI like exceptions that can happen everywhere */
944 .macro paranoidentry sym, ist=0, irqtrace=1
945 SAVE_ALL
946 cld
947 movl $1,%ebx
948 movl $MSR_GS_BASE,%ecx
949 rdmsr
950 testl %edx,%edx
951 js 1f
952 SWAPGS
953 xorl %ebx,%ebx
954 1:
955 .if \ist
956 movq %gs:pda_data_offset, %rbp
957 .endif
958 .if \irqtrace
959 TRACE_IRQS_OFF
960 .endif
961 movq %rsp,%rdi
962 movq ORIG_RAX(%rsp),%rsi
963 movq $-1,ORIG_RAX(%rsp)
964 .if \ist
965 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
966 .endif
967 call \sym
968 .if \ist
969 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
970 .endif
971 DISABLE_INTERRUPTS(CLBR_NONE)
972 .if \irqtrace
973 TRACE_IRQS_OFF
974 .endif
975 .endm
977 /*
978 * "Paranoid" exit path from exception stack.
979 * Paranoid because this is used by NMIs and cannot take
980 * any kernel state for granted.
981 * We don't do kernel preemption checks here, because only
982 * NMI should be common and it does not enable IRQs and
983 * cannot get reschedule ticks.
984 *
985 * "trace" is 0 for the NMI handler only, because irq-tracing
986 * is fundamentally NMI-unsafe. (we cannot change the soft and
987 * hard flags at once, atomically)
988 */
989 .macro paranoidexit trace=1
990 /* ebx: no swapgs flag */
991 paranoid_exit\trace:
992 testl %ebx,%ebx /* swapgs needed? */
993 jnz paranoid_restore\trace
994 testl $3,CS(%rsp)
995 jnz paranoid_userspace\trace
996 paranoid_swapgs\trace:
997 .if \trace
998 TRACE_IRQS_IRETQ 0
999 .endif
1000 SWAPGS_UNSAFE_STACK
1001 paranoid_restore\trace:
1002 RESTORE_ALL 8
1003 jmp irq_return
1004 paranoid_userspace\trace:
1005 GET_THREAD_INFO(%rcx)
1006 movl TI_flags(%rcx),%ebx
1007 andl $_TIF_WORK_MASK,%ebx
1008 jz paranoid_swapgs\trace
1009 movq %rsp,%rdi /* &pt_regs */
1010 call sync_regs
1011 movq %rax,%rsp /* switch stack for scheduling */
1012 testl $_TIF_NEED_RESCHED,%ebx
1013 jnz paranoid_schedule\trace
1014 movl %ebx,%edx /* arg3: thread flags */
1015 .if \trace
1016 TRACE_IRQS_ON
1017 .endif
1018 ENABLE_INTERRUPTS(CLBR_NONE)
1019 xorl %esi,%esi /* arg2: oldset */
1020 movq %rsp,%rdi /* arg1: &pt_regs */
1021 call do_notify_resume
1022 DISABLE_INTERRUPTS(CLBR_NONE)
1023 .if \trace
1024 TRACE_IRQS_OFF
1025 .endif
1026 jmp paranoid_userspace\trace
1027 paranoid_schedule\trace:
1028 .if \trace
1029 TRACE_IRQS_ON
1030 .endif
1031 ENABLE_INTERRUPTS(CLBR_ANY)
1032 call schedule
1033 DISABLE_INTERRUPTS(CLBR_ANY)
1034 .if \trace
1035 TRACE_IRQS_OFF
1036 .endif
1037 jmp paranoid_userspace\trace
1038 CFI_ENDPROC
1039 .endm
1041 /*
1042 * Exception entry point. This expects an error code/orig_rax on the stack
1043 * and the exception handler in %rax.
1044 */
1045 KPROBE_ENTRY(error_entry)
1046 _frame RDI
1047 CFI_REL_OFFSET rax,0
1048 /* rdi slot contains rax, oldrax contains error code */
1049 cld
1050 subq $14*8,%rsp
1051 CFI_ADJUST_CFA_OFFSET (14*8)
1052 movq %rsi,13*8(%rsp)
1053 CFI_REL_OFFSET rsi,RSI
1054 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
1055 CFI_REGISTER rax,rsi
1056 movq %rdx,12*8(%rsp)
1057 CFI_REL_OFFSET rdx,RDX
1058 movq %rcx,11*8(%rsp)
1059 CFI_REL_OFFSET rcx,RCX
1060 movq %rsi,10*8(%rsp) /* store rax */
1061 CFI_REL_OFFSET rax,RAX
1062 movq %r8, 9*8(%rsp)
1063 CFI_REL_OFFSET r8,R8
1064 movq %r9, 8*8(%rsp)
1065 CFI_REL_OFFSET r9,R9
1066 movq %r10,7*8(%rsp)
1067 CFI_REL_OFFSET r10,R10
1068 movq %r11,6*8(%rsp)
1069 CFI_REL_OFFSET r11,R11
1070 movq %rbx,5*8(%rsp)
1071 CFI_REL_OFFSET rbx,RBX
1072 movq %rbp,4*8(%rsp)
1073 CFI_REL_OFFSET rbp,RBP
1074 movq %r12,3*8(%rsp)
1075 CFI_REL_OFFSET r12,R12
1076 movq %r13,2*8(%rsp)
1077 CFI_REL_OFFSET r13,R13
1078 movq %r14,1*8(%rsp)
1079 CFI_REL_OFFSET r14,R14
1080 movq %r15,(%rsp)
1081 CFI_REL_OFFSET r15,R15
1082 xorl %ebx,%ebx
1083 testl $3,CS(%rsp)
1084 je error_kernelspace
1085 error_swapgs:
1086 SWAPGS
1087 error_sti:
1088 TRACE_IRQS_OFF
1089 movq %rdi,RDI(%rsp)
1090 CFI_REL_OFFSET rdi,RDI
1091 movq %rsp,%rdi
1092 movq ORIG_RAX(%rsp),%rsi /* get error code */
1093 movq $-1,ORIG_RAX(%rsp)
1094 call *%rax
1095 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
1096 error_exit:
1097 movl %ebx,%eax
1098 RESTORE_REST
1099 DISABLE_INTERRUPTS(CLBR_NONE)
1100 TRACE_IRQS_OFF
1101 GET_THREAD_INFO(%rcx)
1102 testl %eax,%eax
1103 jne retint_kernel
1104 LOCKDEP_SYS_EXIT_IRQ
1105 movl TI_flags(%rcx),%edx
1106 movl $_TIF_WORK_MASK,%edi
1107 andl %edi,%edx
1108 jnz retint_careful
1109 jmp retint_swapgs
1110 CFI_ENDPROC
1112 error_kernelspace:
1113 incl %ebx
1114 /* There are two places in the kernel that can potentially fault with
1115 usergs. Handle them here. The exception handlers after
1116 iret run with kernel gs again, so don't set the user space flag.
1117 B stepping K8s sometimes report an truncated RIP for IRET
1118 exceptions returning to compat mode. Check for these here too. */
1119 leaq irq_return(%rip),%rcx
1120 cmpq %rcx,RIP(%rsp)
1121 je error_swapgs
1122 movl %ecx,%ecx /* zero extend */
1123 cmpq %rcx,RIP(%rsp)
1124 je error_swapgs
1125 cmpq $gs_change,RIP(%rsp)
1126 je error_swapgs
1127 jmp error_sti
1128 KPROBE_END(error_entry)
1130 /* Reload gs selector with exception handling */
1131 /* edi: new selector */
1132 ENTRY(native_load_gs_index)
1133 CFI_STARTPROC
1134 pushf
1135 CFI_ADJUST_CFA_OFFSET 8
1136 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
1137 SWAPGS
1138 gs_change:
1139 movl %edi,%gs
1140 2: mfence /* workaround */
1141 SWAPGS
1142 popf
1143 CFI_ADJUST_CFA_OFFSET -8
1144 ret
1145 CFI_ENDPROC
1146 ENDPROC(native_load_gs_index)
1148 .section __ex_table,"a"
1149 .align 8
1150 .quad gs_change,bad_gs
1151 .previous
1152 .section .fixup,"ax"
1153 /* running with kernelgs */
1154 bad_gs:
1155 SWAPGS /* switch back to user gs */
1156 xorl %eax,%eax
1157 movl %eax,%gs
1158 jmp 2b
1159 .previous
1161 /*
1162 * Create a kernel thread.
1163 *
1164 * C extern interface:
1165 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
1166 *
1167 * asm input arguments:
1168 * rdi: fn, rsi: arg, rdx: flags
1169 */
1170 ENTRY(kernel_thread)
1171 CFI_STARTPROC
1172 FAKE_STACK_FRAME $child_rip
1173 SAVE_ALL
1175 # rdi: flags, rsi: usp, rdx: will be &pt_regs
1176 movq %rdx,%rdi
1177 orq kernel_thread_flags(%rip),%rdi
1178 movq $-1, %rsi
1179 movq %rsp, %rdx
1181 xorl %r8d,%r8d
1182 xorl %r9d,%r9d
1184 # clone now
1185 call do_fork
1186 movq %rax,RAX(%rsp)
1187 xorl %edi,%edi
1189 /*
1190 * It isn't worth to check for reschedule here,
1191 * so internally to the x86_64 port you can rely on kernel_thread()
1192 * not to reschedule the child before returning, this avoids the need
1193 * of hacks for example to fork off the per-CPU idle tasks.
1194 * [Hopefully no generic code relies on the reschedule -AK]
1195 */
1196 RESTORE_ALL
1197 UNFAKE_STACK_FRAME
1198 ret
1199 CFI_ENDPROC
1200 ENDPROC(kernel_thread)
1202 child_rip:
1203 pushq $0 # fake return address
1204 CFI_STARTPROC
1205 /*
1206 * Here we are in the child and the registers are set as they were
1207 * at kernel_thread() invocation in the parent.
1208 */
1209 movq %rdi, %rax
1210 movq %rsi, %rdi
1211 call *%rax
1212 # exit
1213 mov %eax, %edi
1214 call do_exit
1215 CFI_ENDPROC
1216 ENDPROC(child_rip)
1218 /*
1219 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1220 *
1221 * C extern interface:
1222 * extern long execve(char *name, char **argv, char **envp)
1223 *
1224 * asm input arguments:
1225 * rdi: name, rsi: argv, rdx: envp
1226 *
1227 * We want to fallback into:
1228 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
1229 *
1230 * do_sys_execve asm fallback arguments:
1231 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1232 */
1233 ENTRY(kernel_execve)
1234 CFI_STARTPROC
1235 FAKE_STACK_FRAME $0
1236 SAVE_ALL
1237 movq %rsp,%rcx
1238 call sys_execve
1239 movq %rax, RAX(%rsp)
1240 RESTORE_REST
1241 testq %rax,%rax
1242 je int_ret_from_sys_call
1243 RESTORE_ARGS
1244 UNFAKE_STACK_FRAME
1245 ret
1246 CFI_ENDPROC
1247 ENDPROC(kernel_execve)
1249 KPROBE_ENTRY(page_fault)
1250 errorentry do_page_fault
1251 KPROBE_END(page_fault)
1253 ENTRY(coprocessor_error)
1254 zeroentry do_coprocessor_error
1255 END(coprocessor_error)
1257 ENTRY(simd_coprocessor_error)
1258 zeroentry do_simd_coprocessor_error
1259 END(simd_coprocessor_error)
1261 ENTRY(device_not_available)
1262 zeroentry do_device_not_available
1263 END(device_not_available)
1265 /* runs on exception stack */
1266 KPROBE_ENTRY(debug)
1267 INTR_FRAME
1268 PARAVIRT_ADJUST_EXCEPTION_FRAME
1269 pushq $0
1270 CFI_ADJUST_CFA_OFFSET 8
1271 paranoidentry do_debug, DEBUG_STACK
1272 paranoidexit
1273 KPROBE_END(debug)
1275 /* runs on exception stack */
1276 KPROBE_ENTRY(nmi)
1277 INTR_FRAME
1278 PARAVIRT_ADJUST_EXCEPTION_FRAME
1279 pushq $-1
1280 CFI_ADJUST_CFA_OFFSET 8
1281 paranoidentry do_nmi, 0, 0
1282 #ifdef CONFIG_TRACE_IRQFLAGS
1283 paranoidexit 0
1284 #else
1285 jmp paranoid_exit1
1286 CFI_ENDPROC
1287 #endif
1288 KPROBE_END(nmi)
1290 KPROBE_ENTRY(int3)
1291 INTR_FRAME
1292 PARAVIRT_ADJUST_EXCEPTION_FRAME
1293 pushq $0
1294 CFI_ADJUST_CFA_OFFSET 8
1295 paranoidentry do_int3, DEBUG_STACK
1296 jmp paranoid_exit1
1297 CFI_ENDPROC
1298 KPROBE_END(int3)
1300 ENTRY(overflow)
1301 zeroentry do_overflow
1302 END(overflow)
1304 ENTRY(bounds)
1305 zeroentry do_bounds
1306 END(bounds)
1308 ENTRY(invalid_op)
1309 zeroentry do_invalid_op
1310 END(invalid_op)
1312 ENTRY(coprocessor_segment_overrun)
1313 zeroentry do_coprocessor_segment_overrun
1314 END(coprocessor_segment_overrun)
1316 /* runs on exception stack */
1317 ENTRY(double_fault)
1318 XCPT_FRAME
1319 PARAVIRT_ADJUST_EXCEPTION_FRAME
1320 paranoidentry do_double_fault
1321 jmp paranoid_exit1
1322 CFI_ENDPROC
1323 END(double_fault)
1325 ENTRY(invalid_TSS)
1326 errorentry do_invalid_TSS
1327 END(invalid_TSS)
1329 ENTRY(segment_not_present)
1330 errorentry do_segment_not_present
1331 END(segment_not_present)
1333 /* runs on exception stack */
1334 ENTRY(stack_segment)
1335 XCPT_FRAME
1336 PARAVIRT_ADJUST_EXCEPTION_FRAME
1337 paranoidentry do_stack_segment
1338 jmp paranoid_exit1
1339 CFI_ENDPROC
1340 END(stack_segment)
1342 KPROBE_ENTRY(general_protection)
1343 errorentry do_general_protection
1344 KPROBE_END(general_protection)
1346 ENTRY(alignment_check)
1347 errorentry do_alignment_check
1348 END(alignment_check)
1350 ENTRY(divide_error)
1351 zeroentry do_divide_error
1352 END(divide_error)
1354 ENTRY(spurious_interrupt_bug)
1355 zeroentry do_spurious_interrupt_bug
1356 END(spurious_interrupt_bug)
1358 #ifdef CONFIG_X86_MCE
1359 /* runs on exception stack */
1360 ENTRY(machine_check)
1361 INTR_FRAME
1362 PARAVIRT_ADJUST_EXCEPTION_FRAME
1363 pushq $0
1364 CFI_ADJUST_CFA_OFFSET 8
1365 paranoidentry do_machine_check
1366 jmp paranoid_exit1
1367 CFI_ENDPROC
1368 END(machine_check)
1369 #endif
1371 /* Call softirq on interrupt stack. Interrupts are off. */
1372 ENTRY(call_softirq)
1373 CFI_STARTPROC
1374 push %rbp
1375 CFI_ADJUST_CFA_OFFSET 8
1376 CFI_REL_OFFSET rbp,0
1377 mov %rsp,%rbp
1378 CFI_DEF_CFA_REGISTER rbp
1379 incl %gs:pda_irqcount
1380 cmove %gs:pda_irqstackptr,%rsp
1381 push %rbp # backlink for old unwinder
1382 call __do_softirq
1383 leaveq
1384 CFI_DEF_CFA_REGISTER rsp
1385 CFI_ADJUST_CFA_OFFSET -8
1386 decl %gs:pda_irqcount
1387 ret
1388 CFI_ENDPROC
1389 ENDPROC(call_softirq)
1391 KPROBE_ENTRY(ignore_sysret)
1392 CFI_STARTPROC
1393 mov $-ENOSYS,%eax
1394 sysret
1395 CFI_ENDPROC
1396 ENDPROC(ignore_sysret)
1398 #ifdef CONFIG_XEN
1399 ENTRY(xen_hypervisor_callback)
1400 zeroentry xen_do_hypervisor_callback
1401 END(xen_hypervisor_callback)
1403 /*
1404 # A note on the "critical region" in our callback handler.
1405 # We want to avoid stacking callback handlers due to events occurring
1406 # during handling of the last event. To do this, we keep events disabled
1407 # until we've done all processing. HOWEVER, we must enable events before
1408 # popping the stack frame (can't be done atomically) and so it would still
1409 # be possible to get enough handler activations to overflow the stack.
1410 # Although unlikely, bugs of that kind are hard to track down, so we'd
1411 # like to avoid the possibility.
1412 # So, on entry to the handler we detect whether we interrupted an
1413 # existing activation in its critical region -- if so, we pop the current
1414 # activation and restart the handler using the previous one.
1415 */
1416 ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs)
1417 CFI_STARTPROC
1418 /* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
1419 see the correct pointer to the pt_regs */
1420 movq %rdi, %rsp # we don't return, adjust the stack frame
1421 CFI_ENDPROC
1422 CFI_DEFAULT_STACK
1423 11: incl %gs:pda_irqcount
1424 movq %rsp,%rbp
1425 CFI_DEF_CFA_REGISTER rbp
1426 cmovzq %gs:pda_irqstackptr,%rsp
1427 pushq %rbp # backlink for old unwinder
1428 call xen_evtchn_do_upcall
1429 popq %rsp
1430 CFI_DEF_CFA_REGISTER rsp
1431 decl %gs:pda_irqcount
1432 jmp error_exit
1433 CFI_ENDPROC
1434 END(do_hypervisor_callback)
1436 /*
1437 # Hypervisor uses this for application faults while it executes.
1438 # We get here for two reasons:
1439 # 1. Fault while reloading DS, ES, FS or GS
1440 # 2. Fault while executing IRET
1441 # Category 1 we do not need to fix up as Xen has already reloaded all segment
1442 # registers that could be reloaded and zeroed the others.
1443 # Category 2 we fix up by killing the current process. We cannot use the
1444 # normal Linux return path in this case because if we use the IRET hypercall
1445 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1446 # We distinguish between categories by comparing each saved segment register
1447 # with its current contents: any discrepancy means we in category 1.
1448 */
1449 ENTRY(xen_failsafe_callback)
1450 framesz = (RIP-0x30) /* workaround buggy gas */
1451 _frame framesz
1452 CFI_REL_OFFSET rcx, 0
1453 CFI_REL_OFFSET r11, 8
1454 movw %ds,%cx
1455 cmpw %cx,0x10(%rsp)
1456 CFI_REMEMBER_STATE
1457 jne 1f
1458 movw %es,%cx
1459 cmpw %cx,0x18(%rsp)
1460 jne 1f
1461 movw %fs,%cx
1462 cmpw %cx,0x20(%rsp)
1463 jne 1f
1464 movw %gs,%cx
1465 cmpw %cx,0x28(%rsp)
1466 jne 1f
1467 /* All segments match their saved values => Category 2 (Bad IRET). */
1468 movq (%rsp),%rcx
1469 CFI_RESTORE rcx
1470 movq 8(%rsp),%r11
1471 CFI_RESTORE r11
1472 addq $0x30,%rsp
1473 CFI_ADJUST_CFA_OFFSET -0x30
1474 pushq $0
1475 CFI_ADJUST_CFA_OFFSET 8
1476 pushq %r11
1477 CFI_ADJUST_CFA_OFFSET 8
1478 pushq %rcx
1479 CFI_ADJUST_CFA_OFFSET 8
1480 jmp general_protection
1481 CFI_RESTORE_STATE
1482 1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1483 movq (%rsp),%rcx
1484 CFI_RESTORE rcx
1485 movq 8(%rsp),%r11
1486 CFI_RESTORE r11
1487 addq $0x30,%rsp
1488 CFI_ADJUST_CFA_OFFSET -0x30
1489 pushq $0
1490 CFI_ADJUST_CFA_OFFSET 8
1491 SAVE_ALL
1492 jmp error_exit
1493 CFI_ENDPROC
1494 END(xen_failsafe_callback)
1496 #endif /* CONFIG_XEN */