2 * unaligned.c: Unaligned load/store trap handling with special
3 * cases for the kernel to do them more quickly.
5 * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net)
6 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
10 #include <linux/jiffies.h>
11 #include <linux/kernel.h>
12 #include <linux/sched.h>
14 #include <linux/module.h>
16 #include <asm/ptrace.h>
17 #include <asm/pstate.h>
18 #include <asm/processor.h>
19 #include <asm/system.h>
20 #include <asm/uaccess.h>
21 #include <linux/smp.h>
22 #include <linux/bitops.h>
23 #include <linux/perf_event.h>
24 #include <asm/fpumacro.h>
27 load
, /* ld, ldd, ldh, ldsh */
28 store
, /* st, std, sth, stsh */
29 both
, /* Swap, ldstub, cas, ... */
35 static inline enum direction
decode_direction(unsigned int insn
)
37 unsigned long tmp
= (insn
>> 21) & 1;
42 switch ((insn
>>19)&0xf) {
51 /* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */
52 static inline int decode_access_size(unsigned int insn
)
56 tmp
= ((insn
>> 19) & 0xf);
57 if (tmp
== 11 || tmp
== 14) /* ldx/stx */
63 return 16; /* ldd/std - Although it is actually 8 */
67 printk("Impossible unaligned trap. insn=%08x\n", insn
);
68 die_if_kernel("Byte sized unaligned access?!?!", current_thread_info()->kregs
);
70 /* GCC should never warn that control reaches the end
71 * of this function without returning a value because
72 * die_if_kernel() is marked with attribute 'noreturn'.
73 * Alas, some versions do...
80 static inline int decode_asi(unsigned int insn
, struct pt_regs
*regs
)
82 if (insn
& 0x800000) {
84 return (unsigned char)(regs
->tstate
>> 24); /* %asi */
86 return (unsigned char)(insn
>> 5); /* imm_asi */
91 /* 0x400000 = signed, 0 = unsigned */
92 static inline int decode_signedness(unsigned int insn
)
94 return (insn
& 0x400000);
97 static inline void maybe_flush_windows(unsigned int rs1
, unsigned int rs2
,
98 unsigned int rd
, int from_kernel
)
100 if (rs2
>= 16 || rs1
>= 16 || rd
>= 16) {
101 if (from_kernel
!= 0)
102 __asm__
__volatile__("flushw");
108 static inline long sign_extend_imm13(long imm
)
110 return imm
<< 51 >> 51;
113 static unsigned long fetch_reg(unsigned int reg
, struct pt_regs
*regs
)
118 return (!reg
? 0 : regs
->u_regs
[reg
]);
119 if (regs
->tstate
& TSTATE_PRIV
) {
120 struct reg_window
*win
;
121 win
= (struct reg_window
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
122 value
= win
->locals
[reg
- 16];
123 } else if (test_thread_flag(TIF_32BIT
)) {
124 struct reg_window32 __user
*win32
;
125 win32
= (struct reg_window32 __user
*)((unsigned long)((u32
)regs
->u_regs
[UREG_FP
]));
126 get_user(value
, &win32
->locals
[reg
- 16]);
128 struct reg_window __user
*win
;
129 win
= (struct reg_window __user
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
130 get_user(value
, &win
->locals
[reg
- 16]);
135 static unsigned long *fetch_reg_addr(unsigned int reg
, struct pt_regs
*regs
)
138 return ®s
->u_regs
[reg
];
139 if (regs
->tstate
& TSTATE_PRIV
) {
140 struct reg_window
*win
;
141 win
= (struct reg_window
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
142 return &win
->locals
[reg
- 16];
143 } else if (test_thread_flag(TIF_32BIT
)) {
144 struct reg_window32
*win32
;
145 win32
= (struct reg_window32
*)((unsigned long)((u32
)regs
->u_regs
[UREG_FP
]));
146 return (unsigned long *)&win32
->locals
[reg
- 16];
148 struct reg_window
*win
;
149 win
= (struct reg_window
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
150 return &win
->locals
[reg
- 16];
154 unsigned long compute_effective_address(struct pt_regs
*regs
,
155 unsigned int insn
, unsigned int rd
)
157 unsigned int rs1
= (insn
>> 14) & 0x1f;
158 unsigned int rs2
= insn
& 0x1f;
159 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
162 maybe_flush_windows(rs1
, 0, rd
, from_kernel
);
163 return (fetch_reg(rs1
, regs
) + sign_extend_imm13(insn
));
165 maybe_flush_windows(rs1
, rs2
, rd
, from_kernel
);
166 return (fetch_reg(rs1
, regs
) + fetch_reg(rs2
, regs
));
170 /* This is just to make gcc think die_if_kernel does return... */
171 static void __used
unaligned_panic(char *str
, struct pt_regs
*regs
)
173 die_if_kernel(str
, regs
);
176 extern int do_int_load(unsigned long *dest_reg
, int size
,
177 unsigned long *saddr
, int is_signed
, int asi
);
179 extern int __do_int_store(unsigned long *dst_addr
, int size
,
180 unsigned long src_val
, int asi
);
182 static inline int do_int_store(int reg_num
, int size
, unsigned long *dst_addr
,
183 struct pt_regs
*regs
, int asi
, int orig_asi
)
185 unsigned long zero
= 0;
186 unsigned long *src_val_p
= &zero
;
187 unsigned long src_val
;
191 zero
= (((long)(reg_num
?
192 (unsigned)fetch_reg(reg_num
, regs
) : 0)) << 32) |
193 (unsigned)fetch_reg(reg_num
+ 1, regs
);
194 } else if (reg_num
) {
195 src_val_p
= fetch_reg_addr(reg_num
, regs
);
197 src_val
= *src_val_p
;
198 if (unlikely(asi
!= orig_asi
)) {
201 src_val
= swab16(src_val
);
204 src_val
= swab32(src_val
);
207 src_val
= swab64(src_val
);
215 return __do_int_store(dst_addr
, size
, src_val
, asi
);
218 static inline void advance(struct pt_regs
*regs
)
220 regs
->tpc
= regs
->tnpc
;
222 if (test_thread_flag(TIF_32BIT
)) {
223 regs
->tpc
&= 0xffffffff;
224 regs
->tnpc
&= 0xffffffff;
228 static inline int floating_point_load_or_store_p(unsigned int insn
)
230 return (insn
>> 24) & 1;
233 static inline int ok_for_kernel(unsigned int insn
)
235 return !floating_point_load_or_store_p(insn
);
238 static void kernel_mna_trap_fault(int fixup_tstate_asi
)
240 struct pt_regs
*regs
= current_thread_info()->kern_una_regs
;
241 unsigned int insn
= current_thread_info()->kern_una_insn
;
242 const struct exception_table_entry
*entry
;
244 entry
= search_exception_tables(regs
->tpc
);
246 unsigned long address
;
248 address
= compute_effective_address(regs
, insn
,
249 ((insn
>> 25) & 0x1f));
250 if (address
< PAGE_SIZE
) {
251 printk(KERN_ALERT
"Unable to handle kernel NULL "
252 "pointer dereference in mna handler");
254 printk(KERN_ALERT
"Unable to handle kernel paging "
255 "request in mna handler");
256 printk(KERN_ALERT
" at virtual address %016lx\n",address
);
257 printk(KERN_ALERT
"current->{active_,}mm->context = %016lx\n",
258 (current
->mm
? CTX_HWBITS(current
->mm
->context
) :
259 CTX_HWBITS(current
->active_mm
->context
)));
260 printk(KERN_ALERT
"current->{active_,}mm->pgd = %016lx\n",
261 (current
->mm
? (unsigned long) current
->mm
->pgd
:
262 (unsigned long) current
->active_mm
->pgd
));
263 die_if_kernel("Oops", regs
);
266 regs
->tpc
= entry
->fixup
;
267 regs
->tnpc
= regs
->tpc
+ 4;
269 if (fixup_tstate_asi
) {
270 regs
->tstate
&= ~TSTATE_ASI
;
271 regs
->tstate
|= (ASI_AIUS
<< 24UL);
275 static void log_unaligned(struct pt_regs
*regs
)
277 static unsigned long count
, last_time
;
279 if (time_after(jiffies
, last_time
+ 5 * HZ
))
284 printk("Kernel unaligned access at TPC[%lx] %pS\n",
285 regs
->tpc
, (void *) regs
->tpc
);
289 asmlinkage
void kernel_unaligned_trap(struct pt_regs
*regs
, unsigned int insn
)
291 enum direction dir
= decode_direction(insn
);
292 int size
= decode_access_size(insn
);
295 current_thread_info()->kern_una_regs
= regs
;
296 current_thread_info()->kern_una_insn
= insn
;
298 orig_asi
= asi
= decode_asi(insn
, regs
);
300 /* If this is a {get,put}_user() on an unaligned userspace pointer,
301 * just signal a fault and do not log the event.
303 if (asi
== ASI_AIUS
) {
304 kernel_mna_trap_fault(0);
310 if (!ok_for_kernel(insn
) || dir
== both
) {
311 printk("Unsupported unaligned load/store trap for kernel "
312 "at <%016lx>.\n", regs
->tpc
);
313 unaligned_panic("Kernel does fpu/atomic "
314 "unaligned load/store.", regs
);
316 kernel_mna_trap_fault(0);
318 unsigned long addr
, *reg_addr
;
321 addr
= compute_effective_address(regs
, insn
,
322 ((insn
>> 25) & 0x1f));
323 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, 0, regs
, addr
);
337 reg_addr
= fetch_reg_addr(((insn
>>25)&0x1f), regs
);
338 err
= do_int_load(reg_addr
, size
,
339 (unsigned long *) addr
,
340 decode_signedness(insn
), asi
);
341 if (likely(!err
) && unlikely(asi
!= orig_asi
)) {
342 unsigned long val_in
= *reg_addr
;
345 val_in
= swab16(val_in
);
348 val_in
= swab32(val_in
);
351 val_in
= swab64(val_in
);
363 err
= do_int_store(((insn
>>25)&0x1f), size
,
364 (unsigned long *) addr
, regs
,
369 panic("Impossible kernel unaligned trap.");
373 kernel_mna_trap_fault(1);
379 static char popc_helper
[] = {
380 0, 1, 1, 2, 1, 2, 2, 3,
381 1, 2, 2, 3, 2, 3, 3, 4,
384 int handle_popc(u32 insn
, struct pt_regs
*regs
)
387 int ret
, i
, rd
= ((insn
>> 25) & 0x1f);
388 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
390 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, 0, regs
, 0);
392 maybe_flush_windows(0, 0, rd
, from_kernel
);
393 value
= sign_extend_imm13(insn
);
395 maybe_flush_windows(0, insn
& 0x1f, rd
, from_kernel
);
396 value
= fetch_reg(insn
& 0x1f, regs
);
398 for (ret
= 0, i
= 0; i
< 16; i
++) {
399 ret
+= popc_helper
[value
& 0xf];
404 regs
->u_regs
[rd
] = ret
;
406 if (test_thread_flag(TIF_32BIT
)) {
407 struct reg_window32 __user
*win32
;
408 win32
= (struct reg_window32 __user
*)((unsigned long)((u32
)regs
->u_regs
[UREG_FP
]));
409 put_user(ret
, &win32
->locals
[rd
- 16]);
411 struct reg_window __user
*win
;
412 win
= (struct reg_window __user
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
413 put_user(ret
, &win
->locals
[rd
- 16]);
420 extern void do_fpother(struct pt_regs
*regs
);
421 extern void do_privact(struct pt_regs
*regs
);
422 extern void spitfire_data_access_exception(struct pt_regs
*regs
,
425 extern void sun4v_data_access_exception(struct pt_regs
*regs
,
427 unsigned long type_ctx
);
429 int handle_ldf_stq(u32 insn
, struct pt_regs
*regs
)
431 unsigned long addr
= compute_effective_address(regs
, insn
, 0);
432 int freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
433 struct fpustate
*f
= FPUSTATE
;
434 int asi
= decode_asi(insn
, regs
);
435 int flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
437 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, 0, regs
, 0);
439 save_and_clear_fpu();
440 current_thread_info()->xfsr
[0] &= ~0x1c000;
442 current_thread_info()->xfsr
[0] |= (6 << 14) /* invalid_fp_register */;
446 if (insn
& 0x200000) {
448 u64 first
= 0, second
= 0;
450 if (current_thread_info()->fpsaved
[0] & flag
) {
451 first
= *(u64
*)&f
->regs
[freg
];
452 second
= *(u64
*)&f
->regs
[freg
+2];
464 /* Need to convert endians */
465 u64 tmp
= __swab64p(&first
);
467 first
= __swab64p(&second
);
472 if (tlb_type
== hypervisor
)
473 sun4v_data_access_exception(regs
, addr
, 0);
475 spitfire_data_access_exception(regs
, 0, addr
);
478 if (put_user (first
>> 32, (u32 __user
*)addr
) ||
479 __put_user ((u32
)first
, (u32 __user
*)(addr
+ 4)) ||
480 __put_user (second
>> 32, (u32 __user
*)(addr
+ 8)) ||
481 __put_user ((u32
)second
, (u32 __user
*)(addr
+ 12))) {
482 if (tlb_type
== hypervisor
)
483 sun4v_data_access_exception(regs
, addr
, 0);
485 spitfire_data_access_exception(regs
, 0, addr
);
489 /* LDF, LDDF, LDQF */
490 u32 data
[4] __attribute__ ((aligned(8)));
497 } else if (asi
> ASI_SNFL
) {
498 if (tlb_type
== hypervisor
)
499 sun4v_data_access_exception(regs
, addr
, 0);
501 spitfire_data_access_exception(regs
, 0, addr
);
504 switch (insn
& 0x180000) {
505 case 0x000000: size
= 1; break;
506 case 0x100000: size
= 4; break;
507 default: size
= 2; break;
509 for (i
= 0; i
< size
; i
++)
512 err
= get_user (data
[0], (u32 __user
*) addr
);
514 for (i
= 1; i
< size
; i
++)
515 err
|= __get_user (data
[i
], (u32 __user
*)(addr
+ 4*i
));
517 if (err
&& !(asi
& 0x2 /* NF */)) {
518 if (tlb_type
== hypervisor
)
519 sun4v_data_access_exception(regs
, addr
, 0);
521 spitfire_data_access_exception(regs
, 0, addr
);
524 if (asi
& 0x8) /* Little */ {
528 case 1: data
[0] = le32_to_cpup(data
+ 0); break;
529 default:*(u64
*)(data
+ 0) = le64_to_cpup((u64
*)(data
+ 0));
531 case 4: tmp
= le64_to_cpup((u64
*)(data
+ 0));
532 *(u64
*)(data
+ 0) = le64_to_cpup((u64
*)(data
+ 2));
533 *(u64
*)(data
+ 2) = tmp
;
537 if (!(current_thread_info()->fpsaved
[0] & FPRS_FEF
)) {
538 current_thread_info()->fpsaved
[0] = FPRS_FEF
;
539 current_thread_info()->gsr
[0] = 0;
541 if (!(current_thread_info()->fpsaved
[0] & flag
)) {
543 memset(f
->regs
, 0, 32*sizeof(u32
));
545 memset(f
->regs
+32, 0, 32*sizeof(u32
));
547 memcpy(f
->regs
+ freg
, data
, size
* 4);
548 current_thread_info()->fpsaved
[0] |= flag
;
554 void handle_ld_nf(u32 insn
, struct pt_regs
*regs
)
556 int rd
= ((insn
>> 25) & 0x1f);
557 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
560 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, 0, regs
, 0);
562 maybe_flush_windows(0, 0, rd
, from_kernel
);
563 reg
= fetch_reg_addr(rd
, regs
);
564 if (from_kernel
|| rd
< 16) {
566 if ((insn
& 0x780000) == 0x180000)
568 } else if (test_thread_flag(TIF_32BIT
)) {
569 put_user(0, (int __user
*) reg
);
570 if ((insn
& 0x780000) == 0x180000)
571 put_user(0, ((int __user
*) reg
) + 1);
573 put_user(0, (unsigned long __user
*) reg
);
574 if ((insn
& 0x780000) == 0x180000)
575 put_user(0, (unsigned long __user
*) reg
+ 1);
580 void handle_lddfmna(struct pt_regs
*regs
, unsigned long sfar
, unsigned long sfsr
)
582 unsigned long pc
= regs
->tpc
;
583 unsigned long tstate
= regs
->tstate
;
588 struct fpustate
*f
= FPUSTATE
;
590 if (tstate
& TSTATE_PRIV
)
591 die_if_kernel("lddfmna from kernel", regs
);
592 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, 0, regs
, sfar
);
593 if (test_thread_flag(TIF_32BIT
))
595 if (get_user(insn
, (u32 __user
*) pc
) != -EFAULT
) {
596 int asi
= decode_asi(insn
, regs
);
600 if ((asi
> ASI_SNFL
) ||
604 err
= get_user(first
, (u32 __user
*)sfar
);
606 err
= get_user(second
, (u32 __user
*)(sfar
+ 4));
612 save_and_clear_fpu();
613 freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
614 value
= (((u64
)first
) << 32) | second
;
615 if (asi
& 0x8) /* Little */
616 value
= __swab64p(&value
);
617 flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
618 if (!(current_thread_info()->fpsaved
[0] & FPRS_FEF
)) {
619 current_thread_info()->fpsaved
[0] = FPRS_FEF
;
620 current_thread_info()->gsr
[0] = 0;
622 if (!(current_thread_info()->fpsaved
[0] & flag
)) {
624 memset(f
->regs
, 0, 32*sizeof(u32
));
626 memset(f
->regs
+32, 0, 32*sizeof(u32
));
628 *(u64
*)(f
->regs
+ freg
) = value
;
629 current_thread_info()->fpsaved
[0] |= flag
;
632 if (tlb_type
== hypervisor
)
633 sun4v_data_access_exception(regs
, sfar
, sfsr
);
635 spitfire_data_access_exception(regs
, sfsr
, sfar
);
642 void handle_stdfmna(struct pt_regs
*regs
, unsigned long sfar
, unsigned long sfsr
)
644 unsigned long pc
= regs
->tpc
;
645 unsigned long tstate
= regs
->tstate
;
650 struct fpustate
*f
= FPUSTATE
;
652 if (tstate
& TSTATE_PRIV
)
653 die_if_kernel("stdfmna from kernel", regs
);
654 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, 0, regs
, sfar
);
655 if (test_thread_flag(TIF_32BIT
))
657 if (get_user(insn
, (u32 __user
*) pc
) != -EFAULT
) {
658 int asi
= decode_asi(insn
, regs
);
659 freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
661 flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
662 if ((asi
> ASI_SNFL
) ||
665 save_and_clear_fpu();
666 if (current_thread_info()->fpsaved
[0] & flag
)
667 value
= *(u64
*)&f
->regs
[freg
];
673 value
= __swab64p(&value
); break;
676 if (put_user (value
>> 32, (u32 __user
*) sfar
) ||
677 __put_user ((u32
)value
, (u32 __user
*)(sfar
+ 4)))
681 if (tlb_type
== hypervisor
)
682 sun4v_data_access_exception(regs
, sfar
, sfsr
);
684 spitfire_data_access_exception(regs
, sfsr
, sfar
);