2 * Architecture-specific unaligned trap handling.
4 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
5 * Stephane Eranian <eranian@hpl.hp.com>
6 * David Mosberger-Tang <davidm@hpl.hp.com>
8 * 2002/12/09 Fix rotating register handling (off-by-1 error, missing fr-rotation). Fix
9 * get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
10 * stacked register returns an undefined value; it does NOT trigger a
11 * "rsvd register fault").
12 * 2001/10/11 Fix unaligned access to rotating registers in s/w pipelined loops.
13 * 2001/08/13 Correct size of extended floats (float_fsz) from 16 to 10 bytes.
14 * 2001/01/17 Add support emulation of unaligned kernel accesses.
16 #include <linux/kernel.h>
17 #include <linux/sched.h>
18 #include <linux/tty.h>
20 #include <asm/intrinsics.h>
21 #include <asm/processor.h>
23 #include <asm/uaccess.h>
24 #include <asm/unaligned.h>
26 extern void die_if_kernel(char *str
, struct pt_regs
*regs
, long err
);
28 #undef DEBUG_UNALIGNED_TRAP
30 #ifdef DEBUG_UNALIGNED_TRAP
31 # define DPRINT(a...) do { printk("%s %u: ", __FUNCTION__, __LINE__); printk (a); } while (0)
32 # define DDUMP(str,vp,len) dump(str, vp, len)
35 dump (const char *str
, void *vp
, size_t len
)
37 unsigned char *cp
= vp
;
41 for (i
= 0; i
< len
; ++i
)
42 printk (" %02x", *cp
++);
47 # define DDUMP(str,vp,len)
50 #define IA64_FIRST_STACKED_GR 32
51 #define IA64_FIRST_ROTATING_FR 32
52 #define SIGN_EXT9 0xffffffffffffff00ul
55 * sysctl settable hook which tells the kernel whether to honor the
56 * IA64_THREAD_UAC_NOPRINT prctl. Because this is user settable, we want
57 * to allow the super user to enable/disable this for security reasons
58 * (i.e. don't allow attacker to fill up logs with unaligned accesses).
60 int no_unaligned_warning
;
61 static int noprint_warning
;
67 * --------|------|---------|
68 * [40-37] | [36] | [35:30] |
69 * --------|------|---------|
70 * 4 | 1 | 6 | = 11 bits
71 * --------------------------
72 * However bits [31:30] are not directly useful to distinguish between
73 * load/store so we can use [35:32] instead, which gives the following
74 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
75 * checking the m-bit until later in the load/store emulation.
77 #define IA64_OPCODE_MASK 0x1ef
78 #define IA64_OPCODE_SHIFT 32
81 * Table C-28 Integer Load/Store
83 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
85 * ld8.fill, st8.fill MUST be aligned because the RNATs are based on
86 * the address (bits [8:3]), so we must failed.
92 #define LDBIAS_OP 0x084
93 #define LDACQ_OP 0x085
94 /* 0x086, 0x087 are not relevant */
95 #define LDCCLR_OP 0x088
96 #define LDCNC_OP 0x089
97 #define LDCCLRACQ_OP 0x08a
99 #define STREL_OP 0x08d
100 /* 0x08e,0x8f are not relevant */
103 * Table C-29 Integer Load +Reg
105 * we use the ld->m (bit [36:36]) field to determine whether or not we have
106 * a load/store of this form.
110 * Table C-30 Integer Load/Store +Imm
112 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
114 * ld8.fill, st8.fill must be aligned because the Nat register are based on
115 * the address, so we must fail and the program must be fixed.
117 #define LD_IMM_OP 0x0a0
118 #define LDS_IMM_OP 0x0a1
119 #define LDA_IMM_OP 0x0a2
120 #define LDSA_IMM_OP 0x0a3
121 #define LDBIAS_IMM_OP 0x0a4
122 #define LDACQ_IMM_OP 0x0a5
123 /* 0x0a6, 0xa7 are not relevant */
124 #define LDCCLR_IMM_OP 0x0a8
125 #define LDCNC_IMM_OP 0x0a9
126 #define LDCCLRACQ_IMM_OP 0x0aa
127 #define ST_IMM_OP 0x0ac
128 #define STREL_IMM_OP 0x0ad
129 /* 0x0ae,0xaf are not relevant */
132 * Table C-32 Floating-point Load/Store
135 #define LDFS_OP 0x0c1
136 #define LDFA_OP 0x0c2
137 #define LDFSA_OP 0x0c3
138 /* 0x0c6 is irrelevant */
139 #define LDFCCLR_OP 0x0c8
140 #define LDFCNC_OP 0x0c9
141 /* 0x0cb is irrelevant */
145 * Table C-33 Floating-point Load +Reg
147 * we use the ld->m (bit [36:36]) field to determine whether or not we have
148 * a load/store of this form.
152 * Table C-34 Floating-point Load/Store +Imm
154 #define LDF_IMM_OP 0x0e0
155 #define LDFS_IMM_OP 0x0e1
156 #define LDFA_IMM_OP 0x0e2
157 #define LDFSA_IMM_OP 0x0e3
158 /* 0x0e6 is irrelevant */
159 #define LDFCCLR_IMM_OP 0x0e8
160 #define LDFCNC_IMM_OP 0x0e9
161 #define STF_IMM_OP 0x0ec
164 unsigned long qp
:6; /* [0:5] */
165 unsigned long r1
:7; /* [6:12] */
166 unsigned long imm
:7; /* [13:19] */
167 unsigned long r3
:7; /* [20:26] */
168 unsigned long x
:1; /* [27:27] */
169 unsigned long hint
:2; /* [28:29] */
170 unsigned long x6_sz
:2; /* [30:31] */
171 unsigned long x6_op
:4; /* [32:35], x6 = x6_sz|x6_op */
172 unsigned long m
:1; /* [36:36] */
173 unsigned long op
:4; /* [37:40] */
174 unsigned long pad
:23; /* [41:63] */
179 UPD_IMMEDIATE
, /* ldXZ r1=[r3],imm(9) */
180 UPD_REG
/* ldXZ r1=[r3],r2 */
184 * We use tables to keep track of the offsets of registers in the saved state.
185 * This way we save having big switch/case statements.
187 * We use bit 0 to indicate switch_stack or pt_regs.
188 * The offset is simply shifted by 1 bit.
189 * A 2-byte value should be enough to hold any kind of offset
191 * In case the calling convention changes (and thus pt_regs/switch_stack)
192 * simply use RSW instead of RPT or vice-versa.
195 #define RPO(x) ((size_t) &((struct pt_regs *)0)->x)
196 #define RSO(x) ((size_t) &((struct switch_stack *)0)->x)
198 #define RPT(x) (RPO(x) << 1)
199 #define RSW(x) (1| RSO(x)<<1)
201 #define GR_OFFS(x) (gr_info[x]>>1)
202 #define GR_IN_SW(x) (gr_info[x] & 0x1)
204 #define FR_OFFS(x) (fr_info[x]>>1)
205 #define FR_IN_SW(x) (fr_info[x] & 0x1)
207 static u16 gr_info
[32]={
208 0, /* r0 is read-only : WE SHOULD NEVER GET THIS */
210 RPT(r1
), RPT(r2
), RPT(r3
),
212 RSW(r4
), RSW(r5
), RSW(r6
), RSW(r7
),
214 RPT(r8
), RPT(r9
), RPT(r10
), RPT(r11
),
215 RPT(r12
), RPT(r13
), RPT(r14
), RPT(r15
),
217 RPT(r16
), RPT(r17
), RPT(r18
), RPT(r19
),
218 RPT(r20
), RPT(r21
), RPT(r22
), RPT(r23
),
219 RPT(r24
), RPT(r25
), RPT(r26
), RPT(r27
),
220 RPT(r28
), RPT(r29
), RPT(r30
), RPT(r31
)
223 static u16 fr_info
[32]={
224 0, /* constant : WE SHOULD NEVER GET THIS */
225 0, /* constant : WE SHOULD NEVER GET THIS */
227 RSW(f2
), RSW(f3
), RSW(f4
), RSW(f5
),
229 RPT(f6
), RPT(f7
), RPT(f8
), RPT(f9
),
232 RSW(f12
), RSW(f13
), RSW(f14
),
233 RSW(f15
), RSW(f16
), RSW(f17
), RSW(f18
), RSW(f19
),
234 RSW(f20
), RSW(f21
), RSW(f22
), RSW(f23
), RSW(f24
),
235 RSW(f25
), RSW(f26
), RSW(f27
), RSW(f28
), RSW(f29
),
239 /* Invalidate ALAT entry for integer register REGNO. */
241 invala_gr (int regno
)
243 # define F(reg) case reg: ia64_invala_gr(reg); break
246 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7);
247 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
248 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
249 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
250 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
251 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
252 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
253 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
254 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
255 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
256 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
257 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
258 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
259 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
260 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
261 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
266 /* Invalidate ALAT entry for floating-point register REGNO. */
268 invala_fr (int regno
)
270 # define F(reg) case reg: ia64_invala_fr(reg); break
273 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7);
274 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
275 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
276 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
277 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
278 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
279 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
280 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
281 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
282 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
283 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
284 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
285 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
286 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
287 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
288 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
293 static inline unsigned long
294 rotate_reg (unsigned long sor
, unsigned long rrb
, unsigned long reg
)
303 set_rse_reg (struct pt_regs
*regs
, unsigned long r1
, unsigned long val
, int nat
)
305 struct switch_stack
*sw
= (struct switch_stack
*) regs
- 1;
306 unsigned long *bsp
, *bspstore
, *addr
, *rnat_addr
, *ubs_end
;
307 unsigned long *kbs
= (void *) current
+ IA64_RBS_OFFSET
;
308 unsigned long rnats
, nat_mask
;
309 unsigned long on_kbs
;
310 long sof
= (regs
->cr_ifs
) & 0x7f;
311 long sor
= 8 * ((regs
->cr_ifs
>> 14) & 0xf);
312 long rrb_gr
= (regs
->cr_ifs
>> 18) & 0x7f;
316 /* this should never happen, as the "rsvd register fault" has higher priority */
317 DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1
, sof
);
322 ridx
= rotate_reg(sor
, rrb_gr
, ridx
);
324 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
325 r1
, sw
->ar_bspstore
, regs
->ar_bspstore
, sof
, (regs
->cr_ifs
>> 7) & 0x7f, ridx
);
327 on_kbs
= ia64_rse_num_regs(kbs
, (unsigned long *) sw
->ar_bspstore
);
328 addr
= ia64_rse_skip_regs((unsigned long *) sw
->ar_bspstore
, -sof
+ ridx
);
330 /* the register is on the kernel backing store: easy... */
331 rnat_addr
= ia64_rse_rnat_addr(addr
);
332 if ((unsigned long) rnat_addr
>= sw
->ar_bspstore
)
333 rnat_addr
= &sw
->ar_rnat
;
334 nat_mask
= 1UL << ia64_rse_slot_num(addr
);
338 *rnat_addr
|= nat_mask
;
340 *rnat_addr
&= ~nat_mask
;
344 if (!user_stack(current
, regs
)) {
345 DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1
);
349 bspstore
= (unsigned long *)regs
->ar_bspstore
;
350 ubs_end
= ia64_rse_skip_regs(bspstore
, on_kbs
);
351 bsp
= ia64_rse_skip_regs(ubs_end
, -sof
);
352 addr
= ia64_rse_skip_regs(bsp
, ridx
);
354 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end
, (void *) bsp
, (void *) addr
);
356 ia64_poke(current
, sw
, (unsigned long) ubs_end
, (unsigned long) addr
, val
);
358 rnat_addr
= ia64_rse_rnat_addr(addr
);
360 ia64_peek(current
, sw
, (unsigned long) ubs_end
, (unsigned long) rnat_addr
, &rnats
);
361 DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
362 (void *) rnat_addr
, rnats
, nat
, (rnats
>> ia64_rse_slot_num(addr
)) & 1);
364 nat_mask
= 1UL << ia64_rse_slot_num(addr
);
369 ia64_poke(current
, sw
, (unsigned long) ubs_end
, (unsigned long) rnat_addr
, rnats
);
371 DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr
, rnats
);
376 get_rse_reg (struct pt_regs
*regs
, unsigned long r1
, unsigned long *val
, int *nat
)
378 struct switch_stack
*sw
= (struct switch_stack
*) regs
- 1;
379 unsigned long *bsp
, *addr
, *rnat_addr
, *ubs_end
, *bspstore
;
380 unsigned long *kbs
= (void *) current
+ IA64_RBS_OFFSET
;
381 unsigned long rnats
, nat_mask
;
382 unsigned long on_kbs
;
383 long sof
= (regs
->cr_ifs
) & 0x7f;
384 long sor
= 8 * ((regs
->cr_ifs
>> 14) & 0xf);
385 long rrb_gr
= (regs
->cr_ifs
>> 18) & 0x7f;
389 /* read of out-of-frame register returns an undefined value; 0 in our case. */
390 DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1
, sof
);
395 ridx
= rotate_reg(sor
, rrb_gr
, ridx
);
397 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
398 r1
, sw
->ar_bspstore
, regs
->ar_bspstore
, sof
, (regs
->cr_ifs
>> 7) & 0x7f, ridx
);
400 on_kbs
= ia64_rse_num_regs(kbs
, (unsigned long *) sw
->ar_bspstore
);
401 addr
= ia64_rse_skip_regs((unsigned long *) sw
->ar_bspstore
, -sof
+ ridx
);
403 /* the register is on the kernel backing store: easy... */
406 rnat_addr
= ia64_rse_rnat_addr(addr
);
407 if ((unsigned long) rnat_addr
>= sw
->ar_bspstore
)
408 rnat_addr
= &sw
->ar_rnat
;
409 nat_mask
= 1UL << ia64_rse_slot_num(addr
);
410 *nat
= (*rnat_addr
& nat_mask
) != 0;
415 if (!user_stack(current
, regs
)) {
416 DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1
);
420 bspstore
= (unsigned long *)regs
->ar_bspstore
;
421 ubs_end
= ia64_rse_skip_regs(bspstore
, on_kbs
);
422 bsp
= ia64_rse_skip_regs(ubs_end
, -sof
);
423 addr
= ia64_rse_skip_regs(bsp
, ridx
);
425 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end
, (void *) bsp
, (void *) addr
);
427 ia64_peek(current
, sw
, (unsigned long) ubs_end
, (unsigned long) addr
, val
);
430 rnat_addr
= ia64_rse_rnat_addr(addr
);
431 nat_mask
= 1UL << ia64_rse_slot_num(addr
);
433 DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr
, rnats
);
435 ia64_peek(current
, sw
, (unsigned long) ubs_end
, (unsigned long) rnat_addr
, &rnats
);
436 *nat
= (rnats
& nat_mask
) != 0;
449 setreg (unsigned long regnum
, unsigned long val
, int nat
, struct pt_regs
*regs
)
451 struct switch_stack
*sw
= (struct switch_stack
*) regs
- 1;
453 unsigned long bitmask
;
457 * First takes care of stacked registers
459 if (regnum
>= IA64_FIRST_STACKED_GR
) {
460 set_rse_reg(regs
, regnum
, val
, nat
);
465 * Using r0 as a target raises a General Exception fault which has higher priority
466 * than the Unaligned Reference fault.
470 * Now look at registers in [0-31] range and init correct UNAT
472 if (GR_IN_SW(regnum
)) {
473 addr
= (unsigned long)sw
;
476 addr
= (unsigned long)regs
;
477 unat
= &sw
->caller_unat
;
479 DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
480 addr
, unat
==&sw
->ar_unat
? "yes":"no", GR_OFFS(regnum
));
482 * add offset from base of struct
485 addr
+= GR_OFFS(regnum
);
487 *(unsigned long *)addr
= val
;
490 * We need to clear the corresponding UNAT bit to fully emulate the load
491 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
493 bitmask
= 1UL << (addr
>> 3 & 0x3f);
494 DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr
, val
, nat
, (void *) unat
, *unat
);
500 DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr
, val
, nat
, (void *) unat
,*unat
);
504 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
505 * range from 32-127, result is in the range from 0-95.
507 static inline unsigned long
508 fph_index (struct pt_regs
*regs
, long regnum
)
510 unsigned long rrb_fr
= (regs
->cr_ifs
>> 25) & 0x7f;
511 return rotate_reg(96, rrb_fr
, (regnum
- IA64_FIRST_ROTATING_FR
));
515 setfpreg (unsigned long regnum
, struct ia64_fpreg
*fpval
, struct pt_regs
*regs
)
517 struct switch_stack
*sw
= (struct switch_stack
*)regs
- 1;
521 * From EAS-2.5: FPDisableFault has higher priority than Unaligned
522 * Fault. Thus, when we get here, we know the partition is enabled.
523 * To update f32-f127, there are three choices:
525 * (1) save f32-f127 to thread.fph and update the values there
526 * (2) use a gigantic switch statement to directly access the registers
527 * (3) generate code on the fly to update the desired register
529 * For now, we are using approach (1).
531 if (regnum
>= IA64_FIRST_ROTATING_FR
) {
532 ia64_sync_fph(current
);
533 current
->thread
.fph
[fph_index(regs
, regnum
)] = *fpval
;
536 * pt_regs or switch_stack ?
538 if (FR_IN_SW(regnum
)) {
539 addr
= (unsigned long)sw
;
541 addr
= (unsigned long)regs
;
544 DPRINT("tmp_base=%lx offset=%d\n", addr
, FR_OFFS(regnum
));
546 addr
+= FR_OFFS(regnum
);
547 *(struct ia64_fpreg
*)addr
= *fpval
;
550 * mark the low partition as being used now
552 * It is highly unlikely that this bit is not already set, but
553 * let's do it for safety.
555 regs
->cr_ipsr
|= IA64_PSR_MFL
;
560 * Those 2 inline functions generate the spilled versions of the constant floating point
561 * registers which can be used with stfX
564 float_spill_f0 (struct ia64_fpreg
*final
)
566 ia64_stf_spill(final
, 0);
570 float_spill_f1 (struct ia64_fpreg
*final
)
572 ia64_stf_spill(final
, 1);
576 getfpreg (unsigned long regnum
, struct ia64_fpreg
*fpval
, struct pt_regs
*regs
)
578 struct switch_stack
*sw
= (struct switch_stack
*) regs
- 1;
582 * From EAS-2.5: FPDisableFault has higher priority than
583 * Unaligned Fault. Thus, when we get here, we know the partition is
586 * When regnum > 31, the register is still live and we need to force a save
587 * to current->thread.fph to get access to it. See discussion in setfpreg()
588 * for reasons and other ways of doing this.
590 if (regnum
>= IA64_FIRST_ROTATING_FR
) {
591 ia64_flush_fph(current
);
592 *fpval
= current
->thread
.fph
[fph_index(regs
, regnum
)];
595 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
596 * not saved, we must generate their spilled form on the fly
600 float_spill_f0(fpval
);
603 float_spill_f1(fpval
);
607 * pt_regs or switch_stack ?
609 addr
= FR_IN_SW(regnum
) ? (unsigned long)sw
610 : (unsigned long)regs
;
612 DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
613 FR_IN_SW(regnum
), addr
, FR_OFFS(regnum
));
615 addr
+= FR_OFFS(regnum
);
616 *fpval
= *(struct ia64_fpreg
*)addr
;
623 getreg (unsigned long regnum
, unsigned long *val
, int *nat
, struct pt_regs
*regs
)
625 struct switch_stack
*sw
= (struct switch_stack
*) regs
- 1;
626 unsigned long addr
, *unat
;
628 if (regnum
>= IA64_FIRST_STACKED_GR
) {
629 get_rse_reg(regs
, regnum
, val
, nat
);
634 * take care of r0 (read-only always evaluate to 0)
644 * Now look at registers in [0-31] range and init correct UNAT
646 if (GR_IN_SW(regnum
)) {
647 addr
= (unsigned long)sw
;
650 addr
= (unsigned long)regs
;
651 unat
= &sw
->caller_unat
;
654 DPRINT("addr_base=%lx offset=0x%x\n", addr
, GR_OFFS(regnum
));
656 addr
+= GR_OFFS(regnum
);
658 *val
= *(unsigned long *)addr
;
661 * do it only when requested
664 *nat
= (*unat
>> (addr
>> 3 & 0x3f)) & 0x1UL
;
668 emulate_load_updates (update_t type
, load_store_t ld
, struct pt_regs
*regs
, unsigned long ifa
)
672 * Given the way we handle unaligned speculative loads, we should
673 * not get to this point in the code but we keep this sanity check,
676 if (ld
.x6_op
== 1 || ld
.x6_op
== 3) {
677 printk(KERN_ERR
"%s: register update on speculative load, error\n", __FUNCTION__
);
678 die_if_kernel("unaligned reference on speculative load with register update\n",
684 * at this point, we know that the base register to update is valid i.e.,
687 if (type
== UPD_IMMEDIATE
) {
691 * Load +Imm: ldXZ r1=[r3],imm(9)
694 * form imm9: [13:19] contain the first 7 bits
696 imm
= ld
.x
<< 7 | ld
.imm
;
699 * sign extend (1+8bits) if m set
701 if (ld
.m
) imm
|= SIGN_EXT9
;
704 * ifa == r3 and we know that the NaT bit on r3 was clear so
705 * we can directly use ifa.
709 setreg(ld
.r3
, ifa
, 0, regs
);
711 DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld
.x
, ld
.m
, imm
, ifa
);
718 * Load +Reg Opcode: ldXZ r1=[r3],r2
720 * Note: that we update r3 even in the case of ldfX.a
721 * (where the load does not happen)
723 * The way the load algorithm works, we know that r3 does not
724 * have its NaT bit set (would have gotten NaT consumption
725 * before getting the unaligned fault). So we can use ifa
726 * which equals r3 at this point.
729 * The above statement holds ONLY because we know that we
730 * never reach this code when trying to do a ldX.s.
731 * If we ever make it to here on an ldfX.s then
733 getreg(ld
.imm
, &r2
, &nat_r2
, regs
);
738 * propagate Nat r2 -> r3
740 setreg(ld
.r3
, ifa
, nat_r2
, regs
);
742 DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld
.imm
, r2
, ifa
, nat_r2
);
748 emulate_load_int (unsigned long ifa
, load_store_t ld
, struct pt_regs
*regs
)
750 unsigned int len
= 1 << ld
.x6_sz
;
751 unsigned long val
= 0;
754 * r0, as target, doesn't need to be checked because Illegal Instruction
755 * faults have higher priority than unaligned faults.
757 * r0 cannot be found as the base as it would never generate an
758 * unaligned reference.
762 * ldX.a we will emulate load and also invalidate the ALAT entry.
763 * See comment below for explanation on how we handle ldX.a
766 if (len
!= 2 && len
!= 4 && len
!= 8) {
767 DPRINT("unknown size: x6=%d\n", ld
.x6_sz
);
770 /* this assumes little-endian byte-order: */
771 if (copy_from_user(&val
, (void __user
*) ifa
, len
))
773 setreg(ld
.r1
, val
, 0, regs
);
776 * check for updates on any kind of loads
778 if (ld
.op
== 0x5 || ld
.m
)
779 emulate_load_updates(ld
.op
== 0x5 ? UPD_IMMEDIATE
: UPD_REG
, ld
, regs
, ifa
);
782 * handling of various loads (based on EAS2.4):
784 * ldX.acq (ordered load):
785 * - acquire semantics would have been used, so force fence instead.
787 * ldX.c.clr (check load and clear):
788 * - if we get to this handler, it's because the entry was not in the ALAT.
789 * Therefore the operation reverts to a normal load
791 * ldX.c.nc (check load no clear):
792 * - same as previous one
794 * ldX.c.clr.acq (ordered check load and clear):
795 * - same as above for c.clr part. The load needs to have acquire semantics. So
796 * we use the fence semantics which is stronger and thus ensures correctness.
798 * ldX.a (advanced load):
799 * - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
800 * address doesn't match requested size alignment. This means that we would
801 * possibly need more than one load to get the result.
803 * The load part can be handled just like a normal load, however the difficult
804 * part is to get the right thing into the ALAT. The critical piece of information
805 * in the base address of the load & size. To do that, a ld.a must be executed,
806 * clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
807 * if we use the same target register, we will be okay for the check.a instruction.
808 * If we look at the store, basically a stX [r3]=r1 checks the ALAT for any entry
809 * which would overlap within [r3,r3+X] (the size of the load was store in the
810 * ALAT). If such an entry is found the entry is invalidated. But this is not good
811 * enough, take the following example:
815 * Could be emulated by doing:
817 * store to temporary;
819 * store & shift to temporary;
821 * store & shift to temporary;
823 * store & shift to temporary;
826 * So in this case, you would get the right value is r1 but the wrong info in
827 * the ALAT. Notice that you could do it in reverse to finish with address 3
828 * but you would still get the size wrong. To get the size right, one needs to
829 * execute exactly the same kind of load. You could do it from a aligned
830 * temporary location, but you would get the address wrong.
832 * So no matter what, it is not possible to emulate an advanced load
833 * correctly. But is that really critical ?
835 * We will always convert ld.a into a normal load with ALAT invalidated. This
836 * will enable compiler to do optimization where certain code path after ld.a
837 * is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
839 * If there is a store after the advanced load, one must either do a ld.c.* or
840 * chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
841 * entry found in ALAT), and that's perfectly ok because:
843 * - ld.c.*, if the entry is not present a normal load is executed
844 * - chk.a.*, if the entry is not present, execution jumps to recovery code
846 * In either case, the load can be potentially retried in another form.
848 * ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
849 * up a stale entry later). The register base update MUST also be performed.
853 * when the load has the .acq completer then
854 * use ordering fence.
856 if (ld
.x6_op
== 0x5 || ld
.x6_op
== 0xa)
860 * invalidate ALAT entry in case of advanced load
869 emulate_store_int (unsigned long ifa
, load_store_t ld
, struct pt_regs
*regs
)
872 unsigned int len
= 1 << ld
.x6_sz
;
875 * if we get to this handler, Nat bits on both r3 and r2 have already
876 * been checked. so we don't need to do it
878 * extract the value to be stored
880 getreg(ld
.imm
, &r2
, NULL
, regs
);
883 * we rely on the macros in unaligned.h for now i.e.,
884 * we let the compiler figure out how to read memory gracefully.
886 * We need this switch/case because the way the inline function
887 * works. The code is optimized by the compiler and looks like
888 * a single switch/case.
890 DPRINT("st%d [%lx]=%lx\n", len
, ifa
, r2
);
892 if (len
!= 2 && len
!= 4 && len
!= 8) {
893 DPRINT("unknown size: x6=%d\n", ld
.x6_sz
);
897 /* this assumes little-endian byte-order: */
898 if (copy_to_user((void __user
*) ifa
, &r2
, len
))
905 * ld.r3 can never be r0, because r0 would not generate an
912 * form imm9: [12:6] contain first 7bits
914 imm
= ld
.x
<< 7 | ld
.r1
;
916 * sign extend (8bits) if m set
918 if (ld
.m
) imm
|= SIGN_EXT9
;
920 * ifa == r3 (NaT is necessarily cleared)
924 DPRINT("imm=%lx r3=%lx\n", imm
, ifa
);
926 setreg(ld
.r3
, ifa
, 0, regs
);
929 * we don't have alat_invalidate_multiple() so we need
930 * to do the complete flush :-<<
935 * stX.rel: use fence instead of release
944 * floating point operations sizes in bytes
946 static const unsigned char float_fsz
[4]={
947 10, /* extended precision (e) */
949 4, /* single precision (s) */
950 8 /* double precision (d) */
954 mem2float_extended (struct ia64_fpreg
*init
, struct ia64_fpreg
*final
)
958 ia64_stf_spill(final
, 6);
962 mem2float_integer (struct ia64_fpreg
*init
, struct ia64_fpreg
*final
)
966 ia64_stf_spill(final
, 6);
970 mem2float_single (struct ia64_fpreg
*init
, struct ia64_fpreg
*final
)
974 ia64_stf_spill(final
, 6);
978 mem2float_double (struct ia64_fpreg
*init
, struct ia64_fpreg
*final
)
982 ia64_stf_spill(final
, 6);
986 float2mem_extended (struct ia64_fpreg
*init
, struct ia64_fpreg
*final
)
988 ia64_ldf_fill(6, init
);
994 float2mem_integer (struct ia64_fpreg
*init
, struct ia64_fpreg
*final
)
996 ia64_ldf_fill(6, init
);
1002 float2mem_single (struct ia64_fpreg
*init
, struct ia64_fpreg
*final
)
1004 ia64_ldf_fill(6, init
);
1006 ia64_stfs(final
, 6);
1010 float2mem_double (struct ia64_fpreg
*init
, struct ia64_fpreg
*final
)
1012 ia64_ldf_fill(6, init
);
1014 ia64_stfd(final
, 6);
1018 emulate_load_floatpair (unsigned long ifa
, load_store_t ld
, struct pt_regs
*regs
)
1020 struct ia64_fpreg fpr_init
[2];
1021 struct ia64_fpreg fpr_final
[2];
1022 unsigned long len
= float_fsz
[ld
.x6_sz
];
1025 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1026 * higher priority than unaligned faults.
1028 * r0 cannot be found as the base as it would never generate an unaligned
1033 * make sure we get clean buffers
1035 memset(&fpr_init
, 0, sizeof(fpr_init
));
1036 memset(&fpr_final
, 0, sizeof(fpr_final
));
1039 * ldfpX.a: we don't try to emulate anything but we must
1040 * invalidate the ALAT entry and execute updates, if any.
1042 if (ld
.x6_op
!= 0x2) {
1044 * This assumes little-endian byte-order. Note that there is no "ldfpe"
1047 if (copy_from_user(&fpr_init
[0], (void __user
*) ifa
, len
)
1048 || copy_from_user(&fpr_init
[1], (void __user
*) (ifa
+ len
), len
))
1051 DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld
.r1
, ld
.imm
, ld
.x6_sz
);
1052 DDUMP("frp_init =", &fpr_init
, 2*len
);
1055 * Could optimize inlines by using ldfpX & 2 spills
1057 switch( ld
.x6_sz
) {
1059 mem2float_extended(&fpr_init
[0], &fpr_final
[0]);
1060 mem2float_extended(&fpr_init
[1], &fpr_final
[1]);
1063 mem2float_integer(&fpr_init
[0], &fpr_final
[0]);
1064 mem2float_integer(&fpr_init
[1], &fpr_final
[1]);
1067 mem2float_single(&fpr_init
[0], &fpr_final
[0]);
1068 mem2float_single(&fpr_init
[1], &fpr_final
[1]);
1071 mem2float_double(&fpr_init
[0], &fpr_final
[0]);
1072 mem2float_double(&fpr_init
[1], &fpr_final
[1]);
1075 DDUMP("fpr_final =", &fpr_final
, 2*len
);
1079 * A possible optimization would be to drop fpr_final and directly
1080 * use the storage from the saved context i.e., the actual final
1081 * destination (pt_regs, switch_stack or thread structure).
1083 setfpreg(ld
.r1
, &fpr_final
[0], regs
);
1084 setfpreg(ld
.imm
, &fpr_final
[1], regs
);
1088 * Check for updates: only immediate updates are available for this
1093 * the immediate is implicit given the ldsz of the operation:
1094 * single: 8 (2x4) and for all others it's 16 (2x8)
1100 * the fact that we force the NaT of r3 to zero is ONLY valid
1101 * as long as we don't come here with a ldfpX.s.
1102 * For this reason we keep this sanity check
1104 if (ld
.x6_op
== 1 || ld
.x6_op
== 3)
1105 printk(KERN_ERR
"%s: register update on speculative load pair, error\n",
1108 setreg(ld
.r3
, ifa
, 0, regs
);
1112 * Invalidate ALAT entries, if any, for both registers.
1114 if (ld
.x6_op
== 0x2) {
1123 emulate_load_float (unsigned long ifa
, load_store_t ld
, struct pt_regs
*regs
)
1125 struct ia64_fpreg fpr_init
;
1126 struct ia64_fpreg fpr_final
;
1127 unsigned long len
= float_fsz
[ld
.x6_sz
];
1130 * fr0 & fr1 don't need to be checked because Illegal Instruction
1131 * faults have higher priority than unaligned faults.
1133 * r0 cannot be found as the base as it would never generate an
1134 * unaligned reference.
1138 * make sure we get clean buffers
1140 memset(&fpr_init
,0, sizeof(fpr_init
));
1141 memset(&fpr_final
,0, sizeof(fpr_final
));
1144 * ldfX.a we don't try to emulate anything but we must
1145 * invalidate the ALAT entry.
1146 * See comments in ldX for descriptions on how the various loads are handled.
1148 if (ld
.x6_op
!= 0x2) {
1149 if (copy_from_user(&fpr_init
, (void __user
*) ifa
, len
))
1152 DPRINT("ld.r1=%d x6_sz=%d\n", ld
.r1
, ld
.x6_sz
);
1153 DDUMP("fpr_init =", &fpr_init
, len
);
1155 * we only do something for x6_op={0,8,9}
1157 switch( ld
.x6_sz
) {
1159 mem2float_extended(&fpr_init
, &fpr_final
);
1162 mem2float_integer(&fpr_init
, &fpr_final
);
1165 mem2float_single(&fpr_init
, &fpr_final
);
1168 mem2float_double(&fpr_init
, &fpr_final
);
1171 DDUMP("fpr_final =", &fpr_final
, len
);
1175 * A possible optimization would be to drop fpr_final and directly
1176 * use the storage from the saved context i.e., the actual final
1177 * destination (pt_regs, switch_stack or thread structure).
1179 setfpreg(ld
.r1
, &fpr_final
, regs
);
1183 * check for updates on any loads
1185 if (ld
.op
== 0x7 || ld
.m
)
1186 emulate_load_updates(ld
.op
== 0x7 ? UPD_IMMEDIATE
: UPD_REG
, ld
, regs
, ifa
);
1189 * invalidate ALAT entry in case of advanced floating point loads
1191 if (ld
.x6_op
== 0x2)
1199 emulate_store_float (unsigned long ifa
, load_store_t ld
, struct pt_regs
*regs
)
1201 struct ia64_fpreg fpr_init
;
1202 struct ia64_fpreg fpr_final
;
1203 unsigned long len
= float_fsz
[ld
.x6_sz
];
1206 * make sure we get clean buffers
1208 memset(&fpr_init
,0, sizeof(fpr_init
));
1209 memset(&fpr_final
,0, sizeof(fpr_final
));
1212 * if we get to this handler, Nat bits on both r3 and r2 have already
1213 * been checked. so we don't need to do it
1215 * extract the value to be stored
1217 getfpreg(ld
.imm
, &fpr_init
, regs
);
1219 * during this step, we extract the spilled registers from the saved
1220 * context i.e., we refill. Then we store (no spill) to temporary
1223 switch( ld
.x6_sz
) {
1225 float2mem_extended(&fpr_init
, &fpr_final
);
1228 float2mem_integer(&fpr_init
, &fpr_final
);
1231 float2mem_single(&fpr_init
, &fpr_final
);
1234 float2mem_double(&fpr_init
, &fpr_final
);
1237 DPRINT("ld.r1=%d x6_sz=%d\n", ld
.r1
, ld
.x6_sz
);
1238 DDUMP("fpr_init =", &fpr_init
, len
);
1239 DDUMP("fpr_final =", &fpr_final
, len
);
1241 if (copy_to_user((void __user
*) ifa
, &fpr_final
, len
))
1245 * stfX [r3]=r2,imm(9)
1248 * ld.r3 can never be r0, because r0 would not generate an
1255 * form imm9: [12:6] contain first 7bits
1257 imm
= ld
.x
<< 7 | ld
.r1
;
1259 * sign extend (8bits) if m set
1264 * ifa == r3 (NaT is necessarily cleared)
1268 DPRINT("imm=%lx r3=%lx\n", imm
, ifa
);
1270 setreg(ld
.r3
, ifa
, 0, regs
);
1273 * we don't have alat_invalidate_multiple() so we need
1274 * to do the complete flush :-<<
1282 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1283 * eventually fix the program. However, we don't want to do that for every access so we
1284 * pace it with jiffies. This isn't really MP-safe, but it doesn't really have to be
1288 within_logging_rate_limit (void)
1290 static unsigned long count
, last_time
;
1292 if (jiffies
- last_time
> 5*HZ
)
1295 last_time
= jiffies
;
1304 ia64_handle_unaligned (unsigned long ifa
, struct pt_regs
*regs
)
1306 struct ia64_psr
*ipsr
= ia64_psr(regs
);
1307 mm_segment_t old_fs
= get_fs();
1308 unsigned long bundle
[2];
1309 unsigned long opcode
;
1311 const struct exception_table_entry
*eh
= NULL
;
1318 if (ia64_psr(regs
)->be
) {
1319 /* we don't support big-endian accesses */
1320 die_if_kernel("big-endian unaligned accesses are not supported", regs
, 0);
1325 * Treat kernel accesses for which there is an exception handler entry the same as
1326 * user-level unaligned accesses. Otherwise, a clever program could trick this
1327 * handler into reading an arbitrary kernel addresses...
1329 if (!user_mode(regs
))
1330 eh
= search_exception_tables(regs
->cr_iip
+ ia64_psr(regs
)->ri
);
1331 if (user_mode(regs
) || eh
) {
1332 if ((current
->thread
.flags
& IA64_THREAD_UAC_SIGBUS
) != 0)
1335 if (!no_unaligned_warning
&&
1336 !(current
->thread
.flags
& IA64_THREAD_UAC_NOPRINT
) &&
1337 within_logging_rate_limit())
1339 char buf
[200]; /* comm[] is at most 16 bytes... */
1342 len
= sprintf(buf
, "%s(%d): unaligned access to 0x%016lx, "
1343 "ip=0x%016lx\n\r", current
->comm
,
1344 task_pid_nr(current
),
1345 ifa
, regs
->cr_iip
+ ipsr
->ri
);
1347 * Don't call tty_write_message() if we're in the kernel; we might
1348 * be holding locks...
1350 if (user_mode(regs
))
1351 tty_write_message(current
->signal
->tty
, buf
);
1352 buf
[len
-1] = '\0'; /* drop '\r' */
1353 /* watch for command names containing %s */
1354 printk(KERN_WARNING
"%s", buf
);
1356 if (no_unaligned_warning
&& !noprint_warning
) {
1357 noprint_warning
= 1;
1358 printk(KERN_WARNING
"%s(%d) encountered an "
1359 "unaligned exception which required\n"
1360 "kernel assistance, which degrades "
1361 "the performance of the application.\n"
1362 "Unaligned exception warnings have "
1363 "been disabled by the system "
1365 "echo 0 > /proc/sys/kernel/ignore-"
1366 "unaligned-usertrap to re-enable\n",
1367 current
->comm
, task_pid_nr(current
));
1371 if (within_logging_rate_limit())
1372 printk(KERN_WARNING
"kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1373 ifa
, regs
->cr_iip
+ ipsr
->ri
);
1377 DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1378 regs
->cr_iip
, ifa
, regs
->cr_ipsr
, ipsr
->ri
, ipsr
->it
);
1380 if (__copy_from_user(bundle
, (void __user
*) regs
->cr_iip
, 16))
1384 * extract the instruction from the bundle given the slot number
1387 case 0: u
.l
= (bundle
[0] >> 5); break;
1388 case 1: u
.l
= (bundle
[0] >> 46) | (bundle
[1] << 18); break;
1389 case 2: u
.l
= (bundle
[1] >> 23); break;
1391 opcode
= (u
.l
>> IA64_OPCODE_SHIFT
) & IA64_OPCODE_MASK
;
1393 DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1394 "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode
, u
.insn
.qp
, u
.insn
.r1
, u
.insn
.imm
,
1395 u
.insn
.r3
, u
.insn
.x
, u
.insn
.hint
, u
.insn
.x6_sz
, u
.insn
.m
, u
.insn
.op
);
1399 * Notice that the switch statement DOES not cover all possible instructions
1400 * that DO generate unaligned references. This is made on purpose because for some
1401 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1402 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1403 * the program will get a signal and die:
1408 * Reason: RNATs are based on addresses
1411 * Reason: ld16 and st16 are supposed to occur in a single
1418 * Reason: ATOMIC operations cannot be emulated properly using multiple
1421 * speculative loads:
1423 * Reason: side effects, code must be ready to deal with failure so simpler
1424 * to let the load fail.
1425 * ---------------------------------------------------------------------------------
1428 * I would like to get rid of this switch case and do something
1435 /* oops, really a semaphore op (cmpxchg, etc) */
1444 * The instruction will be retried with deferred exceptions turned on, and
1445 * we should get Nat bit installed
1447 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1448 * are actually executed even though the operation failed. So we don't
1449 * need to take care of this.
1451 DPRINT("forcing PSR_ED\n");
1452 regs
->cr_ipsr
|= IA64_PSR_ED
;
1463 /* oops, really a semaphore op (cmpxchg, etc) */
1472 case LDCCLRACQ_IMM_OP
:
1473 ret
= emulate_load_int(ifa
, u
.insn
, regs
);
1479 /* oops, really a semaphore op (cmpxchg, etc) */
1484 ret
= emulate_store_int(ifa
, u
.insn
, regs
);
1492 ret
= emulate_load_floatpair(ifa
, u
.insn
, regs
);
1494 ret
= emulate_load_float(ifa
, u
.insn
, regs
);
1499 case LDFCCLR_IMM_OP
:
1501 ret
= emulate_load_float(ifa
, u
.insn
, regs
);
1506 ret
= emulate_store_float(ifa
, u
.insn
, regs
);
1512 DPRINT("ret=%d\n", ret
);
1518 * given today's architecture this case is not likely to happen because a
1519 * memory access instruction (M) can never be in the last slot of a
1520 * bundle. But let's keep it for now.
1523 ipsr
->ri
= (ipsr
->ri
+ 1) & 0x3;
1525 DPRINT("ipsr->ri=%d iip=%lx\n", ipsr
->ri
, regs
->cr_iip
);
1527 set_fs(old_fs
); /* restore original address limit */
1531 /* something went wrong... */
1532 if (!user_mode(regs
)) {
1534 ia64_handle_exception(regs
, eh
);
1537 die_if_kernel("error during unaligned kernel access\n", regs
, ret
);
1541 si
.si_signo
= SIGBUS
;
1543 si
.si_code
= BUS_ADRALN
;
1544 si
.si_addr
= (void __user
*) ifa
;
1548 force_sig_info(SIGBUS
, &si
, current
);