| blob | 43b45b65ee5a9da2c446d712b67ba152f1e4fce0 |
1 /*
2 * Architecture-specific unaligned trap handling.
3 *
4 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
5 * Stephane Eranian <eranian@hpl.hp.com>
6 * David Mosberger-Tang <davidm@hpl.hp.com>
7 *
8 * 2002/12/09 Fix rotating register handling (off-by-1 error, missing fr-rotation). Fix
9 * get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
10 * stacked register returns an undefined value; it does NOT trigger a
11 * "rsvd register fault").
12 * 2001/10/11 Fix unaligned access to rotating registers in s/w pipelined loops.
13 * 2001/08/13 Correct size of extended floats (float_fsz) from 16 to 10 bytes.
14 * 2001/01/17 Add support emulation of unaligned kernel accesses.
15 */
16 #include <linux/kernel.h>
17 #include <linux/sched.h>
18 #include <linux/smp_lock.h>
19 #include <linux/tty.h>
21 #include <asm/intrinsics.h>
22 #include <asm/processor.h>
23 #include <asm/rse.h>
24 #include <asm/uaccess.h>
25 #include <asm/unaligned.h>
29 #undef DEBUG_UNALIGNED_TRAP
31 #ifdef DEBUG_UNALIGNED_TRAP
33 # define DDUMP(str,vp,len) dump(str, vp, len)
35 static void
37 {
45 }
46 #else
47 # define DPRINT(a...)
48 # define DDUMP(str,vp,len)
49 #endif
51 #define IA64_FIRST_STACKED_GR 32
52 #define IA64_FIRST_ROTATING_FR 32
53 #define SIGN_EXT9 0xffffffffffffff00ul
55 /*
56 * For M-unit:
57 *
58 * opcode | m | x6 |
59 * --------|------|---------|
60 * [40-37] | [36] | [35:30] |
61 * --------|------|---------|
62 * 4 | 1 | 6 | = 11 bits
63 * --------------------------
64 * However bits [31:30] are not directly useful to distinguish between
65 * load/store so we can use [35:32] instead, which gives the following
66 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
67 * checking the m-bit until later in the load/store emulation.
68 */
69 #define IA64_OPCODE_MASK 0x1ef
70 #define IA64_OPCODE_SHIFT 32
72 /*
73 * Table C-28 Integer Load/Store
74 *
75 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
76 *
77 * ld8.fill, st8.fill MUST be aligned because the RNATs are based on
78 * the address (bits [8:3]), so we must failed.
79 */
80 #define LD_OP 0x080
81 #define LDS_OP 0x081
82 #define LDA_OP 0x082
83 #define LDSA_OP 0x083
84 #define LDBIAS_OP 0x084
85 #define LDACQ_OP 0x085
86 /* 0x086, 0x087 are not relevant */
87 #define LDCCLR_OP 0x088
88 #define LDCNC_OP 0x089
89 #define LDCCLRACQ_OP 0x08a
90 #define ST_OP 0x08c
91 #define STREL_OP 0x08d
92 /* 0x08e,0x8f are not relevant */
94 /*
95 * Table C-29 Integer Load +Reg
96 *
97 * we use the ld->m (bit [36:36]) field to determine whether or not we have
98 * a load/store of this form.
99 */
101 /*
102 * Table C-30 Integer Load/Store +Imm
103 *
104 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
105 *
106 * ld8.fill, st8.fill must be aligned because the Nat register are based on
107 * the address, so we must fail and the program must be fixed.
108 */
109 #define LD_IMM_OP 0x0a0
110 #define LDS_IMM_OP 0x0a1
111 #define LDA_IMM_OP 0x0a2
112 #define LDSA_IMM_OP 0x0a3
113 #define LDBIAS_IMM_OP 0x0a4
114 #define LDACQ_IMM_OP 0x0a5
115 /* 0x0a6, 0xa7 are not relevant */
116 #define LDCCLR_IMM_OP 0x0a8
117 #define LDCNC_IMM_OP 0x0a9
118 #define LDCCLRACQ_IMM_OP 0x0aa
119 #define ST_IMM_OP 0x0ac
120 #define STREL_IMM_OP 0x0ad
121 /* 0x0ae,0xaf are not relevant */
123 /*
124 * Table C-32 Floating-point Load/Store
125 */
126 #define LDF_OP 0x0c0
127 #define LDFS_OP 0x0c1
128 #define LDFA_OP 0x0c2
129 #define LDFSA_OP 0x0c3
130 /* 0x0c6 is irrelevant */
131 #define LDFCCLR_OP 0x0c8
132 #define LDFCNC_OP 0x0c9
133 /* 0x0cb is irrelevant */
134 #define STF_OP 0x0cc
136 /*
137 * Table C-33 Floating-point Load +Reg
138 *
139 * we use the ld->m (bit [36:36]) field to determine whether or not we have
140 * a load/store of this form.
141 */
143 /*
144 * Table C-34 Floating-point Load/Store +Imm
145 */
146 #define LDF_IMM_OP 0x0e0
147 #define LDFS_IMM_OP 0x0e1
148 #define LDFA_IMM_OP 0x0e2
149 #define LDFSA_IMM_OP 0x0e3
150 /* 0x0e6 is irrelevant */
151 #define LDFCCLR_IMM_OP 0x0e8
152 #define LDFCNC_IMM_OP 0x0e9
153 #define STF_IMM_OP 0x0ec
172 UPD_REG /* ldXZ r1=[r3],r2 */
175 /*
176 * We use tables to keep track of the offsets of registers in the saved state.
177 * This way we save having big switch/case statements.
178 *
179 * We use bit 0 to indicate switch_stack or pt_regs.
180 * The offset is simply shifted by 1 bit.
181 * A 2-byte value should be enough to hold any kind of offset
182 *
183 * In case the calling convention changes (and thus pt_regs/switch_stack)
184 * simply use RSW instead of RPT or vice-versa.
185 */
187 #define RPO(x) ((size_t) &((struct pt_regs *)0)->x)
188 #define RSO(x) ((size_t) &((struct switch_stack *)0)->x)
190 #define RPT(x) (RPO(x) << 1)
191 #define RSW(x) (1| RSO(x)<<1)
193 #define GR_OFFS(x) (gr_info[x]>>1)
194 #define GR_IN_SW(x) (gr_info[x] & 0x1)
196 #define FR_OFFS(x) (fr_info[x]>>1)
197 #define FR_IN_SW(x) (fr_info[x] & 0x1)
213 };
229 };
231 /* Invalidate ALAT entry for integer register REGNO. */
232 static void
234 {
235 # define F(reg) case reg: ia64_invala_gr(reg); break
254 }
255 # undef F
256 }
258 /* Invalidate ALAT entry for floating-point register REGNO. */
259 static void
261 {
262 # define F(reg) case reg: ia64_invala_fr(reg); break
281 }
282 # undef F
283 }
287 {
292 }
294 static void
296 {
308 /* this should never happen, as the "rsvd register fault" has higher priority */
311 }
322 /* the register is on the kernel backing store: easy... */
331 else
334 }
339 }
359 else
364 }
367 static void
369 {
381 /* read of out-of-frame register returns an undefined value; 0 in our case. */
384 }
395 /* the register is on the kernel backing store: easy... */
403 }
405 }
410 }
429 }
432 fail:
437 }
440 static void
442 {
448 /*
449 * First takes care of stacked registers
450 */
454 }
456 /*
457 * Using r0 as a target raises a General Exception fault which has higher priority
458 * than the Unaligned Reference fault.
459 */
461 /*
462 * Now look at registers in [0-31] range and init correct UNAT
463 */
470 }
473 /*
474 * add offset from base of struct
475 * and do it !
476 */
481 /*
482 * We need to clear the corresponding UNAT bit to fully emulate the load
483 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
484 */
491 }
493 }
495 /*
496 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
497 * range from 32-127, result is in the range from 0-95.
498 */
501 {
504 }
506 static void
508 {
512 /*
513 * From EAS-2.5: FPDisableFault has higher priority than Unaligned
514 * Fault. Thus, when we get here, we know the partition is enabled.
515 * To update f32-f127, there are three choices:
516 *
517 * (1) save f32-f127 to thread.fph and update the values there
518 * (2) use a gigantic switch statement to directly access the registers
519 * (3) generate code on the fly to update the desired register
520 *
521 * For now, we are using approach (1).
522 */
527 /*
528 * pt_regs or switch_stack ?
529 */
534 }
541 /*
542 * mark the low partition as being used now
543 *
544 * It is highly unlikely that this bit is not already set, but
545 * let's do it for safety.
546 */
548 }
549 }
551 /*
552 * Those 2 inline functions generate the spilled versions of the constant floating point
553 * registers which can be used with stfX
554 */
557 {
559 }
563 {
565 }
567 static void
569 {
573 /*
574 * From EAS-2.5: FPDisableFault has higher priority than
575 * Unaligned Fault. Thus, when we get here, we know the partition is
576 * enabled.
577 *
578 * When regnum > 31, the register is still live and we need to force a save
579 * to current->thread.fph to get access to it. See discussion in setfpreg()
580 * for reasons and other ways of doing this.
581 */
586 /*
587 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
588 * not saved, we must generate their spilled form on the fly
589 */
598 /*
599 * pt_regs or switch_stack ?
600 */
609 }
610 }
611 }
614 static void
616 {
623 }
625 /*
626 * take care of r0 (read-only always evaluate to 0)
627 */
633 }
635 /*
636 * Now look at registers in [0-31] range and init correct UNAT
637 */
644 }
652 /*
653 * do it only when requested
654 */
657 }
659 static void
661 {
662 /*
663 * IMPORTANT:
664 * Given the way we handle unaligned speculative loads, we should
665 * not get to this point in the code but we keep this sanity check,
666 * just in case.
667 */
672 }
675 /*
676 * at this point, we know that the base register to update is valid i.e.,
677 * it's not r0
678 */
682 /*
683 * Load +Imm: ldXZ r1=[r3],imm(9)
684 *
685 *
686 * form imm9: [13:19] contain the first 7 bits
687 */
690 /*
691 * sign extend (1+8bits) if m set
692 */
695 /*
696 * ifa == r3 and we know that the NaT bit on r3 was clear so
697 * we can directly use ifa.
698 */
709 /*
710 * Load +Reg Opcode: ldXZ r1=[r3],r2
711 *
712 * Note: that we update r3 even in the case of ldfX.a
713 * (where the load does not happen)
714 *
715 * The way the load algorithm works, we know that r3 does not
716 * have its NaT bit set (would have gotten NaT consumption
717 * before getting the unaligned fault). So we can use ifa
718 * which equals r3 at this point.
719 *
720 * IMPORTANT:
721 * The above statement holds ONLY because we know that we
722 * never reach this code when trying to do a ldX.s.
723 * If we ever make it to here on an ldfX.s then
724 */
729 /*
730 * propagate Nat r2 -> r3
731 */
735 }
736 }
739 static int
741 {
745 /*
746 * r0, as target, doesn't need to be checked because Illegal Instruction
747 * faults have higher priority than unaligned faults.
748 *
749 * r0 cannot be found as the base as it would never generate an
750 * unaligned reference.
751 */
753 /*
754 * ldX.a we will emulate load and also invalidate the ALAT entry.
755 * See comment below for explanation on how we handle ldX.a
756 */
761 }
762 /* this assumes little-endian byte-order: */
767 /*
768 * check for updates on any kind of loads
769 */
773 /*
774 * handling of various loads (based on EAS2.4):
775 *
776 * ldX.acq (ordered load):
777 * - acquire semantics would have been used, so force fence instead.
778 *
779 * ldX.c.clr (check load and clear):
780 * - if we get to this handler, it's because the entry was not in the ALAT.
781 * Therefore the operation reverts to a normal load
782 *
783 * ldX.c.nc (check load no clear):
784 * - same as previous one
785 *
786 * ldX.c.clr.acq (ordered check load and clear):
787 * - same as above for c.clr part. The load needs to have acquire semantics. So
788 * we use the fence semantics which is stronger and thus ensures correctness.
789 *
790 * ldX.a (advanced load):
791 * - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
792 * address doesn't match requested size alignment. This means that we would
793 * possibly need more than one load to get the result.
794 *
795 * The load part can be handled just like a normal load, however the difficult
796 * part is to get the right thing into the ALAT. The critical piece of information
797 * in the base address of the load & size. To do that, a ld.a must be executed,
798 * clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
799 * if we use the same target register, we will be okay for the check.a instruction.
800 * If we look at the store, basically a stX [r3]=r1 checks the ALAT for any entry
801 * which would overlap within [r3,r3+X] (the size of the load was store in the
802 * ALAT). If such an entry is found the entry is invalidated. But this is not good
803 * enough, take the following example:
804 * r3=3
805 * ld4.a r1=[r3]
806 *
807 * Could be emulated by doing:
808 * ld1.a r1=[r3],1
809 * store to temporary;
810 * ld1.a r1=[r3],1
811 * store & shift to temporary;
812 * ld1.a r1=[r3],1
813 * store & shift to temporary;
814 * ld1.a r1=[r3]
815 * store & shift to temporary;
816 * r1=temporary
817 *
818 * So in this case, you would get the right value is r1 but the wrong info in
819 * the ALAT. Notice that you could do it in reverse to finish with address 3
820 * but you would still get the size wrong. To get the size right, one needs to
821 * execute exactly the same kind of load. You could do it from a aligned
822 * temporary location, but you would get the address wrong.
823 *
824 * So no matter what, it is not possible to emulate an advanced load
825 * correctly. But is that really critical ?
826 *
827 * We will always convert ld.a into a normal load with ALAT invalidated. This
828 * will enable compiler to do optimization where certain code path after ld.a
829 * is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
830 *
831 * If there is a store after the advanced load, one must either do a ld.c.* or
832 * chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
833 * entry found in ALAT), and that's perfectly ok because:
834 *
835 * - ld.c.*, if the entry is not present a normal load is executed
836 * - chk.a.*, if the entry is not present, execution jumps to recovery code
837 *
838 * In either case, the load can be potentially retried in another form.
839 *
840 * ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
841 * up a stale entry later). The register base update MUST also be performed.
842 */
844 /*
845 * when the load has the .acq completer then
846 * use ordering fence.
847 */
851 /*
852 * invalidate ALAT entry in case of advanced load
853 */
858 }
860 static int
862 {
866 /*
867 * if we get to this handler, Nat bits on both r3 and r2 have already
868 * been checked. so we don't need to do it
869 *
870 * extract the value to be stored
871 */
874 /*
875 * we rely on the macros in unaligned.h for now i.e.,
876 * we let the compiler figure out how to read memory gracefully.
877 *
878 * We need this switch/case because the way the inline function
879 * works. The code is optimized by the compiler and looks like
880 * a single switch/case.
881 */
887 }
889 /* this assumes little-endian byte-order: */
893 /*
894 * stX [r3]=r2,imm(9)
895 *
896 * NOTE:
897 * ld.r3 can never be r0, because r0 would not generate an
898 * unaligned access.
899 */
903 /*
904 * form imm9: [12:6] contain first 7bits
905 */
907 /*
908 * sign extend (8bits) if m set
909 */
911 /*
912 * ifa == r3 (NaT is necessarily cleared)
913 */
919 }
920 /*
921 * we don't have alat_invalidate_multiple() so we need
922 * to do the complete flush :-<<
923 */
926 /*
927 * stX.rel: use fence instead of release
928 */
933 }
935 /*
936 * floating point operations sizes in bytes
937 */
943 };
947 {
951 }
955 {
959 }
963 {
967 }
971 {
975 }
979 {
983 }
987 {
991 }
995 {
999 }
1003 {
1007 }
1009 static int
1011 {
1016 /*
1017 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1018 * higher priority than unaligned faults.
1019 *
1020 * r0 cannot be found as the base as it would never generate an unaligned
1021 * reference.
1022 */
1024 /*
1025 * make sure we get clean buffers
1026 */
1030 /*
1031 * ldfpX.a: we don't try to emulate anything but we must
1032 * invalidate the ALAT entry and execute updates, if any.
1033 */
1035 /*
1036 * This assumes little-endian byte-order. Note that there is no "ldfpe"
1037 * instruction:
1038 */
1045 /*
1046 * XXX fixme
1047 * Could optimize inlines by using ldfpX & 2 spills
1048 */
1066 }
1068 /*
1069 * XXX fixme
1070 *
1071 * A possible optimization would be to drop fpr_final and directly
1072 * use the storage from the saved context i.e., the actual final
1073 * destination (pt_regs, switch_stack or thread structure).
1074 */
1077 }
1079 /*
1080 * Check for updates: only immediate updates are available for this
1081 * instruction.
1082 */
1084 /*
1085 * the immediate is implicit given the ldsz of the operation:
1086 * single: 8 (2x4) and for all others it's 16 (2x8)
1087 */
1090 /*
1091 * IMPORTANT:
1092 * the fact that we force the NaT of r3 to zero is ONLY valid
1093 * as long as we don't come here with a ldfpX.s.
1094 * For this reason we keep this sanity check
1095 */
1098 __FUNCTION__);
1101 }
1103 /*
1104 * Invalidate ALAT entries, if any, for both registers.
1105 */
1109 }
1111 }
1114 static int
1116 {
1121 /*
1122 * fr0 & fr1 don't need to be checked because Illegal Instruction
1123 * faults have higher priority than unaligned faults.
1124 *
1125 * r0 cannot be found as the base as it would never generate an
1126 * unaligned reference.
1127 */
1129 /*
1130 * make sure we get clean buffers
1131 */
1135 /*
1136 * ldfX.a we don't try to emulate anything but we must
1137 * invalidate the ALAT entry.
1138 * See comments in ldX for descriptions on how the various loads are handled.
1139 */
1146 /*
1147 * we only do something for x6_op={0,8,9}
1148 */
1162 }
1164 /*
1165 * XXX fixme
1166 *
1167 * A possible optimization would be to drop fpr_final and directly
1168 * use the storage from the saved context i.e., the actual final
1169 * destination (pt_regs, switch_stack or thread structure).
1170 */
1172 }
1174 /*
1175 * check for updates on any loads
1176 */
1180 /*
1181 * invalidate ALAT entry in case of advanced floating point loads
1182 */
1187 }
1190 static int
1192 {
1197 /*
1198 * make sure we get clean buffers
1199 */
1203 /*
1204 * if we get to this handler, Nat bits on both r3 and r2 have already
1205 * been checked. so we don't need to do it
1206 *
1207 * extract the value to be stored
1208 */
1210 /*
1211 * during this step, we extract the spilled registers from the saved
1212 * context i.e., we refill. Then we store (no spill) to temporary
1213 * aligned location
1214 */
1228 }
1236 /*
1237 * stfX [r3]=r2,imm(9)
1238 *
1239 * NOTE:
1240 * ld.r3 can never be r0, because r0 would not generate an
1241 * unaligned access.
1242 */
1246 /*
1247 * form imm9: [12:6] contain first 7bits
1248 */
1250 /*
1251 * sign extend (8bits) if m set
1252 */
1255 /*
1256 * ifa == r3 (NaT is necessarily cleared)
1257 */
1263 }
1264 /*
1265 * we don't have alat_invalidate_multiple() so we need
1266 * to do the complete flush :-<<
1267 */
1271 }
1273 /*
1274 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1275 * eventually fix the program. However, we don't want to do that for every access so we
1276 * pace it with jiffies. This isn't really MP-safe, but it doesn't really have to be
1277 * either...
1278 */
1279 static int
1281 {
1289 }
1292 }
1294 void
1296 {
1305 load_store_t insn;
1310 /* we don't support big-endian accesses */
1313 }
1315 /*
1316 * Treat kernel accesses for which there is an exception handler entry the same as
1317 * user-level unaligned accesses. Otherwise, a clever program could trick this
1318 * handler into reading an arbitrary kernel addresses...
1319 */
1328 {
1335 /*
1336 * Don't call tty_write_message() if we're in the kernel; we might
1337 * be holding locks...
1338 */
1343 }
1349 }
1357 /*
1358 * extract the instruction from the bundle given the slot number
1359 */
1364 }
1371 /*
1372 * IMPORTANT:
1373 * Notice that the switch statement DOES not cover all possible instructions
1374 * that DO generate unaligned references. This is made on purpose because for some
1375 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1376 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1377 * the program will get a signal and die:
1378 *
1379 * load/store:
1380 * - ldX.spill
1381 * - stX.spill
1382 * Reason: RNATs are based on addresses
1383 * - ld16
1384 * - st16
1385 * Reason: ld16 and st16 are supposed to occur in a single
1386 * memory op
1387 *
1388 * synchronization:
1389 * - cmpxchg
1390 * - fetchadd
1391 * - xchg
1392 * Reason: ATOMIC operations cannot be emulated properly using multiple
1393 * instructions.
1394 *
1395 * speculative loads:
1396 * - ldX.sZ
1397 * Reason: side effects, code must be ready to deal with failure so simpler
1398 * to let the load fail.
1399 * ---------------------------------------------------------------------------------
1400 * XXX fixme
1401 *
1402 * I would like to get rid of this switch case and do something
1403 * more elegant.
1404 */
1409 /* oops, really a semaphore op (cmpxchg, etc) */
1411 /* no break */
1417 /*
1418 * The instruction will be retried with deferred exceptions turned on, and
1419 * we should get Nat bit installed
1420 *
1421 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1422 * are actually executed even though the operation failed. So we don't
1423 * need to take care of this.
1424 */
1437 /* oops, really a semaphore op (cmpxchg, etc) */
1439 /* no break */
1453 /* oops, really a semaphore op (cmpxchg, etc) */
1455 /* no break */
1471 else
1482 }
1488 /*
1489 * given today's architecture this case is not likely to happen because a
1490 * memory access instruction (M) can never be in the last slot of a
1491 * bundle. But let's keep it for now.
1492 */
1497 done:
1501 failure:
1502 /* something went wrong... */
1507 }
1509 /* NOT_REACHED */
1510 }
1511 force_sigbus:
1521 }