2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Synthesize TLB refill handlers at runtime.
8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
9 * Copyright (C) 2005, 2007 Maciej W. Rozycki
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
12 * ... and the days got worse and worse and now you see
13 * I've gone completly out of my mind.
15 * They're coming to take me a away haha
16 * they're coming to take me a away hoho hihi haha
17 * to the funny farm where code is beautiful all the time ...
19 * (Condolences to Napoleon XIV)
22 #include <linux/kernel.h>
23 #include <linux/types.h>
24 #include <linux/string.h>
25 #include <linux/init.h>
27 #include <asm/mmu_context.h>
32 static inline int r45k_bvahwbug(void)
34 /* XXX: We should probe for the presence of this bug, but we don't. */
38 static inline int r4k_250MHZhwbug(void)
40 /* XXX: We should probe for the presence of this bug, but we don't. */
44 static inline int __maybe_unused
bcm1250_m3_war(void)
46 return BCM1250_M3_WAR
;
49 static inline int __maybe_unused
r10000_llsc_war(void)
51 return R10000_LLSC_WAR
;
55 * Found by experiment: At least some revisions of the 4kc throw under
56 * some circumstances a machine check exception, triggered by invalid
57 * values in the index register. Delaying the tlbp instruction until
58 * after the next branch, plus adding an additional nop in front of
59 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
60 * why; it's not an issue caused by the core RTL.
63 static int __cpuinit
m4kc_tlbp_war(void)
65 return (current_cpu_data
.processor_id
& 0xffff00) ==
66 (PRID_COMP_MIPS
| PRID_IMP_4KC
);
69 /* Handle labels (which must be positive integers). */
71 label_second_part
= 1,
83 label_smp_pgtable_change
,
84 label_r3000_write_probe_fail
,
87 UASM_L_LA(_second_part
)
90 UASM_L_LA(_module_alloc
)
93 UASM_L_LA(_vmalloc_done
)
94 UASM_L_LA(_tlbw_hazard
)
96 UASM_L_LA(_nopage_tlbl
)
97 UASM_L_LA(_nopage_tlbs
)
98 UASM_L_LA(_nopage_tlbm
)
99 UASM_L_LA(_smp_pgtable_change
)
100 UASM_L_LA(_r3000_write_probe_fail
)
103 * For debug purposes.
105 static inline void dump_handler(const u32
*handler
, int count
)
109 pr_debug("\t.set push\n");
110 pr_debug("\t.set noreorder\n");
112 for (i
= 0; i
< count
; i
++)
113 pr_debug("\t%p\t.word 0x%08x\n", &handler
[i
], handler
[i
]);
115 pr_debug("\t.set pop\n");
118 /* The only general purpose registers allowed in TLB handlers. */
122 /* Some CP0 registers */
123 #define C0_INDEX 0, 0
124 #define C0_ENTRYLO0 2, 0
125 #define C0_TCBIND 2, 2
126 #define C0_ENTRYLO1 3, 0
127 #define C0_CONTEXT 4, 0
128 #define C0_BADVADDR 8, 0
129 #define C0_ENTRYHI 10, 0
131 #define C0_XCONTEXT 20, 0
134 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
136 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
139 /* The worst case length of the handler is around 18 instructions for
140 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
141 * Maximum space available is 32 instructions for R3000 and 64
142 * instructions for R4000.
144 * We deliberately chose a buffer size of 128, so we won't scribble
145 * over anything important on overflow before we panic.
147 static u32 tlb_handler
[128] __cpuinitdata
;
149 /* simply assume worst case size for labels and relocs */
150 static struct uasm_label labels
[128] __cpuinitdata
;
151 static struct uasm_reloc relocs
[128] __cpuinitdata
;
154 * The R3000 TLB handler is simple.
156 static void __cpuinit
build_r3000_tlb_refill_handler(void)
158 long pgdc
= (long)pgd_current
;
161 memset(tlb_handler
, 0, sizeof(tlb_handler
));
164 uasm_i_mfc0(&p
, K0
, C0_BADVADDR
);
165 uasm_i_lui(&p
, K1
, uasm_rel_hi(pgdc
)); /* cp0 delay */
166 uasm_i_lw(&p
, K1
, uasm_rel_lo(pgdc
), K1
);
167 uasm_i_srl(&p
, K0
, K0
, 22); /* load delay */
168 uasm_i_sll(&p
, K0
, K0
, 2);
169 uasm_i_addu(&p
, K1
, K1
, K0
);
170 uasm_i_mfc0(&p
, K0
, C0_CONTEXT
);
171 uasm_i_lw(&p
, K1
, 0, K1
); /* cp0 delay */
172 uasm_i_andi(&p
, K0
, K0
, 0xffc); /* load delay */
173 uasm_i_addu(&p
, K1
, K1
, K0
);
174 uasm_i_lw(&p
, K0
, 0, K1
);
175 uasm_i_nop(&p
); /* load delay */
176 uasm_i_mtc0(&p
, K0
, C0_ENTRYLO0
);
177 uasm_i_mfc0(&p
, K1
, C0_EPC
); /* cp0 delay */
178 uasm_i_tlbwr(&p
); /* cp0 delay */
180 uasm_i_rfe(&p
); /* branch delay */
182 if (p
> tlb_handler
+ 32)
183 panic("TLB refill handler space exceeded");
185 pr_debug("Wrote TLB refill handler (%u instructions).\n",
186 (unsigned int)(p
- tlb_handler
));
188 memcpy((void *)ebase
, tlb_handler
, 0x80);
190 dump_handler((u32
*)ebase
, 32);
194 * The R4000 TLB handler is much more complicated. We have two
195 * consecutive handler areas with 32 instructions space each.
196 * Since they aren't used at the same time, we can overflow in the
197 * other one.To keep things simple, we first assume linear space,
198 * then we relocate it to the final handler layout as needed.
200 static u32 final_handler
[64] __cpuinitdata
;
205 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
206 * 2. A timing hazard exists for the TLBP instruction.
208 * stalling_instruction
211 * The JTLB is being read for the TLBP throughout the stall generated by the
212 * previous instruction. This is not really correct as the stalling instruction
213 * can modify the address used to access the JTLB. The failure symptom is that
214 * the TLBP instruction will use an address created for the stalling instruction
215 * and not the address held in C0_ENHI and thus report the wrong results.
217 * The software work-around is to not allow the instruction preceding the TLBP
218 * to stall - make it an NOP or some other instruction guaranteed not to stall.
220 * Errata 2 will not be fixed. This errata is also on the R5000.
222 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
224 static void __cpuinit __maybe_unused
build_tlb_probe_entry(u32
**p
)
226 switch (current_cpu_type()) {
227 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */
244 * Write random or indexed TLB entry, and care about the hazards from
245 * the preceeding mtc0 and for the following eret.
247 enum tlb_write_entry
{ tlb_random
, tlb_indexed
};
249 static void __cpuinit
build_tlb_write_entry(u32
**p
, struct uasm_label
**l
,
250 struct uasm_reloc
**r
,
251 enum tlb_write_entry wmode
)
253 void(*tlbw
)(u32
**) = NULL
;
256 case tlb_random
: tlbw
= uasm_i_tlbwr
; break;
257 case tlb_indexed
: tlbw
= uasm_i_tlbwi
; break;
260 if (cpu_has_mips_r2
) {
266 switch (current_cpu_type()) {
274 * This branch uses up a mtc0 hazard nop slot and saves
275 * two nops after the tlbw instruction.
277 uasm_il_bgezl(p
, r
, 0, label_tlbw_hazard
);
279 uasm_l_tlbw_hazard(l
, *p
);
320 case CPU_CAVIUM_OCTEON
:
328 uasm_i_nop(p
); /* QED specifies 2 nops hazard */
330 * This branch uses up a mtc0 hazard nop slot and saves
331 * a nop after the tlbw instruction.
333 uasm_il_bgezl(p
, r
, 0, label_tlbw_hazard
);
335 uasm_l_tlbw_hazard(l
, *p
);
348 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
349 * use of the JTLB for instructions should not occur for 4
350 * cpu cycles and use for data translations should not occur
385 panic("No TLB refill handler yet (CPU type: %d)",
386 current_cpu_data
.cputype
);
393 * TMP and PTR are scratch.
394 * TMP will be clobbered, PTR will hold the pmd entry.
396 static void __cpuinit
397 build_get_pmde64(u32
**p
, struct uasm_label
**l
, struct uasm_reloc
**r
,
398 unsigned int tmp
, unsigned int ptr
)
400 long pgdc
= (long)pgd_current
;
403 * The vmalloc handling is not in the hotpath.
405 uasm_i_dmfc0(p
, tmp
, C0_BADVADDR
);
407 uasm_il_bltz(p
, r
, tmp
, label_module_alloc
);
409 uasm_il_bltz(p
, r
, tmp
, label_vmalloc
);
411 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
414 # ifdef CONFIG_MIPS_MT_SMTC
416 * SMTC uses TCBind value as "CPU" index
418 uasm_i_mfc0(p
, ptr
, C0_TCBIND
);
419 uasm_i_dsrl(p
, ptr
, ptr
, 19);
422 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
425 uasm_i_dmfc0(p
, ptr
, C0_CONTEXT
);
426 uasm_i_dsrl(p
, ptr
, ptr
, 23);
428 UASM_i_LA_mostly(p
, tmp
, pgdc
);
429 uasm_i_daddu(p
, ptr
, ptr
, tmp
);
430 uasm_i_dmfc0(p
, tmp
, C0_BADVADDR
);
431 uasm_i_ld(p
, ptr
, uasm_rel_lo(pgdc
), ptr
);
433 UASM_i_LA_mostly(p
, ptr
, pgdc
);
434 uasm_i_ld(p
, ptr
, uasm_rel_lo(pgdc
), ptr
);
437 uasm_l_vmalloc_done(l
, *p
);
439 if (PGDIR_SHIFT
- 3 < 32) /* get pgd offset in bytes */
440 uasm_i_dsrl(p
, tmp
, tmp
, PGDIR_SHIFT
-3);
442 uasm_i_dsrl32(p
, tmp
, tmp
, PGDIR_SHIFT
- 3 - 32);
444 uasm_i_andi(p
, tmp
, tmp
, (PTRS_PER_PGD
- 1)<<3);
445 uasm_i_daddu(p
, ptr
, ptr
, tmp
); /* add in pgd offset */
446 uasm_i_dmfc0(p
, tmp
, C0_BADVADDR
); /* get faulting address */
447 uasm_i_ld(p
, ptr
, 0, ptr
); /* get pmd pointer */
448 uasm_i_dsrl(p
, tmp
, tmp
, PMD_SHIFT
-3); /* get pmd offset in bytes */
449 uasm_i_andi(p
, tmp
, tmp
, (PTRS_PER_PMD
- 1)<<3);
450 uasm_i_daddu(p
, ptr
, ptr
, tmp
); /* add in pmd offset */
454 * BVADDR is the faulting address, PTR is scratch.
455 * PTR will hold the pgd for vmalloc.
457 static void __cpuinit
458 build_get_pgd_vmalloc64(u32
**p
, struct uasm_label
**l
, struct uasm_reloc
**r
,
459 unsigned int bvaddr
, unsigned int ptr
)
461 long swpd
= (long)swapper_pg_dir
;
464 long modd
= (long)module_pg_dir
;
466 uasm_l_module_alloc(l
, *p
);
469 * VMALLOC_START >= 0xc000000000000000UL
470 * MODULE_START >= 0xe000000000000000UL
472 UASM_i_SLL(p
, ptr
, bvaddr
, 2);
473 uasm_il_bgez(p
, r
, ptr
, label_vmalloc
);
475 if (uasm_in_compat_space_p(MODULE_START
) &&
476 !uasm_rel_lo(MODULE_START
)) {
477 uasm_i_lui(p
, ptr
, uasm_rel_hi(MODULE_START
)); /* delay slot */
479 /* unlikely configuration */
480 uasm_i_nop(p
); /* delay slot */
481 UASM_i_LA(p
, ptr
, MODULE_START
);
483 uasm_i_dsubu(p
, bvaddr
, bvaddr
, ptr
);
485 if (uasm_in_compat_space_p(modd
) && !uasm_rel_lo(modd
)) {
486 uasm_il_b(p
, r
, label_vmalloc_done
);
487 uasm_i_lui(p
, ptr
, uasm_rel_hi(modd
));
489 UASM_i_LA_mostly(p
, ptr
, modd
);
490 uasm_il_b(p
, r
, label_vmalloc_done
);
491 if (uasm_in_compat_space_p(modd
))
492 uasm_i_addiu(p
, ptr
, ptr
, uasm_rel_lo(modd
));
494 uasm_i_daddiu(p
, ptr
, ptr
, uasm_rel_lo(modd
));
497 uasm_l_vmalloc(l
, *p
);
498 if (uasm_in_compat_space_p(MODULE_START
) &&
499 !uasm_rel_lo(MODULE_START
) &&
500 MODULE_START
<< 32 == VMALLOC_START
)
501 uasm_i_dsll32(p
, ptr
, ptr
, 0); /* typical case */
503 UASM_i_LA(p
, ptr
, VMALLOC_START
);
505 uasm_l_vmalloc(l
, *p
);
506 UASM_i_LA(p
, ptr
, VMALLOC_START
);
508 uasm_i_dsubu(p
, bvaddr
, bvaddr
, ptr
);
510 if (uasm_in_compat_space_p(swpd
) && !uasm_rel_lo(swpd
)) {
511 uasm_il_b(p
, r
, label_vmalloc_done
);
512 uasm_i_lui(p
, ptr
, uasm_rel_hi(swpd
));
514 UASM_i_LA_mostly(p
, ptr
, swpd
);
515 uasm_il_b(p
, r
, label_vmalloc_done
);
516 if (uasm_in_compat_space_p(swpd
))
517 uasm_i_addiu(p
, ptr
, ptr
, uasm_rel_lo(swpd
));
519 uasm_i_daddiu(p
, ptr
, ptr
, uasm_rel_lo(swpd
));
523 #else /* !CONFIG_64BIT */
526 * TMP and PTR are scratch.
527 * TMP will be clobbered, PTR will hold the pgd entry.
529 static void __cpuinit __maybe_unused
530 build_get_pgde32(u32
**p
, unsigned int tmp
, unsigned int ptr
)
532 long pgdc
= (long)pgd_current
;
534 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
536 #ifdef CONFIG_MIPS_MT_SMTC
538 * SMTC uses TCBind value as "CPU" index
540 uasm_i_mfc0(p
, ptr
, C0_TCBIND
);
541 UASM_i_LA_mostly(p
, tmp
, pgdc
);
542 uasm_i_srl(p
, ptr
, ptr
, 19);
545 * smp_processor_id() << 3 is stored in CONTEXT.
547 uasm_i_mfc0(p
, ptr
, C0_CONTEXT
);
548 UASM_i_LA_mostly(p
, tmp
, pgdc
);
549 uasm_i_srl(p
, ptr
, ptr
, 23);
551 uasm_i_addu(p
, ptr
, tmp
, ptr
);
553 UASM_i_LA_mostly(p
, ptr
, pgdc
);
555 uasm_i_mfc0(p
, tmp
, C0_BADVADDR
); /* get faulting address */
556 uasm_i_lw(p
, ptr
, uasm_rel_lo(pgdc
), ptr
);
557 uasm_i_srl(p
, tmp
, tmp
, PGDIR_SHIFT
); /* get pgd only bits */
558 uasm_i_sll(p
, tmp
, tmp
, PGD_T_LOG2
);
559 uasm_i_addu(p
, ptr
, ptr
, tmp
); /* add in pgd offset */
562 #endif /* !CONFIG_64BIT */
564 static void __cpuinit
build_adjust_context(u32
**p
, unsigned int ctx
)
566 unsigned int shift
= 4 - (PTE_T_LOG2
+ 1) + PAGE_SHIFT
- 12;
567 unsigned int mask
= (PTRS_PER_PTE
/ 2 - 1) << (PTE_T_LOG2
+ 1);
569 switch (current_cpu_type()) {
586 UASM_i_SRL(p
, ctx
, ctx
, shift
);
587 uasm_i_andi(p
, ctx
, ctx
, mask
);
590 static void __cpuinit
build_get_ptep(u32
**p
, unsigned int tmp
, unsigned int ptr
)
593 * Bug workaround for the Nevada. It seems as if under certain
594 * circumstances the move from cp0_context might produce a
595 * bogus result when the mfc0 instruction and its consumer are
596 * in a different cacheline or a load instruction, probably any
597 * memory reference, is between them.
599 switch (current_cpu_type()) {
601 UASM_i_LW(p
, ptr
, 0, ptr
);
602 GET_CONTEXT(p
, tmp
); /* get context reg */
606 GET_CONTEXT(p
, tmp
); /* get context reg */
607 UASM_i_LW(p
, ptr
, 0, ptr
);
611 build_adjust_context(p
, tmp
);
612 UASM_i_ADDU(p
, ptr
, ptr
, tmp
); /* add in offset */
615 static void __cpuinit
build_update_entries(u32
**p
, unsigned int tmp
,
619 * 64bit address support (36bit on a 32bit CPU) in a 32bit
620 * Kernel is a special case. Only a few CPUs use it.
622 #ifdef CONFIG_64BIT_PHYS_ADDR
623 if (cpu_has_64bits
) {
624 uasm_i_ld(p
, tmp
, 0, ptep
); /* get even pte */
625 uasm_i_ld(p
, ptep
, sizeof(pte_t
), ptep
); /* get odd pte */
626 uasm_i_dsrl(p
, tmp
, tmp
, 6); /* convert to entrylo0 */
627 uasm_i_mtc0(p
, tmp
, C0_ENTRYLO0
); /* load it */
628 uasm_i_dsrl(p
, ptep
, ptep
, 6); /* convert to entrylo1 */
629 uasm_i_mtc0(p
, ptep
, C0_ENTRYLO1
); /* load it */
631 int pte_off_even
= sizeof(pte_t
) / 2;
632 int pte_off_odd
= pte_off_even
+ sizeof(pte_t
);
634 /* The pte entries are pre-shifted */
635 uasm_i_lw(p
, tmp
, pte_off_even
, ptep
); /* get even pte */
636 uasm_i_mtc0(p
, tmp
, C0_ENTRYLO0
); /* load it */
637 uasm_i_lw(p
, ptep
, pte_off_odd
, ptep
); /* get odd pte */
638 uasm_i_mtc0(p
, ptep
, C0_ENTRYLO1
); /* load it */
641 UASM_i_LW(p
, tmp
, 0, ptep
); /* get even pte */
642 UASM_i_LW(p
, ptep
, sizeof(pte_t
), ptep
); /* get odd pte */
644 build_tlb_probe_entry(p
);
645 UASM_i_SRL(p
, tmp
, tmp
, 6); /* convert to entrylo0 */
646 if (r4k_250MHZhwbug())
647 uasm_i_mtc0(p
, 0, C0_ENTRYLO0
);
648 uasm_i_mtc0(p
, tmp
, C0_ENTRYLO0
); /* load it */
649 UASM_i_SRL(p
, ptep
, ptep
, 6); /* convert to entrylo1 */
651 uasm_i_mfc0(p
, tmp
, C0_INDEX
);
652 if (r4k_250MHZhwbug())
653 uasm_i_mtc0(p
, 0, C0_ENTRYLO1
);
654 uasm_i_mtc0(p
, ptep
, C0_ENTRYLO1
); /* load it */
658 static void __cpuinit
build_r4000_tlb_refill_handler(void)
660 u32
*p
= tlb_handler
;
661 struct uasm_label
*l
= labels
;
662 struct uasm_reloc
*r
= relocs
;
664 unsigned int final_len
;
666 memset(tlb_handler
, 0, sizeof(tlb_handler
));
667 memset(labels
, 0, sizeof(labels
));
668 memset(relocs
, 0, sizeof(relocs
));
669 memset(final_handler
, 0, sizeof(final_handler
));
672 * create the plain linear handler
674 if (bcm1250_m3_war()) {
675 UASM_i_MFC0(&p
, K0
, C0_BADVADDR
);
676 UASM_i_MFC0(&p
, K1
, C0_ENTRYHI
);
677 uasm_i_xor(&p
, K0
, K0
, K1
);
678 UASM_i_SRL(&p
, K0
, K0
, PAGE_SHIFT
+ 1);
679 uasm_il_bnez(&p
, &r
, K0
, label_leave
);
680 /* No need for uasm_i_nop */
684 build_get_pmde64(&p
, &l
, &r
, K0
, K1
); /* get pmd in K1 */
686 build_get_pgde32(&p
, K0
, K1
); /* get pgd in K1 */
689 build_get_ptep(&p
, K0
, K1
);
690 build_update_entries(&p
, K0
, K1
);
691 build_tlb_write_entry(&p
, &l
, &r
, tlb_random
);
693 uasm_i_eret(&p
); /* return from trap */
696 build_get_pgd_vmalloc64(&p
, &l
, &r
, K0
, K1
);
700 * Overflow check: For the 64bit handler, we need at least one
701 * free instruction slot for the wrap-around branch. In worst
702 * case, if the intended insertion point is a delay slot, we
703 * need three, with the second nop'ed and the third being
706 /* Loongson2 ebase is different than r4k, we have more space */
707 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
708 if ((p
- tlb_handler
) > 64)
709 panic("TLB refill handler space exceeded");
711 if (((p
- tlb_handler
) > 63)
712 || (((p
- tlb_handler
) > 61)
713 && uasm_insn_has_bdelay(relocs
, tlb_handler
+ 29)))
714 panic("TLB refill handler space exceeded");
718 * Now fold the handler in the TLB refill handler space.
720 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
722 /* Simplest case, just copy the handler. */
723 uasm_copy_handler(relocs
, labels
, tlb_handler
, p
, f
);
724 final_len
= p
- tlb_handler
;
725 #else /* CONFIG_64BIT */
726 f
= final_handler
+ 32;
727 if ((p
- tlb_handler
) <= 32) {
728 /* Just copy the handler. */
729 uasm_copy_handler(relocs
, labels
, tlb_handler
, p
, f
);
730 final_len
= p
- tlb_handler
;
732 u32
*split
= tlb_handler
+ 30;
735 * Find the split point.
737 if (uasm_insn_has_bdelay(relocs
, split
- 1))
740 /* Copy first part of the handler. */
741 uasm_copy_handler(relocs
, labels
, tlb_handler
, split
, f
);
742 f
+= split
- tlb_handler
;
745 uasm_l_split(&l
, final_handler
);
746 uasm_il_b(&f
, &r
, label_split
);
747 if (uasm_insn_has_bdelay(relocs
, split
))
750 uasm_copy_handler(relocs
, labels
, split
, split
+ 1, f
);
751 uasm_move_labels(labels
, f
, f
+ 1, -1);
756 /* Copy the rest of the handler. */
757 uasm_copy_handler(relocs
, labels
, split
, p
, final_handler
);
758 final_len
= (f
- (final_handler
+ 32)) + (p
- split
);
760 #endif /* CONFIG_64BIT */
762 uasm_resolve_relocs(relocs
, labels
);
763 pr_debug("Wrote TLB refill handler (%u instructions).\n",
766 memcpy((void *)ebase
, final_handler
, 0x100);
768 dump_handler((u32
*)ebase
, 64);
772 * TLB load/store/modify handlers.
774 * Only the fastpath gets synthesized at runtime, the slowpath for
775 * do_page_fault remains normal asm.
777 extern void tlb_do_page_fault_0(void);
778 extern void tlb_do_page_fault_1(void);
781 * 128 instructions for the fastpath handler is generous and should
784 #define FASTPATH_SIZE 128
786 u32 handle_tlbl
[FASTPATH_SIZE
] __cacheline_aligned
;
787 u32 handle_tlbs
[FASTPATH_SIZE
] __cacheline_aligned
;
788 u32 handle_tlbm
[FASTPATH_SIZE
] __cacheline_aligned
;
790 static void __cpuinit
791 iPTE_LW(u32
**p
, struct uasm_label
**l
, unsigned int pte
, unsigned int ptr
)
794 # ifdef CONFIG_64BIT_PHYS_ADDR
796 uasm_i_lld(p
, pte
, 0, ptr
);
799 UASM_i_LL(p
, pte
, 0, ptr
);
801 # ifdef CONFIG_64BIT_PHYS_ADDR
803 uasm_i_ld(p
, pte
, 0, ptr
);
806 UASM_i_LW(p
, pte
, 0, ptr
);
810 static void __cpuinit
811 iPTE_SW(u32
**p
, struct uasm_reloc
**r
, unsigned int pte
, unsigned int ptr
,
814 #ifdef CONFIG_64BIT_PHYS_ADDR
815 unsigned int hwmode
= mode
& (_PAGE_VALID
| _PAGE_DIRTY
);
818 uasm_i_ori(p
, pte
, pte
, mode
);
820 # ifdef CONFIG_64BIT_PHYS_ADDR
822 uasm_i_scd(p
, pte
, 0, ptr
);
825 UASM_i_SC(p
, pte
, 0, ptr
);
827 if (r10000_llsc_war())
828 uasm_il_beqzl(p
, r
, pte
, label_smp_pgtable_change
);
830 uasm_il_beqz(p
, r
, pte
, label_smp_pgtable_change
);
832 # ifdef CONFIG_64BIT_PHYS_ADDR
833 if (!cpu_has_64bits
) {
834 /* no uasm_i_nop needed */
835 uasm_i_ll(p
, pte
, sizeof(pte_t
) / 2, ptr
);
836 uasm_i_ori(p
, pte
, pte
, hwmode
);
837 uasm_i_sc(p
, pte
, sizeof(pte_t
) / 2, ptr
);
838 uasm_il_beqz(p
, r
, pte
, label_smp_pgtable_change
);
839 /* no uasm_i_nop needed */
840 uasm_i_lw(p
, pte
, 0, ptr
);
847 # ifdef CONFIG_64BIT_PHYS_ADDR
849 uasm_i_sd(p
, pte
, 0, ptr
);
852 UASM_i_SW(p
, pte
, 0, ptr
);
854 # ifdef CONFIG_64BIT_PHYS_ADDR
855 if (!cpu_has_64bits
) {
856 uasm_i_lw(p
, pte
, sizeof(pte_t
) / 2, ptr
);
857 uasm_i_ori(p
, pte
, pte
, hwmode
);
858 uasm_i_sw(p
, pte
, sizeof(pte_t
) / 2, ptr
);
859 uasm_i_lw(p
, pte
, 0, ptr
);
866 * Check if PTE is present, if not then jump to LABEL. PTR points to
867 * the page table where this PTE is located, PTE will be re-loaded
868 * with it's original value.
870 static void __cpuinit
871 build_pte_present(u32
**p
, struct uasm_label
**l
, struct uasm_reloc
**r
,
872 unsigned int pte
, unsigned int ptr
, enum label_id lid
)
874 uasm_i_andi(p
, pte
, pte
, _PAGE_PRESENT
| _PAGE_READ
);
875 uasm_i_xori(p
, pte
, pte
, _PAGE_PRESENT
| _PAGE_READ
);
876 uasm_il_bnez(p
, r
, pte
, lid
);
877 iPTE_LW(p
, l
, pte
, ptr
);
880 /* Make PTE valid, store result in PTR. */
881 static void __cpuinit
882 build_make_valid(u32
**p
, struct uasm_reloc
**r
, unsigned int pte
,
885 unsigned int mode
= _PAGE_VALID
| _PAGE_ACCESSED
;
887 iPTE_SW(p
, r
, pte
, ptr
, mode
);
891 * Check if PTE can be written to, if not branch to LABEL. Regardless
892 * restore PTE with value from PTR when done.
894 static void __cpuinit
895 build_pte_writable(u32
**p
, struct uasm_label
**l
, struct uasm_reloc
**r
,
896 unsigned int pte
, unsigned int ptr
, enum label_id lid
)
898 uasm_i_andi(p
, pte
, pte
, _PAGE_PRESENT
| _PAGE_WRITE
);
899 uasm_i_xori(p
, pte
, pte
, _PAGE_PRESENT
| _PAGE_WRITE
);
900 uasm_il_bnez(p
, r
, pte
, lid
);
901 iPTE_LW(p
, l
, pte
, ptr
);
904 /* Make PTE writable, update software status bits as well, then store
907 static void __cpuinit
908 build_make_write(u32
**p
, struct uasm_reloc
**r
, unsigned int pte
,
911 unsigned int mode
= (_PAGE_ACCESSED
| _PAGE_MODIFIED
| _PAGE_VALID
914 iPTE_SW(p
, r
, pte
, ptr
, mode
);
918 * Check if PTE can be modified, if not branch to LABEL. Regardless
919 * restore PTE with value from PTR when done.
921 static void __cpuinit
922 build_pte_modifiable(u32
**p
, struct uasm_label
**l
, struct uasm_reloc
**r
,
923 unsigned int pte
, unsigned int ptr
, enum label_id lid
)
925 uasm_i_andi(p
, pte
, pte
, _PAGE_WRITE
);
926 uasm_il_beqz(p
, r
, pte
, lid
);
927 iPTE_LW(p
, l
, pte
, ptr
);
931 * R3000 style TLB load/store/modify handlers.
935 * This places the pte into ENTRYLO0 and writes it with tlbwi.
938 static void __cpuinit
939 build_r3000_pte_reload_tlbwi(u32
**p
, unsigned int pte
, unsigned int tmp
)
941 uasm_i_mtc0(p
, pte
, C0_ENTRYLO0
); /* cp0 delay */
942 uasm_i_mfc0(p
, tmp
, C0_EPC
); /* cp0 delay */
945 uasm_i_rfe(p
); /* branch delay */
949 * This places the pte into ENTRYLO0 and writes it with tlbwi
950 * or tlbwr as appropriate. This is because the index register
951 * may have the probe fail bit set as a result of a trap on a
952 * kseg2 access, i.e. without refill. Then it returns.
954 static void __cpuinit
955 build_r3000_tlb_reload_write(u32
**p
, struct uasm_label
**l
,
956 struct uasm_reloc
**r
, unsigned int pte
,
959 uasm_i_mfc0(p
, tmp
, C0_INDEX
);
960 uasm_i_mtc0(p
, pte
, C0_ENTRYLO0
); /* cp0 delay */
961 uasm_il_bltz(p
, r
, tmp
, label_r3000_write_probe_fail
); /* cp0 delay */
962 uasm_i_mfc0(p
, tmp
, C0_EPC
); /* branch delay */
963 uasm_i_tlbwi(p
); /* cp0 delay */
965 uasm_i_rfe(p
); /* branch delay */
966 uasm_l_r3000_write_probe_fail(l
, *p
);
967 uasm_i_tlbwr(p
); /* cp0 delay */
969 uasm_i_rfe(p
); /* branch delay */
972 static void __cpuinit
973 build_r3000_tlbchange_handler_head(u32
**p
, unsigned int pte
,
976 long pgdc
= (long)pgd_current
;
978 uasm_i_mfc0(p
, pte
, C0_BADVADDR
);
979 uasm_i_lui(p
, ptr
, uasm_rel_hi(pgdc
)); /* cp0 delay */
980 uasm_i_lw(p
, ptr
, uasm_rel_lo(pgdc
), ptr
);
981 uasm_i_srl(p
, pte
, pte
, 22); /* load delay */
982 uasm_i_sll(p
, pte
, pte
, 2);
983 uasm_i_addu(p
, ptr
, ptr
, pte
);
984 uasm_i_mfc0(p
, pte
, C0_CONTEXT
);
985 uasm_i_lw(p
, ptr
, 0, ptr
); /* cp0 delay */
986 uasm_i_andi(p
, pte
, pte
, 0xffc); /* load delay */
987 uasm_i_addu(p
, ptr
, ptr
, pte
);
988 uasm_i_lw(p
, pte
, 0, ptr
);
989 uasm_i_tlbp(p
); /* load delay */
992 static void __cpuinit
build_r3000_tlb_load_handler(void)
994 u32
*p
= handle_tlbl
;
995 struct uasm_label
*l
= labels
;
996 struct uasm_reloc
*r
= relocs
;
998 memset(handle_tlbl
, 0, sizeof(handle_tlbl
));
999 memset(labels
, 0, sizeof(labels
));
1000 memset(relocs
, 0, sizeof(relocs
));
1002 build_r3000_tlbchange_handler_head(&p
, K0
, K1
);
1003 build_pte_present(&p
, &l
, &r
, K0
, K1
, label_nopage_tlbl
);
1004 uasm_i_nop(&p
); /* load delay */
1005 build_make_valid(&p
, &r
, K0
, K1
);
1006 build_r3000_tlb_reload_write(&p
, &l
, &r
, K0
, K1
);
1008 uasm_l_nopage_tlbl(&l
, p
);
1009 uasm_i_j(&p
, (unsigned long)tlb_do_page_fault_0
& 0x0fffffff);
1012 if ((p
- handle_tlbl
) > FASTPATH_SIZE
)
1013 panic("TLB load handler fastpath space exceeded");
1015 uasm_resolve_relocs(relocs
, labels
);
1016 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1017 (unsigned int)(p
- handle_tlbl
));
1019 dump_handler(handle_tlbl
, ARRAY_SIZE(handle_tlbl
));
1022 static void __cpuinit
build_r3000_tlb_store_handler(void)
1024 u32
*p
= handle_tlbs
;
1025 struct uasm_label
*l
= labels
;
1026 struct uasm_reloc
*r
= relocs
;
1028 memset(handle_tlbs
, 0, sizeof(handle_tlbs
));
1029 memset(labels
, 0, sizeof(labels
));
1030 memset(relocs
, 0, sizeof(relocs
));
1032 build_r3000_tlbchange_handler_head(&p
, K0
, K1
);
1033 build_pte_writable(&p
, &l
, &r
, K0
, K1
, label_nopage_tlbs
);
1034 uasm_i_nop(&p
); /* load delay */
1035 build_make_write(&p
, &r
, K0
, K1
);
1036 build_r3000_tlb_reload_write(&p
, &l
, &r
, K0
, K1
);
1038 uasm_l_nopage_tlbs(&l
, p
);
1039 uasm_i_j(&p
, (unsigned long)tlb_do_page_fault_1
& 0x0fffffff);
1042 if ((p
- handle_tlbs
) > FASTPATH_SIZE
)
1043 panic("TLB store handler fastpath space exceeded");
1045 uasm_resolve_relocs(relocs
, labels
);
1046 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1047 (unsigned int)(p
- handle_tlbs
));
1049 dump_handler(handle_tlbs
, ARRAY_SIZE(handle_tlbs
));
1052 static void __cpuinit
build_r3000_tlb_modify_handler(void)
1054 u32
*p
= handle_tlbm
;
1055 struct uasm_label
*l
= labels
;
1056 struct uasm_reloc
*r
= relocs
;
1058 memset(handle_tlbm
, 0, sizeof(handle_tlbm
));
1059 memset(labels
, 0, sizeof(labels
));
1060 memset(relocs
, 0, sizeof(relocs
));
1062 build_r3000_tlbchange_handler_head(&p
, K0
, K1
);
1063 build_pte_modifiable(&p
, &l
, &r
, K0
, K1
, label_nopage_tlbm
);
1064 uasm_i_nop(&p
); /* load delay */
1065 build_make_write(&p
, &r
, K0
, K1
);
1066 build_r3000_pte_reload_tlbwi(&p
, K0
, K1
);
1068 uasm_l_nopage_tlbm(&l
, p
);
1069 uasm_i_j(&p
, (unsigned long)tlb_do_page_fault_1
& 0x0fffffff);
1072 if ((p
- handle_tlbm
) > FASTPATH_SIZE
)
1073 panic("TLB modify handler fastpath space exceeded");
1075 uasm_resolve_relocs(relocs
, labels
);
1076 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1077 (unsigned int)(p
- handle_tlbm
));
1079 dump_handler(handle_tlbm
, ARRAY_SIZE(handle_tlbm
));
1083 * R4000 style TLB load/store/modify handlers.
1085 static void __cpuinit
1086 build_r4000_tlbchange_handler_head(u32
**p
, struct uasm_label
**l
,
1087 struct uasm_reloc
**r
, unsigned int pte
,
1091 build_get_pmde64(p
, l
, r
, pte
, ptr
); /* get pmd in ptr */
1093 build_get_pgde32(p
, pte
, ptr
); /* get pgd in ptr */
1096 UASM_i_MFC0(p
, pte
, C0_BADVADDR
);
1097 UASM_i_LW(p
, ptr
, 0, ptr
);
1098 UASM_i_SRL(p
, pte
, pte
, PAGE_SHIFT
+ PTE_ORDER
- PTE_T_LOG2
);
1099 uasm_i_andi(p
, pte
, pte
, (PTRS_PER_PTE
- 1) << PTE_T_LOG2
);
1100 UASM_i_ADDU(p
, ptr
, ptr
, pte
);
1103 uasm_l_smp_pgtable_change(l
, *p
);
1105 iPTE_LW(p
, l
, pte
, ptr
); /* get even pte */
1106 if (!m4kc_tlbp_war())
1107 build_tlb_probe_entry(p
);
1110 static void __cpuinit
1111 build_r4000_tlbchange_handler_tail(u32
**p
, struct uasm_label
**l
,
1112 struct uasm_reloc
**r
, unsigned int tmp
,
1115 uasm_i_ori(p
, ptr
, ptr
, sizeof(pte_t
));
1116 uasm_i_xori(p
, ptr
, ptr
, sizeof(pte_t
));
1117 build_update_entries(p
, tmp
, ptr
);
1118 build_tlb_write_entry(p
, l
, r
, tlb_indexed
);
1119 uasm_l_leave(l
, *p
);
1120 uasm_i_eret(p
); /* return from trap */
1123 build_get_pgd_vmalloc64(p
, l
, r
, tmp
, ptr
);
1127 static void __cpuinit
build_r4000_tlb_load_handler(void)
1129 u32
*p
= handle_tlbl
;
1130 struct uasm_label
*l
= labels
;
1131 struct uasm_reloc
*r
= relocs
;
1133 memset(handle_tlbl
, 0, sizeof(handle_tlbl
));
1134 memset(labels
, 0, sizeof(labels
));
1135 memset(relocs
, 0, sizeof(relocs
));
1137 if (bcm1250_m3_war()) {
1138 UASM_i_MFC0(&p
, K0
, C0_BADVADDR
);
1139 UASM_i_MFC0(&p
, K1
, C0_ENTRYHI
);
1140 uasm_i_xor(&p
, K0
, K0
, K1
);
1141 UASM_i_SRL(&p
, K0
, K0
, PAGE_SHIFT
+ 1);
1142 uasm_il_bnez(&p
, &r
, K0
, label_leave
);
1143 /* No need for uasm_i_nop */
1146 build_r4000_tlbchange_handler_head(&p
, &l
, &r
, K0
, K1
);
1147 build_pte_present(&p
, &l
, &r
, K0
, K1
, label_nopage_tlbl
);
1148 if (m4kc_tlbp_war())
1149 build_tlb_probe_entry(&p
);
1150 build_make_valid(&p
, &r
, K0
, K1
);
1151 build_r4000_tlbchange_handler_tail(&p
, &l
, &r
, K0
, K1
);
1153 uasm_l_nopage_tlbl(&l
, p
);
1154 uasm_i_j(&p
, (unsigned long)tlb_do_page_fault_0
& 0x0fffffff);
1157 if ((p
- handle_tlbl
) > FASTPATH_SIZE
)
1158 panic("TLB load handler fastpath space exceeded");
1160 uasm_resolve_relocs(relocs
, labels
);
1161 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1162 (unsigned int)(p
- handle_tlbl
));
1164 dump_handler(handle_tlbl
, ARRAY_SIZE(handle_tlbl
));
1167 static void __cpuinit
build_r4000_tlb_store_handler(void)
1169 u32
*p
= handle_tlbs
;
1170 struct uasm_label
*l
= labels
;
1171 struct uasm_reloc
*r
= relocs
;
1173 memset(handle_tlbs
, 0, sizeof(handle_tlbs
));
1174 memset(labels
, 0, sizeof(labels
));
1175 memset(relocs
, 0, sizeof(relocs
));
1177 build_r4000_tlbchange_handler_head(&p
, &l
, &r
, K0
, K1
);
1178 build_pte_writable(&p
, &l
, &r
, K0
, K1
, label_nopage_tlbs
);
1179 if (m4kc_tlbp_war())
1180 build_tlb_probe_entry(&p
);
1181 build_make_write(&p
, &r
, K0
, K1
);
1182 build_r4000_tlbchange_handler_tail(&p
, &l
, &r
, K0
, K1
);
1184 uasm_l_nopage_tlbs(&l
, p
);
1185 uasm_i_j(&p
, (unsigned long)tlb_do_page_fault_1
& 0x0fffffff);
1188 if ((p
- handle_tlbs
) > FASTPATH_SIZE
)
1189 panic("TLB store handler fastpath space exceeded");
1191 uasm_resolve_relocs(relocs
, labels
);
1192 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1193 (unsigned int)(p
- handle_tlbs
));
1195 dump_handler(handle_tlbs
, ARRAY_SIZE(handle_tlbs
));
1198 static void __cpuinit
build_r4000_tlb_modify_handler(void)
1200 u32
*p
= handle_tlbm
;
1201 struct uasm_label
*l
= labels
;
1202 struct uasm_reloc
*r
= relocs
;
1204 memset(handle_tlbm
, 0, sizeof(handle_tlbm
));
1205 memset(labels
, 0, sizeof(labels
));
1206 memset(relocs
, 0, sizeof(relocs
));
1208 build_r4000_tlbchange_handler_head(&p
, &l
, &r
, K0
, K1
);
1209 build_pte_modifiable(&p
, &l
, &r
, K0
, K1
, label_nopage_tlbm
);
1210 if (m4kc_tlbp_war())
1211 build_tlb_probe_entry(&p
);
1212 /* Present and writable bits set, set accessed and dirty bits. */
1213 build_make_write(&p
, &r
, K0
, K1
);
1214 build_r4000_tlbchange_handler_tail(&p
, &l
, &r
, K0
, K1
);
1216 uasm_l_nopage_tlbm(&l
, p
);
1217 uasm_i_j(&p
, (unsigned long)tlb_do_page_fault_1
& 0x0fffffff);
1220 if ((p
- handle_tlbm
) > FASTPATH_SIZE
)
1221 panic("TLB modify handler fastpath space exceeded");
1223 uasm_resolve_relocs(relocs
, labels
);
1224 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1225 (unsigned int)(p
- handle_tlbm
));
1227 dump_handler(handle_tlbm
, ARRAY_SIZE(handle_tlbm
));
1230 void __cpuinit
build_tlb_refill_handler(void)
1233 * The refill handler is generated per-CPU, multi-node systems
1234 * may have local storage for it. The other handlers are only
1237 static int run_once
= 0;
1239 switch (current_cpu_type()) {
1247 build_r3000_tlb_refill_handler();
1249 build_r3000_tlb_load_handler();
1250 build_r3000_tlb_store_handler();
1251 build_r3000_tlb_modify_handler();
1258 panic("No R6000 TLB refill handler yet");
1262 panic("No R8000 TLB refill handler yet");
1266 build_r4000_tlb_refill_handler();
1268 build_r4000_tlb_load_handler();
1269 build_r4000_tlb_store_handler();
1270 build_r4000_tlb_modify_handler();
1276 void __cpuinit
flush_tlb_handlers(void)
1278 local_flush_icache_range((unsigned long)handle_tlbl
,
1279 (unsigned long)handle_tlbl
+ sizeof(handle_tlbl
));
1280 local_flush_icache_range((unsigned long)handle_tlbs
,
1281 (unsigned long)handle_tlbs
+ sizeof(handle_tlbs
));
1282 local_flush_icache_range((unsigned long)handle_tlbm
,
1283 (unsigned long)handle_tlbm
+ sizeof(handle_tlbm
));