2 * Kernel execution entry point code.
4 * Copyright (c) 1995-1996 Gary Thomas <gdt@linuxppc.org>
5 * Initial PowerPC version.
6 * Copyright (c) 1996 Cort Dougan <cort@cs.nmt.edu>
8 * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au>
9 * Low-level exception handers, MMU support, and rewrite.
10 * Copyright (c) 1997 Dan Malek <dmalek@jlc.net>
11 * PowerPC 8xx modifications.
12 * Copyright (c) 1998-1999 TiVo, Inc.
13 * PowerPC 403GCX modifications.
14 * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu>
15 * PowerPC 403GCX/405GP modifications.
16 * Copyright 2000 MontaVista Software Inc.
17 * PPC405 modifications
18 * PowerPC 403GCX/405GP modifications.
19 * Author: MontaVista Software, Inc.
20 * frank_rowand@mvista.com or source@mvista.com
21 * debbie_chu@mvista.com
22 * Copyright 2002-2005 MontaVista Software, Inc.
23 * PowerPC 44x support, Matt Porter <mporter@kernel.crashing.org>
25 * This program is free software; you can redistribute it and/or modify it
26 * under the terms of the GNU General Public License as published by the
27 * Free Software Foundation; either version 2 of the License, or (at your
28 * option) any later version.
31 #include <linux/init.h>
32 #include <asm/processor.h>
35 #include <asm/pgtable.h>
36 #include <asm/cputable.h>
37 #include <asm/thread_info.h>
38 #include <asm/ppc_asm.h>
39 #include <asm/asm-offsets.h>
40 #include "head_booke.h"
43 /* As with the other PowerPC ports, it is expected that when code
44 * execution begins here, the following registers contain valid, yet
45 * optional, information:
47 * r3 - Board info structure pointer (DRAM, frequency, MAC address, etc.)
48 * r4 - Starting address of the init RAM disk
49 * r5 - Ending address of the init RAM disk
50 * r6 - Start of kernel command line string (e.g. "mem=128")
51 * r7 - End of kernel command line string
58 * Reserve a word at a fixed location to store the address
63 * Save parameters we are passed
70 li r24,0 /* CPU number */
73 * In case the firmware didn't do it, we apply some workarounds
74 * that are good for all 440 core variants here
77 rlwinm r3,r3,0,0,27 /* disable icache prefetch */
84 * Set up the initial MMU state
86 * We are still executing code at the virtual address
87 * mappings set by the firmware for the base of RAM.
89 * We first invalidate all TLB entries but the one
90 * we are running from. We then load the KERNELBASE
91 * mappings so we can begin to use kernel addresses
92 * natively and so the interrupt vector locations are
93 * permanently pinned (necessary since Book E
94 * implementations always have translation enabled).
96 * TODO: Use the known TLB entry we are running from to
97 * determine which physical region we are located
98 * in. This can be used to determine where in RAM
99 * (on a shared CPU system) or PCI memory space
100 * (on a DRAMless system) we are located.
101 * For now, we assume a perfect world which means
102 * we are located at the base of DRAM (physical 0).
106 * Search TLB for entry that we are currently using.
107 * Invalidate all entries but the one we are using.
109 /* Load our current PID->MMUCR TID and MSR IS->MMUCR STS */
110 mfspr r3,SPRN_PID /* Get PID */
111 mfmsr r4 /* Get MSR */
112 andi. r4,r4,MSR_IS@l /* TS=1? */
113 beq wmmucr /* If not, leave STS=0 */
114 oris r3,r3,PPC44x_MMUCR_STS@h /* Set STS=1 */
115 wmmucr: mtspr SPRN_MMUCR,r3 /* Put MMUCR */
118 bl invstr /* Find our address */
119 invstr: mflr r5 /* Make it accessible */
120 tlbsx r23,0,r5 /* Find entry we are in */
121 li r4,0 /* Start at TLB entry 0 */
122 li r3,0 /* Set PAGEID inval value */
123 1: cmpw r23,r4 /* Is this our entry? */
124 beq skpinv /* If so, skip the inval */
125 tlbwe r3,r4,PPC44x_TLB_PAGEID /* If not, inval the entry */
126 skpinv: addi r4,r4,1 /* Increment */
127 cmpwi r4,64 /* Are we done? */
128 bne 1b /* If not, repeat */
129 isync /* If so, context change */
132 * Configure and load pinned entry into TLB slot 63.
136 ori r3,r3,PAGE_OFFSET@l
138 /* Kernel is at the base of RAM */
139 li r4, 0 /* Load the kernel physical address */
141 /* Load the kernel PID = 0 */
146 /* Initialize MMUCR */
152 clrrwi r3,r3,10 /* Mask off the effective page number */
153 ori r3,r3,PPC44x_TLB_VALID | PPC44x_TLB_256M
156 clrrwi r4,r4,10 /* Mask off the real page number */
157 /* ERPN is 0 for first 4GB page */
160 /* Added guarded bit to protect against speculative loads/stores */
162 ori r5,r5,(PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G)
164 li r0,63 /* TLB slot 63 */
166 tlbwe r3,r0,PPC44x_TLB_PAGEID /* Load the pageid fields */
167 tlbwe r4,r0,PPC44x_TLB_XLAT /* Load the translation fields */
168 tlbwe r5,r0,PPC44x_TLB_ATTRIB /* Load the attrib/access fields */
170 /* Force context change */
179 /* If necessary, invalidate original entry we used */
183 tlbwe r6,r23,PPC44x_TLB_PAGEID
187 #ifdef CONFIG_PPC_EARLY_DEBUG_44x
188 /* Add UART mapping for early debug. */
191 lis r3,PPC44x_EARLY_DEBUG_VIRTADDR@h
192 ori r3,r3,PPC44x_TLB_VALID|PPC44x_TLB_TS|PPC44x_TLB_64K
195 lis r4,CONFIG_PPC_EARLY_DEBUG_44x_PHYSLOW@h
196 ori r4,r4,CONFIG_PPC_EARLY_DEBUG_44x_PHYSHIGH
199 li r5,(PPC44x_TLB_SW|PPC44x_TLB_SR|PPC44x_TLB_I|PPC44x_TLB_G)
200 li r0,62 /* TLB slot 0 */
202 tlbwe r3,r0,PPC44x_TLB_PAGEID
203 tlbwe r4,r0,PPC44x_TLB_XLAT
204 tlbwe r5,r0,PPC44x_TLB_ATTRIB
206 /* Force context change */
208 #endif /* CONFIG_PPC_EARLY_DEBUG_44x */
210 /* Establish the interrupt vector offsets */
211 SET_IVOR(0, CriticalInput);
212 SET_IVOR(1, MachineCheck);
213 SET_IVOR(2, DataStorage);
214 SET_IVOR(3, InstructionStorage);
215 SET_IVOR(4, ExternalInput);
216 SET_IVOR(5, Alignment);
217 SET_IVOR(6, Program);
218 SET_IVOR(7, FloatingPointUnavailable);
219 SET_IVOR(8, SystemCall);
220 SET_IVOR(9, AuxillaryProcessorUnavailable);
221 SET_IVOR(10, Decrementer);
222 SET_IVOR(11, FixedIntervalTimer);
223 SET_IVOR(12, WatchdogTimer);
224 SET_IVOR(13, DataTLBError);
225 SET_IVOR(14, InstructionTLBError);
226 SET_IVOR(15, DebugCrit);
228 /* Establish the interrupt vector base */
229 lis r4,interrupt_base@h /* IVPR only uses the high 16-bits */
233 * This is where the main kernel code starts.
238 ori r2,r2,init_task@l
240 /* ptr to current thread */
241 addi r4,r2,THREAD /* init task's THREAD */
242 mtspr SPRN_SPRG_THREAD,r4
245 lis r1,init_thread_union@h
246 ori r1,r1,init_thread_union@l
248 stwu r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
253 * Decide what sort of machine this is and initialize the MMU.
263 /* Setup PTE pointers for the Abatron bdiGDB */
264 lis r6, swapper_pg_dir@h
265 ori r6, r6, swapper_pg_dir@l
266 lis r5, abatron_pteptrs@h
267 ori r5, r5, abatron_pteptrs@l
269 ori r4, r4, KERNELBASE@l
270 stw r5, 0(r4) /* Save abatron_pteptrs at a fixed location */
274 lis r4,start_kernel@h
275 ori r4,r4,start_kernel@l
277 ori r3,r3,MSR_KERNEL@l
280 rfi /* change context and jump to start_kernel */
283 * Interrupt vector entry code
285 * The Book E MMUs are always on so we don't need to handle
286 * interrupts in real mode as with previous PPC processors. In
287 * this case we handle interrupts in the kernel virtual address
290 * Interrupt vectors are dynamically placed relative to the
291 * interrupt prefix as determined by the address of interrupt_base.
292 * The interrupt vectors offsets are programmed using the labels
293 * for each interrupt vector entry.
295 * Interrupt vectors must be aligned on a 16 byte boundary.
296 * We align on a 32 byte cache line boundary for good measure.
300 /* Critical Input Interrupt */
301 CRITICAL_EXCEPTION(0x0100, CriticalInput, unknown_exception)
303 /* Machine Check Interrupt */
304 CRITICAL_EXCEPTION(0x0200, MachineCheck, machine_check_exception)
305 MCHECK_EXCEPTION(0x0210, MachineCheckA, machine_check_exception)
307 /* Data Storage Interrupt */
308 DATA_STORAGE_EXCEPTION
310 /* Instruction Storage Interrupt */
311 INSTRUCTION_STORAGE_EXCEPTION
313 /* External Input Interrupt */
314 EXCEPTION(0x0500, ExternalInput, do_IRQ, EXC_XFER_LITE)
316 /* Alignment Interrupt */
319 /* Program Interrupt */
322 /* Floating Point Unavailable Interrupt */
323 #ifdef CONFIG_PPC_FPU
324 FP_UNAVAILABLE_EXCEPTION
326 EXCEPTION(0x2010, FloatingPointUnavailable, unknown_exception, EXC_XFER_EE)
328 /* System Call Interrupt */
329 START_EXCEPTION(SystemCall)
330 NORMAL_EXCEPTION_PROLOG
331 EXC_XFER_EE_LITE(0x0c00, DoSyscall)
333 /* Auxillary Processor Unavailable Interrupt */
334 EXCEPTION(0x2020, AuxillaryProcessorUnavailable, unknown_exception, EXC_XFER_EE)
336 /* Decrementer Interrupt */
337 DECREMENTER_EXCEPTION
339 /* Fixed Internal Timer Interrupt */
340 /* TODO: Add FIT support */
341 EXCEPTION(0x1010, FixedIntervalTimer, unknown_exception, EXC_XFER_EE)
343 /* Watchdog Timer Interrupt */
344 /* TODO: Add watchdog support */
345 #ifdef CONFIG_BOOKE_WDT
346 CRITICAL_EXCEPTION(0x1020, WatchdogTimer, WatchdogException)
348 CRITICAL_EXCEPTION(0x1020, WatchdogTimer, unknown_exception)
351 /* Data TLB Error Interrupt */
352 START_EXCEPTION(DataTLBError)
353 mtspr SPRN_SPRG_WSCRATCH0, r10 /* Save some working registers */
354 mtspr SPRN_SPRG_WSCRATCH1, r11
355 mtspr SPRN_SPRG_WSCRATCH2, r12
356 mtspr SPRN_SPRG_WSCRATCH3, r13
358 mtspr SPRN_SPRG_WSCRATCH4, r11
359 mfspr r10, SPRN_DEAR /* Get faulting address */
361 /* If we are faulting a kernel address, we have to use the
362 * kernel page tables.
364 lis r11, PAGE_OFFSET@h
367 lis r11, swapper_pg_dir@h
368 ori r11, r11, swapper_pg_dir@l
371 rlwinm r12,r12,0,0,23 /* Clear TID */
375 /* Get the PGD for the current thread */
377 mfspr r11,SPRN_SPRG_THREAD
380 /* Load PID into MMUCR TID */
382 mfspr r13,SPRN_PID /* Get PID */
383 rlwimi r12,r13,0,24,31 /* Set TID */
388 /* Mask of required permission bits. Note that while we
389 * do copy ESR:ST to _PAGE_RW position as trying to write
390 * to an RO page is pretty common, we don't do it with
391 * _PAGE_DIRTY. We could do it, but it's a fairly rare
392 * event so I'd rather take the overhead when it happens
393 * rather than adding an instruction here. We should measure
394 * whether the whole thing is worth it in the first place
395 * as we could avoid loading SPRN_ESR completely in the first
398 * TODO: Is it worth doing that mfspr & rlwimi in the first
399 * place or can we save a couple of instructions here ?
402 li r13,_PAGE_PRESENT|_PAGE_ACCESSED
403 rlwimi r13,r12,10,30,30
406 /* Compute pgdir/pmd offset */
407 rlwinm r12, r10, PPC44x_PGD_OFF_SHIFT, PPC44x_PGD_OFF_MASK_BIT, 29
408 lwzx r11, r12, r11 /* Get pgd/pmd entry */
409 rlwinm. r12, r11, 0, 0, 20 /* Extract pt base address */
410 beq 2f /* Bail if no table */
412 /* Compute pte address */
413 rlwimi r12, r10, PPC44x_PTE_ADD_SHIFT, PPC44x_PTE_ADD_MASK_BIT, 28
414 lwz r11, 0(r12) /* Get high word of pte entry */
415 lwz r12, 4(r12) /* Get low word of pte entry */
417 lis r10,tlb_44x_index@ha
419 andc. r13,r13,r12 /* Check permission */
421 /* Load the next available TLB index */
422 lwz r13,tlb_44x_index@l(r10)
424 bne 2f /* Bail if permission mismach */
426 /* Increment, rollover, and store TLB index */
429 /* Compare with watermark (instruction gets patched) */
430 .globl tlb_44x_patch_hwater_D
431 tlb_44x_patch_hwater_D:
432 cmpwi 0,r13,1 /* reserve entries */
436 /* Store the next available TLB index */
437 stw r13,tlb_44x_index@l(r10)
439 /* Re-load the faulting address */
442 /* Jump to common tlb load */
446 /* The bailout. Restore registers to pre-exception conditions
447 * and call the heavyweights to help us out.
449 mfspr r11, SPRN_SPRG_RSCRATCH4
451 mfspr r13, SPRN_SPRG_RSCRATCH3
452 mfspr r12, SPRN_SPRG_RSCRATCH2
453 mfspr r11, SPRN_SPRG_RSCRATCH1
454 mfspr r10, SPRN_SPRG_RSCRATCH0
457 /* Instruction TLB Error Interrupt */
459 * Nearly the same as above, except we get our
460 * information from different registers and bailout
461 * to a different point.
463 START_EXCEPTION(InstructionTLBError)
464 mtspr SPRN_SPRG_WSCRATCH0, r10 /* Save some working registers */
465 mtspr SPRN_SPRG_WSCRATCH1, r11
466 mtspr SPRN_SPRG_WSCRATCH2, r12
467 mtspr SPRN_SPRG_WSCRATCH3, r13
469 mtspr SPRN_SPRG_WSCRATCH4, r11
470 mfspr r10, SPRN_SRR0 /* Get faulting address */
472 /* If we are faulting a kernel address, we have to use the
473 * kernel page tables.
475 lis r11, PAGE_OFFSET@h
478 lis r11, swapper_pg_dir@h
479 ori r11, r11, swapper_pg_dir@l
482 rlwinm r12,r12,0,0,23 /* Clear TID */
486 /* Get the PGD for the current thread */
488 mfspr r11,SPRN_SPRG_THREAD
491 /* Load PID into MMUCR TID */
493 mfspr r13,SPRN_PID /* Get PID */
494 rlwimi r12,r13,0,24,31 /* Set TID */
499 /* Make up the required permissions */
500 li r13,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_HWEXEC
502 /* Compute pgdir/pmd offset */
503 rlwinm r12, r10, PPC44x_PGD_OFF_SHIFT, PPC44x_PGD_OFF_MASK_BIT, 29
504 lwzx r11, r12, r11 /* Get pgd/pmd entry */
505 rlwinm. r12, r11, 0, 0, 20 /* Extract pt base address */
506 beq 2f /* Bail if no table */
508 /* Compute pte address */
509 rlwimi r12, r10, PPC44x_PTE_ADD_SHIFT, PPC44x_PTE_ADD_MASK_BIT, 28
510 lwz r11, 0(r12) /* Get high word of pte entry */
511 lwz r12, 4(r12) /* Get low word of pte entry */
513 lis r10,tlb_44x_index@ha
515 andc. r13,r13,r12 /* Check permission */
517 /* Load the next available TLB index */
518 lwz r13,tlb_44x_index@l(r10)
520 bne 2f /* Bail if permission mismach */
522 /* Increment, rollover, and store TLB index */
525 /* Compare with watermark (instruction gets patched) */
526 .globl tlb_44x_patch_hwater_I
527 tlb_44x_patch_hwater_I:
528 cmpwi 0,r13,1 /* reserve entries */
532 /* Store the next available TLB index */
533 stw r13,tlb_44x_index@l(r10)
535 /* Re-load the faulting address */
538 /* Jump to common TLB load point */
542 /* The bailout. Restore registers to pre-exception conditions
543 * and call the heavyweights to help us out.
545 mfspr r11, SPRN_SPRG_RSCRATCH4
547 mfspr r13, SPRN_SPRG_RSCRATCH3
548 mfspr r12, SPRN_SPRG_RSCRATCH2
549 mfspr r11, SPRN_SPRG_RSCRATCH1
550 mfspr r10, SPRN_SPRG_RSCRATCH0
553 /* Debug Interrupt */
562 * Both the instruction and data TLB miss get to this
563 * point to load the TLB.
565 * r11 - PTE high word value
566 * r12 - PTE low word value
568 * MMUCR - loaded with proper value when we get here
569 * Upon exit, we reload everything and RFI.
572 /* Combine RPN & ERPN an write WS 0 */
573 rlwimi r11,r12,0,0,31-PAGE_SHIFT
574 tlbwe r11,r13,PPC44x_TLB_XLAT
577 * Create WS1. This is the faulting address (EPN),
578 * page size, and valid flag.
580 li r11,PPC44x_TLB_VALID | PPC44x_TLBE_SIZE
581 /* Insert valid and page size */
582 rlwimi r10,r11,0,PPC44x_PTE_ADD_MASK_BIT,31
583 tlbwe r10,r13,PPC44x_TLB_PAGEID /* Write PAGEID */
586 li r10,0xf85 /* Mask to apply from PTE */
587 rlwimi r10,r12,29,30,30 /* DIRTY -> SW position */
588 and r11,r12,r10 /* Mask PTE bits to keep */
589 andi. r10,r12,_PAGE_USER /* User page ? */
590 beq 1f /* nope, leave U bits empty */
591 rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */
592 1: tlbwe r11,r13,PPC44x_TLB_ATTRIB /* Write ATTRIB */
594 /* Done...restore registers and get out of here.
596 mfspr r11, SPRN_SPRG_RSCRATCH4
598 mfspr r13, SPRN_SPRG_RSCRATCH3
599 mfspr r12, SPRN_SPRG_RSCRATCH2
600 mfspr r11, SPRN_SPRG_RSCRATCH1
601 mfspr r10, SPRN_SPRG_RSCRATCH0
602 rfi /* Force context change */
609 * Adjust the machine check IVOR on 440A cores
611 _GLOBAL(__fixup_440A_mcheck)
612 li r3,MachineCheckA@l
618 * extern void giveup_altivec(struct task_struct *prev)
620 * The 44x core does not have an AltiVec unit.
622 _GLOBAL(giveup_altivec)
626 * extern void giveup_fpu(struct task_struct *prev)
628 * The 44x core does not have an FPU.
630 #ifndef CONFIG_PPC_FPU
637 #ifdef CONFIG_BDI_SWITCH
638 /* Context switch the PTE pointer for the Abatron BDI2000.
639 * The PGDIR is the second parameter.
641 lis r5, abatron_pteptrs@h
642 ori r5, r5, abatron_pteptrs@l
646 isync /* Force context change */
650 * We put a few things here that have to be page-aligned. This stuff
651 * goes at the beginning of the data segment, which is page-aligned.
657 .globl empty_zero_page
662 * To support >32-bit physical addresses, we use an 8KB pgdir.
664 .globl swapper_pg_dir
666 .space PGD_TABLE_SIZE
669 * Room for two PTE pointers, usually the kernel and current user pointers
670 * to their respective root page table.