| blob | 56585086413a4f9876c70eb6b6ea9f3cf1c1ef69 |
1 /*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
22 .text
24 /*
25 * Stackframe:
26 *
27 * +-> Back chain (SP + 256)
28 * | General register save area (SP + 112)
29 * | Parameter save area (SP + 48)
30 * | TOC save area (SP + 40)
31 * | link editor doubleword (SP + 32)
32 * | compiler doubleword (SP + 24)
33 * | LR save area (SP + 16)
34 * | CR save area (SP + 8)
35 * SP ---> +-- Back chain (SP + 0)
36 */
38 #ifndef CONFIG_PPC_64K_PAGES
40 /*****************************************************************************
41 * *
42 * 4K SW & 4K HW pages implementation *
43 * *
44 *****************************************************************************/
47 /*
48 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
49 * pte_t *ptep, unsigned long trap, int local, int ssize)
50 *
51 * Adds a 4K page to the hash table in a segment of 4K pages only
52 */
54 _GLOBAL(__hash_page_4K)
55 mflr r0
56 std r0,16(r1)
57 stdu r1,-STACKFRAMESIZE(r1)
58 /* Save all params that we need after a function call */
59 std r6,STK_PARAM(R6)(r1)
60 std r8,STK_PARAM(R8)(r1)
61 std r9,STK_PARAM(R9)(r1)
63 /* Save non-volatile registers.
64 * r31 will hold "old PTE"
65 * r30 is "new PTE"
66 * r29 is vpn
67 * r28 is a hash value
68 * r27 is hashtab mask (maybe dynamic patched instead ?)
69 */
70 std r27,STK_REG(R27)(r1)
71 std r28,STK_REG(R28)(r1)
72 std r29,STK_REG(R29)(r1)
73 std r30,STK_REG(R30)(r1)
74 std r31,STK_REG(R31)(r1)
76 /* Step 1:
77 *
78 * Check permissions, atomically mark the linux PTE busy
79 * and hashed.
80 */
81 1:
82 ldarx r31,0,r6
83 /* Check access rights (access & ~(pte_val(*ptep))) */
84 andc. r0,r4,r31
85 bne- htab_wrong_access
86 /* Check if PTE is busy */
87 andi. r0,r31,_PAGE_BUSY
88 /* If so, just bail out and refault if needed. Someone else
89 * is changing this PTE anyway and might hash it.
90 */
91 bne- htab_bail_ok
93 /* Prepare new PTE value (turn access RW into DIRTY, then
94 * add BUSY,HASHPTE and ACCESSED)
95 */
96 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
97 or r30,r30,r31
98 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
99 /* Write the linux PTE atomically (setting busy) */
100 stdcx. r30,0,r6
101 bne- 1b
102 isync
104 /* Step 2:
105 *
106 * Insert/Update the HPTE in the hash table. At this point,
107 * r4 (access) is re-useable, we use it for the new HPTE flags
108 */
110 BEGIN_FTR_SECTION
111 cmpdi r9,0 /* check segment size */
112 bne 3f
113 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
114 /* Calc vpn and put it in r29 */
115 sldi r29,r5,SID_SHIFT - VPN_SHIFT
116 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
117 or r29,r28,r29
119 /* Calculate hash value for primary slot and store it in r28 */
120 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
121 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
122 xor r28,r5,r0
123 b 4f
125 3: /* Calc vpn and put it in r29 */
126 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT
127 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
128 or r29,r28,r29
130 /*
131 * calculate hash value for primary slot and
132 * store it in r28 for 1T segment
133 */
134 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
135 clrldi r5,r5,40 /* vsid & 0xffffff */
136 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
137 xor r28,r28,r5
138 xor r28,r28,r0 /* hash */
140 /* Convert linux PTE bits into HW equivalents */
141 4: andi. r3,r30,0x1fe /* Get basic set of flags */
142 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
143 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
144 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
145 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
146 andc r0,r30,r0 /* r0 = pte & ~r0 */
147 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
148 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
150 /* We eventually do the icache sync here (maybe inline that
151 * code rather than call a C function...)
152 */
153 BEGIN_FTR_SECTION
154 mr r4,r30
155 mr r5,r7
156 bl .hash_page_do_lazy_icache
157 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
159 /* At this point, r3 contains new PP bits, save them in
160 * place of "access" in the param area (sic)
161 */
162 std r3,STK_PARAM(R4)(r1)
164 /* Get htab_hash_mask */
165 ld r4,htab_hash_mask@got(2)
166 ld r27,0(r4) /* htab_hash_mask -> r27 */
168 /* Check if we may already be in the hashtable, in this case, we
169 * go to out-of-line code to try to modify the HPTE
170 */
171 andi. r0,r31,_PAGE_HASHPTE
172 bne htab_modify_pte
174 htab_insert_pte:
175 /* Clear hpte bits in new pte (we also clear BUSY btw) and
176 * add _PAGE_HASHPTE
177 */
178 lis r0,_PAGE_HPTEFLAGS@h
179 ori r0,r0,_PAGE_HPTEFLAGS@l
180 andc r30,r30,r0
181 ori r30,r30,_PAGE_HASHPTE
183 /* physical address r5 */
184 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
185 sldi r5,r5,PAGE_SHIFT
187 /* Calculate primary group hash */
188 and r0,r28,r27
189 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
191 /* Call ppc_md.hpte_insert */
192 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
193 mr r4,r29 /* Retrieve vpn */
194 li r7,0 /* !bolted, !secondary */
195 li r8,MMU_PAGE_4K /* page size */
196 ld r9,STK_PARAM(R9)(r1) /* segment size */
197 _GLOBAL(htab_call_hpte_insert1)
198 bl . /* Patched by htab_finish_init() */
199 cmpdi 0,r3,0
200 bge htab_pte_insert_ok /* Insertion successful */
201 cmpdi 0,r3,-2 /* Critical failure */
202 beq- htab_pte_insert_failure
204 /* Now try secondary slot */
206 /* physical address r5 */
207 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
208 sldi r5,r5,PAGE_SHIFT
210 /* Calculate secondary group hash */
211 andc r0,r27,r28
212 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
214 /* Call ppc_md.hpte_insert */
215 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
216 mr r4,r29 /* Retrieve vpn */
217 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
218 li r8,MMU_PAGE_4K /* page size */
219 ld r9,STK_PARAM(R9)(r1) /* segment size */
220 _GLOBAL(htab_call_hpte_insert2)
221 bl . /* Patched by htab_finish_init() */
222 cmpdi 0,r3,0
223 bge+ htab_pte_insert_ok /* Insertion successful */
224 cmpdi 0,r3,-2 /* Critical failure */
225 beq- htab_pte_insert_failure
227 /* Both are full, we need to evict something */
228 mftb r0
229 /* Pick a random group based on TB */
230 andi. r0,r0,1
231 mr r5,r28
232 bne 2f
233 not r5,r5
234 2: and r0,r5,r27
235 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
236 /* Call ppc_md.hpte_remove */
237 _GLOBAL(htab_call_hpte_remove)
238 bl . /* Patched by htab_finish_init() */
240 /* Try all again */
241 b htab_insert_pte
243 htab_bail_ok:
244 li r3,0
245 b htab_bail
247 htab_pte_insert_ok:
248 /* Insert slot number & secondary bit in PTE */
249 rldimi r30,r3,12,63-15
251 /* Write out the PTE with a normal write
252 * (maybe add eieio may be good still ?)
253 */
254 htab_write_out_pte:
255 ld r6,STK_PARAM(R6)(r1)
256 std r30,0(r6)
257 li r3, 0
258 htab_bail:
259 ld r27,STK_REG(R27)(r1)
260 ld r28,STK_REG(R28)(r1)
261 ld r29,STK_REG(R29)(r1)
262 ld r30,STK_REG(R30)(r1)
263 ld r31,STK_REG(R31)(r1)
264 addi r1,r1,STACKFRAMESIZE
265 ld r0,16(r1)
266 mtlr r0
267 blr
269 htab_modify_pte:
270 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
271 mr r4,r3
272 rlwinm r3,r31,32-12,29,31
274 /* Secondary group ? if yes, get a inverted hash value */
275 mr r5,r28
276 andi. r0,r31,_PAGE_SECONDARY
277 beq 1f
278 not r5,r5
279 1:
280 /* Calculate proper slot value for ppc_md.hpte_updatepp */
281 and r0,r5,r27
282 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
283 add r3,r0,r3 /* add slot idx */
285 /* Call ppc_md.hpte_updatepp */
286 mr r5,r29 /* vpn */
287 li r6,MMU_PAGE_4K /* page size */
288 ld r7,STK_PARAM(R9)(r1) /* segment size */
289 ld r8,STK_PARAM(R8)(r1) /* get "local" param */
290 _GLOBAL(htab_call_hpte_updatepp)
291 bl . /* Patched by htab_finish_init() */
293 /* if we failed because typically the HPTE wasn't really here
294 * we try an insertion.
295 */
296 cmpdi 0,r3,-1
297 beq- htab_insert_pte
299 /* Clear the BUSY bit and Write out the PTE */
300 li r0,_PAGE_BUSY
301 andc r30,r30,r0
302 b htab_write_out_pte
304 htab_wrong_access:
305 /* Bail out clearing reservation */
306 stdcx. r31,0,r6
307 li r3,1
308 b htab_bail
310 htab_pte_insert_failure:
311 /* Bail out restoring old PTE */
312 ld r6,STK_PARAM(R6)(r1)
313 std r31,0(r6)
314 li r3,-1
315 b htab_bail
318 #else /* CONFIG_PPC_64K_PAGES */
321 /*****************************************************************************
322 * *
323 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
324 * *
325 *****************************************************************************/
327 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
328 * pte_t *ptep, unsigned long trap, int local, int ssize,
329 * int subpg_prot)
330 */
332 /*
333 * For now, we do NOT implement Admixed pages
334 */
335 _GLOBAL(__hash_page_4K)
336 mflr r0
337 std r0,16(r1)
338 stdu r1,-STACKFRAMESIZE(r1)
339 /* Save all params that we need after a function call */
340 std r6,STK_PARAM(R6)(r1)
341 std r8,STK_PARAM(R8)(r1)
342 std r9,STK_PARAM(R9)(r1)
344 /* Save non-volatile registers.
345 * r31 will hold "old PTE"
346 * r30 is "new PTE"
347 * r29 is vpn
348 * r28 is a hash value
349 * r27 is hashtab mask (maybe dynamic patched instead ?)
350 * r26 is the hidx mask
351 * r25 is the index in combo page
352 */
353 std r25,STK_REG(R25)(r1)
354 std r26,STK_REG(R26)(r1)
355 std r27,STK_REG(R27)(r1)
356 std r28,STK_REG(R28)(r1)
357 std r29,STK_REG(R29)(r1)
358 std r30,STK_REG(R30)(r1)
359 std r31,STK_REG(R31)(r1)
361 /* Step 1:
362 *
363 * Check permissions, atomically mark the linux PTE busy
364 * and hashed.
365 */
366 1:
367 ldarx r31,0,r6
368 /* Check access rights (access & ~(pte_val(*ptep))) */
369 andc. r0,r4,r31
370 bne- htab_wrong_access
371 /* Check if PTE is busy */
372 andi. r0,r31,_PAGE_BUSY
373 /* If so, just bail out and refault if needed. Someone else
374 * is changing this PTE anyway and might hash it.
375 */
376 bne- htab_bail_ok
377 /* Prepare new PTE value (turn access RW into DIRTY, then
378 * add BUSY and ACCESSED)
379 */
380 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
381 or r30,r30,r31
382 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
383 oris r30,r30,_PAGE_COMBO@h
384 /* Write the linux PTE atomically (setting busy) */
385 stdcx. r30,0,r6
386 bne- 1b
387 isync
389 /* Step 2:
390 *
391 * Insert/Update the HPTE in the hash table. At this point,
392 * r4 (access) is re-useable, we use it for the new HPTE flags
393 */
395 /* Load the hidx index */
396 rldicl r25,r3,64-12,60
398 BEGIN_FTR_SECTION
399 cmpdi r9,0 /* check segment size */
400 bne 3f
401 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
402 /* Calc vpn and put it in r29 */
403 sldi r29,r5,SID_SHIFT - VPN_SHIFT
404 /*
405 * clrldi r3,r3,64 - SID_SHIFT --> ea & 0xfffffff
406 * srdi r28,r3,VPN_SHIFT
407 */
408 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
409 or r29,r28,r29
411 /* Calculate hash value for primary slot and store it in r28 */
412 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
413 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
414 xor r28,r5,r0
415 b 4f
417 3: /* Calc vpn and put it in r29 */
418 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT
419 /*
420 * clrldi r3,r3,64 - SID_SHIFT_1T --> ea & 0xffffffffff
421 * srdi r28,r3,VPN_SHIFT
422 */
423 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
424 or r29,r28,r29
426 /*
427 * Calculate hash value for primary slot and
428 * store it in r28 for 1T segment
429 */
430 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
431 clrldi r5,r5,40 /* vsid & 0xffffff */
432 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
433 xor r28,r28,r5
434 xor r28,r28,r0 /* hash */
436 /* Convert linux PTE bits into HW equivalents */
437 4:
438 #ifdef CONFIG_PPC_SUBPAGE_PROT
439 andc r10,r30,r10
440 andi. r3,r10,0x1fe /* Get basic set of flags */
441 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
442 #else
443 andi. r3,r30,0x1fe /* Get basic set of flags */
444 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
445 #endif
446 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
447 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
448 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
449 andc r0,r3,r0 /* r0 = pte & ~r0 */
450 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
451 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
453 /* We eventually do the icache sync here (maybe inline that
454 * code rather than call a C function...)
455 */
456 BEGIN_FTR_SECTION
457 mr r4,r30
458 mr r5,r7
459 bl .hash_page_do_lazy_icache
460 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
462 /* At this point, r3 contains new PP bits, save them in
463 * place of "access" in the param area (sic)
464 */
465 std r3,STK_PARAM(R4)(r1)
467 /* Get htab_hash_mask */
468 ld r4,htab_hash_mask@got(2)
469 ld r27,0(r4) /* htab_hash_mask -> r27 */
471 /* Check if we may already be in the hashtable, in this case, we
472 * go to out-of-line code to try to modify the HPTE. We look for
473 * the bit at (1 >> (index + 32))
474 */
475 rldicl. r0,r31,64-12,48
476 li r26,0 /* Default hidx */
477 beq htab_insert_pte
479 /*
480 * Check if the pte was already inserted into the hash table
481 * as a 64k HW page, and invalidate the 64k HPTE if so.
482 */
483 andis. r0,r31,_PAGE_COMBO@h
484 beq htab_inval_old_hpte
486 ld r6,STK_PARAM(R6)(r1)
487 ori r26,r6,0x8000 /* Load the hidx mask */
488 ld r26,0(r26)
489 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
490 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
491 bne htab_modify_pte
493 htab_insert_pte:
494 /* real page number in r5, PTE RPN value + index */
495 andis. r0,r31,_PAGE_4K_PFN@h
496 srdi r5,r31,PTE_RPN_SHIFT
497 bne- htab_special_pfn
498 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
499 add r5,r5,r25
500 htab_special_pfn:
501 sldi r5,r5,HW_PAGE_SHIFT
503 /* Calculate primary group hash */
504 and r0,r28,r27
505 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
507 /* Call ppc_md.hpte_insert */
508 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
509 mr r4,r29 /* Retrieve vpn */
510 li r7,0 /* !bolted, !secondary */
511 li r8,MMU_PAGE_4K /* page size */
512 ld r9,STK_PARAM(R9)(r1) /* segment size */
513 _GLOBAL(htab_call_hpte_insert1)
514 bl . /* patched by htab_finish_init() */
515 cmpdi 0,r3,0
516 bge htab_pte_insert_ok /* Insertion successful */
517 cmpdi 0,r3,-2 /* Critical failure */
518 beq- htab_pte_insert_failure
520 /* Now try secondary slot */
522 /* real page number in r5, PTE RPN value + index */
523 andis. r0,r31,_PAGE_4K_PFN@h
524 srdi r5,r31,PTE_RPN_SHIFT
525 bne- 3f
526 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
527 add r5,r5,r25
528 3: sldi r5,r5,HW_PAGE_SHIFT
530 /* Calculate secondary group hash */
531 andc r0,r27,r28
532 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
534 /* Call ppc_md.hpte_insert */
535 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
536 mr r4,r29 /* Retrieve vpn */
537 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
538 li r8,MMU_PAGE_4K /* page size */
539 ld r9,STK_PARAM(R9)(r1) /* segment size */
540 _GLOBAL(htab_call_hpte_insert2)
541 bl . /* patched by htab_finish_init() */
542 cmpdi 0,r3,0
543 bge+ htab_pte_insert_ok /* Insertion successful */
544 cmpdi 0,r3,-2 /* Critical failure */
545 beq- htab_pte_insert_failure
547 /* Both are full, we need to evict something */
548 mftb r0
549 /* Pick a random group based on TB */
550 andi. r0,r0,1
551 mr r5,r28
552 bne 2f
553 not r5,r5
554 2: and r0,r5,r27
555 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
556 /* Call ppc_md.hpte_remove */
557 _GLOBAL(htab_call_hpte_remove)
558 bl . /* patched by htab_finish_init() */
560 /* Try all again */
561 b htab_insert_pte
563 /*
564 * Call out to C code to invalidate an 64k HW HPTE that is
565 * useless now that the segment has been switched to 4k pages.
566 */
567 htab_inval_old_hpte:
568 mr r3,r29 /* vpn */
569 mr r4,r31 /* PTE.pte */
570 li r5,0 /* PTE.hidx */
571 li r6,MMU_PAGE_64K /* psize */
572 ld r7,STK_PARAM(R9)(r1) /* ssize */
573 ld r8,STK_PARAM(R8)(r1) /* local */
574 bl .flush_hash_page
575 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
576 lis r0,_PAGE_HPTE_SUB@h
577 ori r0,r0,_PAGE_HPTE_SUB@l
578 andc r30,r30,r0
579 b htab_insert_pte
581 htab_bail_ok:
582 li r3,0
583 b htab_bail
585 htab_pte_insert_ok:
586 /* Insert slot number & secondary bit in PTE second half,
587 * clear _PAGE_BUSY and set approriate HPTE slot bit
588 */
589 ld r6,STK_PARAM(R6)(r1)
590 li r0,_PAGE_BUSY
591 andc r30,r30,r0
592 /* HPTE SUB bit */
593 li r0,1
594 subfic r5,r25,27 /* Must match bit position in */
595 sld r0,r0,r5 /* pgtable.h */
596 or r30,r30,r0
597 /* hindx */
598 sldi r5,r25,2
599 sld r3,r3,r5
600 li r4,0xf
601 sld r4,r4,r5
602 andc r26,r26,r4
603 or r26,r26,r3
604 ori r5,r6,0x8000
605 std r26,0(r5)
606 lwsync
607 std r30,0(r6)
608 li r3, 0
609 htab_bail:
610 ld r25,STK_REG(R25)(r1)
611 ld r26,STK_REG(R26)(r1)
612 ld r27,STK_REG(R27)(r1)
613 ld r28,STK_REG(R28)(r1)
614 ld r29,STK_REG(R29)(r1)
615 ld r30,STK_REG(R30)(r1)
616 ld r31,STK_REG(R31)(r1)
617 addi r1,r1,STACKFRAMESIZE
618 ld r0,16(r1)
619 mtlr r0
620 blr
622 htab_modify_pte:
623 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
624 mr r4,r3
625 sldi r5,r25,2
626 srd r3,r26,r5
628 /* Secondary group ? if yes, get a inverted hash value */
629 mr r5,r28
630 andi. r0,r3,0x8 /* page secondary ? */
631 beq 1f
632 not r5,r5
633 1: andi. r3,r3,0x7 /* extract idx alone */
635 /* Calculate proper slot value for ppc_md.hpte_updatepp */
636 and r0,r5,r27
637 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
638 add r3,r0,r3 /* add slot idx */
640 /* Call ppc_md.hpte_updatepp */
641 mr r5,r29 /* vpn */
642 li r6,MMU_PAGE_4K /* page size */
643 ld r7,STK_PARAM(R9)(r1) /* segment size */
644 ld r8,STK_PARAM(R8)(r1) /* get "local" param */
645 _GLOBAL(htab_call_hpte_updatepp)
646 bl . /* patched by htab_finish_init() */
648 /* if we failed because typically the HPTE wasn't really here
649 * we try an insertion.
650 */
651 cmpdi 0,r3,-1
652 beq- htab_insert_pte
654 /* Clear the BUSY bit and Write out the PTE */
655 li r0,_PAGE_BUSY
656 andc r30,r30,r0
657 ld r6,STK_PARAM(R6)(r1)
658 std r30,0(r6)
659 li r3,0
660 b htab_bail
662 htab_wrong_access:
663 /* Bail out clearing reservation */
664 stdcx. r31,0,r6
665 li r3,1
666 b htab_bail
668 htab_pte_insert_failure:
669 /* Bail out restoring old PTE */
670 ld r6,STK_PARAM(R6)(r1)
671 std r31,0(r6)
672 li r3,-1
673 b htab_bail
675 #endif /* CONFIG_PPC_64K_PAGES */
677 #ifdef CONFIG_PPC_HAS_HASH_64K
679 /*****************************************************************************
680 * *
681 * 64K SW & 64K HW in a 64K segment pages implementation *
682 * *
683 *****************************************************************************/
685 _GLOBAL(__hash_page_64K)
686 mflr r0
687 std r0,16(r1)
688 stdu r1,-STACKFRAMESIZE(r1)
689 /* Save all params that we need after a function call */
690 std r6,STK_PARAM(R6)(r1)
691 std r8,STK_PARAM(R8)(r1)
692 std r9,STK_PARAM(R9)(r1)
694 /* Save non-volatile registers.
695 * r31 will hold "old PTE"
696 * r30 is "new PTE"
697 * r29 is vpn
698 * r28 is a hash value
699 * r27 is hashtab mask (maybe dynamic patched instead ?)
700 */
701 std r27,STK_REG(R27)(r1)
702 std r28,STK_REG(R28)(r1)
703 std r29,STK_REG(R29)(r1)
704 std r30,STK_REG(R30)(r1)
705 std r31,STK_REG(R31)(r1)
707 /* Step 1:
708 *
709 * Check permissions, atomically mark the linux PTE busy
710 * and hashed.
711 */
712 1:
713 ldarx r31,0,r6
714 /* Check access rights (access & ~(pte_val(*ptep))) */
715 andc. r0,r4,r31
716 bne- ht64_wrong_access
717 /* Check if PTE is busy */
718 andi. r0,r31,_PAGE_BUSY
719 /* If so, just bail out and refault if needed. Someone else
720 * is changing this PTE anyway and might hash it.
721 */
722 bne- ht64_bail_ok
723 BEGIN_FTR_SECTION
724 /* Check if PTE has the cache-inhibit bit set */
725 andi. r0,r31,_PAGE_NO_CACHE
726 /* If so, bail out and refault as a 4k page */
727 bne- ht64_bail_ok
728 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
729 /* Prepare new PTE value (turn access RW into DIRTY, then
730 * add BUSY and ACCESSED)
731 */
732 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
733 or r30,r30,r31
734 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
735 /* Write the linux PTE atomically (setting busy) */
736 stdcx. r30,0,r6
737 bne- 1b
738 isync
740 /* Step 2:
741 *
742 * Insert/Update the HPTE in the hash table. At this point,
743 * r4 (access) is re-useable, we use it for the new HPTE flags
744 */
746 BEGIN_FTR_SECTION
747 cmpdi r9,0 /* check segment size */
748 bne 3f
749 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
750 /* Calc vpn and put it in r29 */
751 sldi r29,r5,SID_SHIFT - VPN_SHIFT
752 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
753 or r29,r28,r29
755 /* Calculate hash value for primary slot and store it in r28 */
756 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
757 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
758 xor r28,r5,r0
759 b 4f
761 3: /* Calc vpn and put it in r29 */
762 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT
763 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
764 or r29,r28,r29
766 /*
767 * calculate hash value for primary slot and
768 * store it in r28 for 1T segment
769 */
770 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
771 clrldi r5,r5,40 /* vsid & 0xffffff */
772 rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */
773 xor r28,r28,r5
774 xor r28,r28,r0 /* hash */
776 /* Convert linux PTE bits into HW equivalents */
777 4: andi. r3,r30,0x1fe /* Get basic set of flags */
778 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
779 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
780 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
781 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
782 andc r0,r30,r0 /* r0 = pte & ~r0 */
783 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
784 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
786 /* We eventually do the icache sync here (maybe inline that
787 * code rather than call a C function...)
788 */
789 BEGIN_FTR_SECTION
790 mr r4,r30
791 mr r5,r7
792 bl .hash_page_do_lazy_icache
793 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
795 /* At this point, r3 contains new PP bits, save them in
796 * place of "access" in the param area (sic)
797 */
798 std r3,STK_PARAM(R4)(r1)
800 /* Get htab_hash_mask */
801 ld r4,htab_hash_mask@got(2)
802 ld r27,0(r4) /* htab_hash_mask -> r27 */
804 /* Check if we may already be in the hashtable, in this case, we
805 * go to out-of-line code to try to modify the HPTE
806 */
807 rldicl. r0,r31,64-12,48
808 bne ht64_modify_pte
810 ht64_insert_pte:
811 /* Clear hpte bits in new pte (we also clear BUSY btw) and
812 * add _PAGE_HPTE_SUB0
813 */
814 lis r0,_PAGE_HPTEFLAGS@h
815 ori r0,r0,_PAGE_HPTEFLAGS@l
816 andc r30,r30,r0
817 #ifdef CONFIG_PPC_64K_PAGES
818 oris r30,r30,_PAGE_HPTE_SUB0@h
819 #else
820 ori r30,r30,_PAGE_HASHPTE
821 #endif
822 /* Phyical address in r5 */
823 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
824 sldi r5,r5,PAGE_SHIFT
826 /* Calculate primary group hash */
827 and r0,r28,r27
828 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
830 /* Call ppc_md.hpte_insert */
831 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
832 mr r4,r29 /* Retrieve vpn */
833 li r7,0 /* !bolted, !secondary */
834 li r8,MMU_PAGE_64K
835 ld r9,STK_PARAM(R9)(r1) /* segment size */
836 _GLOBAL(ht64_call_hpte_insert1)
837 bl . /* patched by htab_finish_init() */
838 cmpdi 0,r3,0
839 bge ht64_pte_insert_ok /* Insertion successful */
840 cmpdi 0,r3,-2 /* Critical failure */
841 beq- ht64_pte_insert_failure
843 /* Now try secondary slot */
845 /* Phyical address in r5 */
846 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
847 sldi r5,r5,PAGE_SHIFT
849 /* Calculate secondary group hash */
850 andc r0,r27,r28
851 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
853 /* Call ppc_md.hpte_insert */
854 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */
855 mr r4,r29 /* Retrieve vpn */
856 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
857 li r8,MMU_PAGE_64K
858 ld r9,STK_PARAM(R9)(r1) /* segment size */
859 _GLOBAL(ht64_call_hpte_insert2)
860 bl . /* patched by htab_finish_init() */
861 cmpdi 0,r3,0
862 bge+ ht64_pte_insert_ok /* Insertion successful */
863 cmpdi 0,r3,-2 /* Critical failure */
864 beq- ht64_pte_insert_failure
866 /* Both are full, we need to evict something */
867 mftb r0
868 /* Pick a random group based on TB */
869 andi. r0,r0,1
870 mr r5,r28
871 bne 2f
872 not r5,r5
873 2: and r0,r5,r27
874 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
875 /* Call ppc_md.hpte_remove */
876 _GLOBAL(ht64_call_hpte_remove)
877 bl . /* patched by htab_finish_init() */
879 /* Try all again */
880 b ht64_insert_pte
882 ht64_bail_ok:
883 li r3,0
884 b ht64_bail
886 ht64_pte_insert_ok:
887 /* Insert slot number & secondary bit in PTE */
888 rldimi r30,r3,12,63-15
890 /* Write out the PTE with a normal write
891 * (maybe add eieio may be good still ?)
892 */
893 ht64_write_out_pte:
894 ld r6,STK_PARAM(R6)(r1)
895 std r30,0(r6)
896 li r3, 0
897 ht64_bail:
898 ld r27,STK_REG(R27)(r1)
899 ld r28,STK_REG(R28)(r1)
900 ld r29,STK_REG(R29)(r1)
901 ld r30,STK_REG(R30)(r1)
902 ld r31,STK_REG(R31)(r1)
903 addi r1,r1,STACKFRAMESIZE
904 ld r0,16(r1)
905 mtlr r0
906 blr
908 ht64_modify_pte:
909 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
910 mr r4,r3
911 rlwinm r3,r31,32-12,29,31
913 /* Secondary group ? if yes, get a inverted hash value */
914 mr r5,r28
915 andi. r0,r31,_PAGE_F_SECOND
916 beq 1f
917 not r5,r5
918 1:
919 /* Calculate proper slot value for ppc_md.hpte_updatepp */
920 and r0,r5,r27
921 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
922 add r3,r0,r3 /* add slot idx */
924 /* Call ppc_md.hpte_updatepp */
925 mr r5,r29 /* vpn */
926 li r6,MMU_PAGE_64K
927 ld r7,STK_PARAM(R9)(r1) /* segment size */
928 ld r8,STK_PARAM(R8)(r1) /* get "local" param */
929 _GLOBAL(ht64_call_hpte_updatepp)
930 bl . /* patched by htab_finish_init() */
932 /* if we failed because typically the HPTE wasn't really here
933 * we try an insertion.
934 */
935 cmpdi 0,r3,-1
936 beq- ht64_insert_pte
938 /* Clear the BUSY bit and Write out the PTE */
939 li r0,_PAGE_BUSY
940 andc r30,r30,r0
941 b ht64_write_out_pte
943 ht64_wrong_access:
944 /* Bail out clearing reservation */
945 stdcx. r31,0,r6
946 li r3,1
947 b ht64_bail
949 ht64_pte_insert_failure:
950 /* Bail out restoring old PTE */
951 ld r6,STK_PARAM(R6)(r1)
952 std r31,0(r6)
953 li r3,-1
954 b ht64_bail
957 #endif /* CONFIG_PPC_HAS_HASH_64K */
960 /*****************************************************************************
961 * *
962 * Huge pages implementation is in hugetlbpage.c *
963 * *
964 *****************************************************************************/