| blob | 5593ebe18d9b9bae028dd41f79ba9743ca5cde58 |
1 /*
2 * arch/ppc/kernel/hashtable.S
3 *
4 * $Id: hashtable.S,v 1.6 1999/10/08 01:56:15 paulus Exp $
5 *
6 * PowerPC version
7 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
8 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
9 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
10 * Adapted for Power Macintosh by Paul Mackerras.
11 * Low-level exception handlers and MMU support
12 * rewritten by Paul Mackerras.
13 * Copyright (C) 1996 Paul Mackerras.
14 *
15 * This file contains low-level assembler routines for managing
16 * the PowerPC MMU hash table. (PPC 8xx processors don't use a
17 * hash table, so this file is not used on them.)
18 *
19 * This program is free software; you can redistribute it and/or
20 * modify it under the terms of the GNU General Public License
21 * as published by the Free Software Foundation; either version
22 * 2 of the License, or (at your option) any later version.
23 *
24 */
26 #include "ppc_asm.h"
27 #include <asm/processor.h>
28 #include <asm/page.h>
29 #include <linux/config.h>
31 /*
32 * Load a PTE into the hash table, if possible.
33 * The address is in r3, and r4 contains access flags:
34 * _PAGE_USER (4) if a user-mode access, ored with
35 * _PAGE_RW (2) if a write. r20 contains DSISR or SRR1,
36 * so bit 1 (0x40000000) is set if the exception was due
37 * to no matching PTE being found in the hash table.
38 * SPRG3 contains the physical address of the current task's thread.
39 *
40 * Returns to the caller if the access is illegal or there is no
41 * mapping for the address. Otherwise it places an appropriate PTE
42 * in the hash table and returns from the exception.
43 * Uses r0, r2 - r6, ctr, lr.
44 *
45 * For speed, 4 of the instructions get patched once the size and
46 * physical address of the hash table are known. These definitions
47 * of Hash_base and Hash_bits below are just an example.
48 */
49 Hash_base = 0x180000
50 Hash_bits = 12 /* e.g. 256kB hash table */
51 Hash_msk = (((1 << Hash_bits) - 1) * 64)
53 .globl hash_page
54 hash_page:
55 #ifdef __SMP__
56 SAVE_2GPRS(7,r21)
57 eieio
58 lis r2,hash_table_lock@h
59 ori r2,r2,hash_table_lock@l
60 tophys(r2,r2)
61 lis r6,0x0fff0000@h
62 mtctr r6
63 mfspr r5,SPRG3
64 lwz r0,PROCESSOR-THREAD(r5)
65 or r0,r0,r6
66 10: lwarx r6,0,r2
67 cmpi 0,r6,0
68 bne- 12f
69 stwcx. r0,0,r2
70 beq+ 11f
71 /* spin here a bit */
72 12: mfctr r7
73 li r8,1000
74 mtctr r8
75 13:
76 bdnz 13b
77 mtctr r7
78 cmpw r6,r0
79 bdnzf 2,10b
80 tw 31,31,31
81 11: eieio
82 REST_2GPRS(7, r21)
83 #endif
84 /* Get PTE (linux-style) and check access */
85 mfspr r2,SPRG3 /* current task's THREAD (phys) */
86 lwz r5,PGDIR(r2) /* virt page-table root */
87 tophys(r5,r5) /* convert to phys addr */
88 rlwimi r5,r3,12,20,29 /* insert top 10 bits of address */
89 lwz r5,0(r5) /* get pmd entry */
90 rlwinm. r5,r5,0,0,19 /* extract address of pte page */
91 #ifdef __SMP__
92 beq- hash_page_out /* return if no mapping */
93 #else
94 /* XXX it seems like the 601 will give a machine fault on the
95 rfi if its alignment is wrong (bottom 4 bits of address are
96 8 or 0xc) and we have had a not-taken conditional branch
97 to the address following the rfi. */
98 beqlr-
99 #endif
100 tophys(r2,r5)
101 rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */
102 lwz r6,0(r2) /* get linux-style pte */
103 ori r4,r4,1 /* set _PAGE_PRESENT bit in access */
104 andc. r0,r4,r6 /* check access & ~permission */
105 #ifdef __SMP__
106 bne- hash_page_out /* return if access not permitted */
107 #else
108 bnelr-
109 #endif
111 ori r6,r6,0x100 /* set _PAGE_ACCESSED in pte */
112 rlwinm r5,r4,5,24,24 /* _PAGE_RW access -> _PAGE_DIRTY */
113 rlwimi r5,r4,7,22,22 /* _PAGE_RW -> _PAGE_HWWRITE */
114 or r6,r6,r5
115 stw r6,0(r2) /* update PTE (accessed/dirty bits) */
117 /* Convert linux-style PTE to low word of PPC-style PTE */
118 rlwinm r4,r6,32-9,31,31 /* _PAGE_HWWRITE -> PP lsb */
119 rlwimi r6,r6,32-1,31,31 /* _PAGE_USER -> PP (both bits now) */
120 ori r4,r4,0xe04 /* clear out reserved bits */
121 andc r6,r6,r4 /* PP=2 or 0, when _PAGE_HWWRITE */
123 /* Construct the high word of the PPC-style PTE */
124 mfsrin r5,r3 /* get segment reg for segment */
125 #ifdef CONFIG_PPC64
126 sldi r5,r5,12
127 #else /* CONFIG_PPC64 */
128 rlwinm r5,r5,7,1,24 /* put VSID in 0x7fffff80 bits */
129 #endif /* CONFIG_PPC64 */
131 #ifndef __SMP__ /* do this later for SMP */
132 #ifdef CONFIG_PPC64
133 ori r5,r5,1 /* set V (valid) bit */
134 #else /* CONFIG_PPC64 */
135 oris r5,r5,0x8000 /* set V (valid) bit */
136 #endif /* CONFIG_PPC64 */
137 #endif
139 #ifdef CONFIG_PPC64
140 /* XXX: does this insert the api correctly? -- Cort */
141 rlwimi r5,r3,17,21,25 /* put in API (abbrev page index) */
142 #else /* CONFIG_PPC64 */
143 rlwimi r5,r3,10,26,31 /* put in API (abbrev page index) */
144 #endif /* CONFIG_PPC64 */
145 /* Get the address of the primary PTE group in the hash table */
146 .globl hash_page_patch_A
147 hash_page_patch_A:
148 lis r4,Hash_base@h /* base address of hash table */
149 rlwimi r4,r5,32-1,26-Hash_bits,25 /* (VSID & hash_mask) << 6 */
150 rlwinm r0,r3,32-6,26-Hash_bits,25 /* (PI & hash_mask) << 6 */
151 xor r4,r4,r0 /* make primary hash */
153 /* See whether it was a PTE not found exception or a
154 protection violation. */
155 andis. r0,r20,0x4000
156 li r2,8 /* PTEs/group */
157 bne 10f /* no PTE: go look for an empty slot */
158 tlbie r3 /* invalidate TLB entry */
160 /* Search the primary PTEG for a PTE whose 1st word matches r5 */
161 mtctr r2
162 addi r3,r4,-8
163 1:
164 #ifdef CONFIG_PPC64
165 lwzu r0,16(r3) /* get next PTE */
166 #else
167 lwzu r0,8(r3) /* get next PTE */
168 #endif
169 cmp 0,r0,r5
170 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
171 beq+ found_slot
173 /* Search the secondary PTEG for a matching PTE */
174 #ifdef CONFIG_PPC64
175 ori r5,r5,0x2 /* set H (secondary hash) bit */
176 #else
177 ori r5,r5,0x40 /* set H (secondary hash) bit */
178 #endif
179 .globl hash_page_patch_B
180 hash_page_patch_B:
181 xoris r3,r4,Hash_msk>>16 /* compute secondary hash */
182 xori r3,r3,0xffc0
183 #ifdef CONFIG_PPC64
184 addi r3,r3,-16
185 #else
186 addi r3,r3,-8
187 #endif
188 mtctr r2
189 2:
190 #ifdef CONFIG_PPC64
191 lwzu r0,16(r3)
192 #else
193 lwzu r0,8(r3)
194 #endif
195 cmp 0,r0,r5
196 bdnzf 2,2b
197 beq+ found_slot
198 #ifdef CONFIG_PPC64
199 xori r5,r5,0x2 /* clear H bit again */
200 #else
201 xori r5,r5,0x40 /* clear H bit again */
202 #endif
204 /* Search the primary PTEG for an empty slot */
205 10: mtctr r2
206 #ifdef CONFIG_PPC64
207 addi r3,r4,-16 /* search primary PTEG */
208 #else
209 addi r3,r4,-8 /* search primary PTEG */
210 #endif
211 1:
212 #ifdef CONFIG_PPC64
213 lwzu r0,16(r3) /* get next PTE */
214 andi. r0,r0,1
215 #else
216 lwzu r0,8(r3) /* get next PTE */
217 rlwinm. r0,r0,0,0,0 /* only want to check valid bit */
218 #endif
219 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
220 beq+ found_empty
222 /* Search the secondary PTEG for an empty slot */
223 #ifdef CONFIG_PPC64
224 ori r5,r5,0x2 /* set H (secondary hash) bit */
225 #else
226 ori r5,r5,0x40 /* set H (secondary hash) bit */
227 #endif
228 .globl hash_page_patch_C
229 hash_page_patch_C:
230 xoris r3,r4,Hash_msk>>16 /* compute secondary hash */
231 xori r3,r3,0xffc0
232 #ifdef CONFIG_PPC64
233 addi r3,r3,-16
234 #else
235 addi r3,r3,-8
236 #endif
237 mtctr r2
238 2:
239 #ifdef CONFIG_PPC64
240 lwzu r0,16(r3)
241 andi. r0,r0,1
242 #else
243 lwzu r0,8(r3)
244 rlwinm. r0,r0,0,0,0 /* only want to check valid bit */
245 #endif
246 bdnzf 2,2b
247 beq+ found_empty
249 /*
250 * Choose an arbitrary slot in the primary PTEG to overwrite.
251 * Since both the primary and secondary PTEGs are full, and we
252 * have no information that the PTEs in the primary PTEG are
253 * more important or useful than those in the secondary PTEG,
254 * and we know there is a definite (although small) speed
255 * advantage to putting the PTE in the primary PTEG, we always
256 * put the PTE in the primary PTEG.
257 */
258 #ifdef CONFIG_PPC64
259 xori r5,r5,0x2 /* clear H bit again */
260 #else
261 xori r5,r5,0x40 /* clear H bit again */
262 #endif
263 lis r3,next_slot@ha
264 tophys(r3,r3)
265 lwz r2,next_slot@l(r3)
266 #ifdef CONFIG_PPC64
267 addi r2,r2,16
268 andi. r2,r2,0x78
269 #else
270 addi r2,r2,8
271 andi. r2,r2,0x38
272 #endif
273 stw r2,next_slot@l(r3)
274 add r3,r4,r2
275 11:
276 /* update counter of evicted pages */
277 lis r2,htab_evicts@ha
278 tophys(r2,r2)
279 lwz r4,htab_evicts@l(r2)
280 addi r4,r4,1
281 stw r4,htab_evicts@l(r2)
283 #ifndef __SMP__
284 /* Store PTE in PTEG */
285 found_empty:
286 #ifdef CONFIG_PPC64
287 std r5,0(r3)
288 #else
289 stw r5,0(r3)
290 #endif
291 found_slot:
292 #ifdef CONFIG_PPC64
293 std r6,8(r3)
294 #else
295 stw r6,4(r3)
296 #endif
297 sync
299 #else /* __SMP__ */
300 /*
301 * Between the tlbie above and updating the hash table entry below,
302 * another CPU could read the hash table entry and put it in its TLB.
303 * There are 3 cases:
304 * 1. using an empty slot
305 * 2. updating an earlier entry to change permissions (i.e. enable write)
306 * 3. taking over the PTE for an unrelated address
307 *
308 * In each case it doesn't really matter if the other CPUs have the old
309 * PTE in their TLB. So we don't need to bother with another tlbie here,
310 * which is convenient as we've overwritten the register that had the
311 * address. :-) The tlbie above is mainly to make sure that this CPU comes
312 * and gets the new PTE from the hash table.
313 *
314 * We do however have to make sure that the PTE is never in an invalid
315 * state with the V bit set.
316 */
317 found_empty:
318 found_slot:
319 stw r5,0(r3) /* clear V (valid) bit in PTE */
320 sync
321 tlbsync
322 sync
323 stw r6,4(r3) /* put in correct RPN, WIMG, PP bits */
324 sync
325 oris r5,r5,0x8000
326 stw r5,0(r3) /* finally set V bit in PTE */
327 #endif /* __SMP__ */
329 /*
330 * Update the hash table miss count. We only want misses here
331 * that _are_ valid addresses and have a pte otherwise we don't
332 * count it as a reload. do_page_fault() takes care of bad addrs
333 * and entries that need linux-style pte's created.
334 *
335 * safe to use r2 here since we're not using it as current yet
336 * update the htab misses count
337 * -- Cort
338 */
339 lis r2,htab_reloads@ha
340 tophys(r2,r2)
341 lwz r3,htab_reloads@l(r2)
342 addi r3,r3,1
343 stw r3,htab_reloads@l(r2)
345 #ifdef __SMP__
346 lis r2,hash_table_lock@ha
347 tophys(r2,r2)
348 li r0,0
349 stw r0,hash_table_lock@l(r2)
350 eieio
351 #endif
353 /* Return from the exception */
354 lwz r3,_CCR(r21)
355 lwz r4,_LINK(r21)
356 lwz r5,_CTR(r21)
357 mtcrf 0xff,r3
358 mtlr r4
359 mtctr r5
360 lwz r0,GPR0(r21)
361 lwz r1,GPR1(r21)
362 lwz r2,GPR2(r21)
363 lwz r3,GPR3(r21)
364 lwz r4,GPR4(r21)
365 lwz r5,GPR5(r21)
366 lwz r6,GPR6(r21)
367 /* we haven't used xer */
368 mtspr SRR1,r23
369 mtspr SRR0,r22
370 lwz r20,GPR20(r21)
371 lwz r22,GPR22(r21)
372 lwz r23,GPR23(r21)
373 lwz r21,GPR21(r21)
374 rfi
376 #ifdef __SMP__
377 hash_page_out:
378 lis r2,hash_table_lock@ha
379 tophys(r2,r2)
380 li r0,0
381 stw r0,hash_table_lock@l(r2)
382 eieio
383 blr
385 .data
386 .globl hash_table_lock
387 hash_table_lock:
388 .long 0
389 #endif /* __SMP__ */
391 .data
392 next_slot:
393 .long 0
395 .text
396 /*
397 * Flush entries from the hash table with VSIDs in the range
398 * given.
399 */
400 _GLOBAL(flush_hash_segments)
401 lis r5,Hash@ha
402 lwz r5,Hash@l(r5) /* base of hash table */
403 cmpwi 0,r5,0
404 bne+ 99f
405 tlbia
406 sync
407 #ifdef __SMP__
408 tlbsync
409 sync
410 #endif
411 blr
412 99:
413 #ifdef __SMP__
414 /* Note - we had better not do anything which could generate
415 a hash table miss while we have the hash table locked,
416 or we'll get a deadlock. -paulus */
417 mfmsr r10
418 sync
419 rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
420 mtmsr r0
421 SYNC
422 lis r9,hash_table_lock@h
423 ori r9,r9,hash_table_lock@l
424 lwz r8,PROCESSOR(r2)
425 oris r8,r8,8
426 10: lwarx r6,0,r9
427 cmpi 0,r6,0
428 bne- 10b
429 stwcx. r8,0,r9
430 bne- 10b
431 eieio
432 #endif
433 rlwinm r3,r3,7,1,24 /* put VSID lower limit in position */
434 oris r3,r3,0x8000 /* set V bit */
435 rlwinm r4,r4,7,1,24 /* put VSID upper limit in position */
436 oris r4,r4,0x8000
437 ori r4,r4,0x7f
438 lis r6,Hash_size@ha
439 lwz r6,Hash_size@l(r6) /* size in bytes */
440 srwi r6,r6,3 /* # PTEs */
441 mtctr r6
442 addi r5,r5,-8
443 li r0,0
444 1: lwzu r6,8(r5) /* get next tag word */
445 cmplw 0,r6,r3
446 cmplw 1,r6,r4
447 cror 0,0,5 /* set cr0.lt if out of range */
448 blt 2f /* branch if out of range */
449 stw r0,0(r5) /* invalidate entry */
450 2: bdnz 1b /* continue with loop */
451 sync
452 tlbia
453 sync
454 #ifdef __SMP__
455 tlbsync
456 sync
457 lis r3,hash_table_lock@ha
458 stw r0,hash_table_lock@l(r3)
459 mtmsr r10
460 SYNC
461 #endif
462 blr
464 /*
465 * Flush the entry for a particular page from the hash table.
466 *
467 * flush_hash_page(unsigned context, unsigned long va)
468 */
469 _GLOBAL(flush_hash_page)
470 lis r6,Hash@ha
471 lwz r6,Hash@l(r6) /* hash table base */
472 cmpwi 0,r6,0 /* hash table in use? */
473 bne+ 99f
474 tlbie r4 /* in hw tlb too */
475 sync
476 #ifdef __SMP__
477 tlbsync
478 sync
479 #endif
480 blr
481 99:
482 #ifdef __SMP__
483 /* Note - we had better not do anything which could generate
484 a hash table miss while we have the hash table locked,
485 or we'll get a deadlock. -paulus */
486 mfmsr r10
487 sync
488 rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
489 mtmsr r0
490 SYNC
491 lis r9,hash_table_lock@h
492 ori r9,r9,hash_table_lock@l
493 lwz r8,PROCESSOR(r2)
494 oris r8,r8,9
495 10: lwarx r7,0,r9
496 cmpi 0,r7,0
497 bne- 10b
498 stwcx. r8,0,r9
499 bne- 10b
500 eieio
501 #endif
502 rlwinm r3,r3,11,1,20 /* put context into vsid */
503 rlwimi r3,r4,11,21,24 /* put top 4 bits of va into vsid */
504 oris r3,r3,0x8000 /* set V (valid) bit */
505 rlwimi r3,r4,10,26,31 /* put in API (abbrev page index) */
506 rlwinm r7,r4,32-6,10,25 /* get page index << 6 */
507 rlwinm r5,r3,32-1,7,25 /* vsid << 6 */
508 xor r7,r7,r5 /* primary hash << 6 */
509 lis r5,Hash_mask@ha
510 lwz r5,Hash_mask@l(r5) /* hash mask */
511 slwi r5,r5,6 /* << 6 */
512 and r7,r7,r5
513 add r6,r6,r7 /* address of primary PTEG */
514 li r8,8
515 mtctr r8
516 addi r7,r6,-8
517 1: lwzu r0,8(r7) /* get next PTE */
518 cmpw 0,r0,r3 /* see if tag matches */
519 bdnzf 2,1b /* while --ctr != 0 && !cr0.eq */
520 beq 3f /* if we found it */
521 ori r3,r3,0x40 /* set H (alt. hash) bit */
522 xor r6,r6,r5 /* address of secondary PTEG */
523 mtctr r8
524 addi r7,r6,-8
525 2: lwzu r0,8(r7) /* get next PTE */
526 cmpw 0,r0,r3 /* see if tag matches */
527 bdnzf 2,2b /* while --ctr != 0 && !cr0.eq */
528 bne 4f /* if we didn't find it */
529 3: li r0,0
530 stw r0,0(r7) /* invalidate entry */
531 4: sync
532 tlbie r4 /* in hw tlb too */
533 sync
534 #ifdef __SMP__
535 tlbsync
536 sync
537 li r0,0
538 stw r0,0(r9) /* clear hash_table_lock */
539 mtmsr r10
540 SYNC
541 #endif
542 blr