| blob | 8c02d02b5d16d0c1a8a5f40be6828e30dde2bb70 |
1 /*
2 * arch/ppc/kernel/hashtable.S
3 *
4 * $Id: hashtable.S,v 1.2 1999/08/23 02:53:17 paulus Exp $
5 *
6 * PowerPC version
7 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
8 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
9 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
10 * Adapted for Power Macintosh by Paul Mackerras.
11 * Low-level exception handlers and MMU support
12 * rewritten by Paul Mackerras.
13 * Copyright (C) 1996 Paul Mackerras.
14 *
15 * This file contains low-level assembler routines for managing
16 * the PowerPC MMU hash table. (PPC 8xx processors don't use a
17 * hash table, so this file is not used on them.)
18 *
19 * This program is free software; you can redistribute it and/or
20 * modify it under the terms of the GNU General Public License
21 * as published by the Free Software Foundation; either version
22 * 2 of the License, or (at your option) any later version.
23 *
24 */
26 #include "ppc_asm.h"
27 #include <asm/processor.h>
28 #include <asm/page.h>
29 #include <linux/config.h>
31 /*
32 * Load a PTE into the hash table, if possible.
33 * The address is in r3, and r4 contains access flags:
34 * _PAGE_USER (4) if a user-mode access, ored with
35 * _PAGE_RW (2) if a write. r20 contains DSISR or SRR1,
36 * so bit 1 (0x40000000) is set if the exception was due
37 * to no matching PTE being found in the hash table.
38 * r5 contains the physical address of the current task's thread.
39 *
40 * Returns to the caller if the access is illegal or there is no
41 * mapping for the address. Otherwise it places an appropriate PTE
42 * in the hash table and returns from the exception.
43 * Uses r0, r2 - r6, ctr, lr.
44 *
45 * For speed, 4 of the instructions get patched once the size and
46 * physical address of the hash table are known. These definitions
47 * of Hash_base and Hash_bits below are just an example.
48 */
49 Hash_base = 0x180000
50 Hash_bits = 12 /* e.g. 256kB hash table */
51 Hash_msk = (((1 << Hash_bits) - 1) * 64)
53 .globl hash_page
54 hash_page:
55 #ifdef __SMP__
56 eieio
57 lis r2,hash_table_lock@h
58 ori r2,r2,hash_table_lock@l
59 tophys(r2,r2)
60 lis r6,100000000@h
61 mtctr r6
62 lwz r0,PROCESSOR-THREAD(r5)
63 or r0,r0,r6
64 10: lwarx r6,0,r2
65 cmpi 0,r6,0
66 bne- 12f
67 stwcx. r0,0,r2
68 beq+ 11f
69 12: cmpw r6,r0
70 bdnzf 2,10b
71 tw 31,31,31
72 11: eieio
73 #endif
74 /* Get PTE (linux-style) and check access */
75 mfspr r2,SPRG3 /* current task's THREAD (phys) */
76 lwz r5,PGDIR(r2) /* virt page-table root */
77 tophys(r5,r5) /* convert to phys addr */
78 rlwimi r5,r3,12,20,29 /* insert top 10 bits of address */
79 lwz r5,0(r5) /* get pmd entry */
80 rlwinm. r5,r5,0,0,19 /* extract address of pte page */
81 #ifdef __SMP__
82 beq- hash_page_out /* return if no mapping */
83 #else
84 /* XXX it seems like the 601 will give a machine fault on the
85 rfi if its alignment is wrong (bottom 4 bits of address are
86 8 or 0xc) and we have had a not-taken conditional branch
87 to the address following the rfi. */
88 beqlr-
89 #endif
90 tophys(r2,r5)
91 rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */
92 lwz r6,0(r2) /* get linux-style pte */
93 ori r4,r4,1 /* set _PAGE_PRESENT bit in access */
94 andc. r0,r4,r6 /* check access & ~permission */
95 #ifdef __SMP__
96 bne- hash_page_out /* return if access not permitted */
97 #else
98 bnelr-
99 #endif
101 ori r6,r6,0x100 /* set _PAGE_ACCESSED in pte */
102 rlwinm r5,r4,5,24,24 /* _PAGE_RW access -> _PAGE_DIRTY */
103 rlwimi r5,r4,7,22,22 /* _PAGE_RW -> _PAGE_HWWRITE */
104 or r6,r6,r5
105 stw r6,0(r2) /* update PTE (accessed/dirty bits) */
107 /* Convert linux-style PTE to low word of PPC-style PTE */
108 #ifdef CONFIG_PPC64
109 /* clear the high 32 bits just in case */
110 clrldi r6,r6,32
111 clrldi r4,r4,32
112 #endif /* CONFIG_PPC64 */
113 rlwinm r4,r6,32-9,31,31 /* _PAGE_HWWRITE -> PP lsb */
114 rlwimi r6,r6,32-1,31,31 /* _PAGE_USER -> PP (both bits now) */
115 ori r4,r4,0xe04 /* clear out reserved bits */
116 andc r6,r6,r4 /* PP=2 or 0, when _PAGE_HWWRITE */
118 /* Construct the high word of the PPC-style PTE */
119 mfsrin r5,r3 /* get segment reg for segment */
120 #ifdef CONFIG_PPC64
121 sldi r5,r5,12
122 #else /* CONFIG_PPC64 */
123 rlwinm r5,r5,7,1,24 /* put VSID in 0x7fffff80 bits */
124 #endif /* CONFIG_PPC64 */
126 #ifndef __SMP__ /* do this later for SMP */
127 #ifdef CONFIG_PPC64
128 ori r5,r5,1 /* set V (valid) bit */
129 #else /* CONFIG_PPC64 */
130 oris r5,r5,0x8000 /* set V (valid) bit */
131 #endif /* CONFIG_PPC64 */
132 #endif
134 #ifdef CONFIG_PPC64
135 /* XXX: does this insert the api correctly? -- Cort */
136 rlwimi r5,r3,17,21,25 /* put in API (abbrev page index) */
137 #else /* CONFIG_PPC64 */
138 rlwimi r5,r3,10,26,31 /* put in API (abbrev page index) */
139 #endif /* CONFIG_PPC64 */
140 /* Get the address of the primary PTE group in the hash table */
141 .globl hash_page_patch_A
142 hash_page_patch_A:
143 lis r4,Hash_base@h /* base address of hash table */
144 #ifdef CONFIG_PPC64
145 /* just in case */
146 clrldi r4,r4,32
147 #endif
148 rlwimi r4,r5,32-1,26-Hash_bits,25 /* (VSID & hash_mask) << 6 */
149 rlwinm r0,r3,32-6,26-Hash_bits,25 /* (PI & hash_mask) << 6 */
150 xor r4,r4,r0 /* make primary hash */
152 /* See whether it was a PTE not found exception or a
153 protection violation. */
154 andis. r0,r20,0x4000
155 li r2,8 /* PTEs/group */
156 bne 10f /* no PTE: go look for an empty slot */
157 tlbie r3 /* invalidate TLB entry */
159 /* Search the primary PTEG for a PTE whose 1st word matches r5 */
160 mtctr r2
161 addi r3,r4,-8
162 1: lwzu r0,8(r3) /* get next PTE */
163 cmp 0,r0,r5
164 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
165 beq+ found_slot
167 /* Search the secondary PTEG for a matching PTE */
168 ori r5,r5,0x40 /* set H (secondary hash) bit */
169 .globl hash_page_patch_B
170 hash_page_patch_B:
171 xoris r3,r4,Hash_msk>>16 /* compute secondary hash */
172 xori r3,r3,0xffc0
173 addi r3,r3,-8
174 mtctr r2
175 2: lwzu r0,8(r3)
176 cmp 0,r0,r5
177 bdnzf 2,2b
178 beq+ found_slot
179 xori r5,r5,0x40 /* clear H bit again */
181 /* Search the primary PTEG for an empty slot */
182 10: mtctr r2
183 addi r3,r4,-8 /* search primary PTEG */
184 1: lwzu r0,8(r3) /* get next PTE */
185 srwi. r0,r0,31 /* only want to check valid bit */
186 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
187 beq+ found_empty
189 /* Search the secondary PTEG for an empty slot */
190 ori r5,r5,0x40 /* set H (secondary hash) bit */
191 .globl hash_page_patch_C
192 hash_page_patch_C:
193 xoris r3,r4,Hash_msk>>16 /* compute secondary hash */
194 xori r3,r3,0xffc0
195 addi r3,r3,-8
196 mtctr r2
197 2: lwzu r0,8(r3)
198 srwi. r0,r0,31 /* only want to check valid bit */
199 bdnzf 2,2b
200 beq+ found_empty
202 /*
203 * Choose an arbitrary slot in the primary PTEG to overwrite.
204 * Since both the primary and secondary PTEGs are full, and we
205 * have no information that the PTEs in the primary PTEG are
206 * more important or useful than those in the secondary PTEG,
207 * and we know there is a definite (although small) speed
208 * advantage to putting the PTE in the primary PTEG, we always
209 * put the PTE in the primary PTEG.
210 */
211 xori r5,r5,0x40 /* clear H bit again */
212 lwz r2,next_slot@l(0)
213 addi r2,r2,8
214 andi. r2,r2,0x38
215 stw r2,next_slot@l(0)
216 add r3,r4,r2
217 11:
218 /* update counter of evicted pages */
219 lis r2,htab_evicts@h
220 ori r2,r2,htab_evicts@l
221 tophys(r2,r2)
222 lwz r4,0(r2)
223 addi r4,r4,1
224 stw r4,0(r2)
226 #ifndef __SMP__
227 /* Store PTE in PTEG */
228 found_empty:
229 stw r5,0(r3)
230 found_slot:
231 stw r6,4(r3)
232 sync
234 #else /* __SMP__ */
235 /*
236 * Between the tlbie above and updating the hash table entry below,
237 * another CPU could read the hash table entry and put it in its TLB.
238 * There are 3 cases:
239 * 1. using an empty slot
240 * 2. updating an earlier entry to change permissions (i.e. enable write)
241 * 3. taking over the PTE for an unrelated address
242 *
243 * In each case it doesn't really matter if the other CPUs have the old
244 * PTE in their TLB. So we don't need to bother with another tlbie here,
245 * which is convenient as we've overwritten the register that had the
246 * address. :-) The tlbie above is mainly to make sure that this CPU comes
247 * and gets the new PTE from the hash table.
248 *
249 * We do however have to make sure that the PTE is never in an invalid
250 * state with the V bit set.
251 */
252 found_empty:
253 found_slot:
254 stw r5,0(r3) /* clear V (valid) bit in PTE */
255 sync
256 tlbsync
257 sync
258 stw r6,4(r3) /* put in correct RPN, WIMG, PP bits */
259 sync
260 oris r5,r5,0x8000
261 stw r5,0(r3) /* finally set V bit in PTE */
262 #endif /* __SMP__ */
264 /*
265 * Update the hash table miss count. We only want misses here
266 * that _are_ valid addresses and have a pte otherwise we don't
267 * count it as a reload. do_page_fault() takes care of bad addrs
268 * and entries that need linux-style pte's created.
269 *
270 * safe to use r2 here since we're not using it as current yet
271 * update the htab misses count
272 * -- Cort
273 */
274 lis r2,htab_reloads@h
275 ori r2,r2,htab_reloads@l
276 tophys(r2,r2)
277 lwz r3,0(r2)
278 addi r3,r3,1
279 stw r3,0(r2)
281 #ifdef __SMP__
282 lis r2,hash_table_lock@ha
283 tophys(r2,r2)
284 li r0,0
285 stw r0,hash_table_lock@l(r2)
286 eieio
287 #endif
289 /* Return from the exception */
290 lwz r3,_CCR(r21)
291 lwz r4,_LINK(r21)
292 lwz r5,_CTR(r21)
293 mtcrf 0xff,r3
294 mtlr r4
295 mtctr r5
296 lwz r0,GPR0(r21)
297 lwz r1,GPR1(r21)
298 lwz r2,GPR2(r21)
299 lwz r3,GPR3(r21)
300 lwz r4,GPR4(r21)
301 lwz r5,GPR5(r21)
302 lwz r6,GPR6(r21)
303 /* we haven't used xer */
304 mtspr SRR1,r23
305 mtspr SRR0,r22
306 lwz r20,GPR20(r21)
307 lwz r22,GPR22(r21)
308 lwz r23,GPR23(r21)
309 lwz r21,GPR21(r21)
310 rfi
312 #ifdef __SMP__
313 hash_page_out:
314 lis r2,hash_table_lock@ha
315 tophys(r2,r2)
316 li r0,0
317 stw r0,hash_table_lock@l(r2)
318 eieio
319 blr
321 .globl hash_table_lock
322 hash_table_lock:
323 .long 0
324 #endif
326 next_slot:
327 .long 0
329 /*
330 * Flush entries from the hash table with VSIDs in the range
331 * given.
332 */
333 _GLOBAL(flush_hash_segments)
334 lis r5,Hash@ha
335 lwz r5,Hash@l(r5) /* base of hash table */
336 cmpwi 0,r5,0
337 bne+ 99f
338 tlbia
339 sync
340 #ifdef __SMP__
341 tlbsync
342 sync
343 #endif
344 blr
345 99:
346 #ifdef __SMP__
347 /* Note - we had better not do anything which could generate
348 a hash table miss while we have the hash table locked,
349 or we'll get a deadlock. -paulus */
350 mfmsr r10
351 sync
352 rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
353 mtmsr r0
354 SYNC
355 lis r9,hash_table_lock@h
356 ori r9,r9,hash_table_lock@l
357 lwz r8,PROCESSOR(r2)
358 oris r8,r8,8
359 10: lwarx r6,0,r9
360 cmpi 0,r6,0
361 bne- 10b
362 stwcx. r8,0,r9
363 bne- 10b
364 eieio
365 #endif
366 rlwinm r3,r3,7,1,24 /* put VSID lower limit in position */
367 oris r3,r3,0x8000 /* set V bit */
368 rlwinm r4,r4,7,1,24 /* put VSID upper limit in position */
369 oris r4,r4,0x8000
370 ori r4,r4,0x7f
371 lis r6,Hash_size@ha
372 lwz r6,Hash_size@l(r6) /* size in bytes */
373 srwi r6,r6,3 /* # PTEs */
374 mtctr r6
375 addi r5,r5,-8
376 li r0,0
377 1: lwzu r6,8(r5) /* get next tag word */
378 cmplw 0,r6,r3
379 cmplw 1,r6,r4
380 cror 0,0,5 /* set cr0.lt if out of range */
381 blt 2f /* branch if out of range */
382 stw r0,0(r5) /* invalidate entry */
383 2: bdnz 1b /* continue with loop */
384 sync
385 tlbia
386 sync
387 #ifdef __SMP__
388 tlbsync
389 sync
390 lis r3,hash_table_lock@ha
391 stw r0,hash_table_lock@l(r3)
392 mtmsr r10
393 SYNC
394 #endif
395 blr
397 /*
398 * Flush the entry for a particular page from the hash table.
399 *
400 * flush_hash_page(unsigned context, unsigned long va)
401 */
402 _GLOBAL(flush_hash_page)
403 lis r6,Hash@ha
404 lwz r6,Hash@l(r6) /* hash table base */
405 cmpwi 0,r6,0 /* hash table in use? */
406 bne+ 99f
407 tlbie r4 /* in hw tlb too */
408 sync
409 #ifdef __SMP__
410 tlbsync
411 sync
412 #endif
413 blr
414 99:
415 #ifdef __SMP__
416 /* Note - we had better not do anything which could generate
417 a hash table miss while we have the hash table locked,
418 or we'll get a deadlock. -paulus */
419 mfmsr r10
420 sync
421 rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
422 mtmsr r0
423 SYNC
424 lis r9,hash_table_lock@h
425 ori r9,r9,hash_table_lock@l
426 lwz r8,PROCESSOR(r2)
427 oris r8,r8,9
428 10: lwarx r7,0,r9
429 cmpi 0,r7,0
430 bne- 10b
431 stwcx. r8,0,r9
432 bne- 10b
433 eieio
434 #endif
435 rlwinm r3,r3,11,1,20 /* put context into vsid */
436 rlwimi r3,r4,11,21,24 /* put top 4 bits of va into vsid */
437 oris r3,r3,0x8000 /* set V (valid) bit */
438 rlwimi r3,r4,10,26,31 /* put in API (abbrev page index) */
439 rlwinm r7,r4,32-6,10,25 /* get page index << 6 */
440 rlwinm r5,r3,32-1,7,25 /* vsid << 6 */
441 xor r7,r7,r5 /* primary hash << 6 */
442 lis r5,Hash_mask@ha
443 lwz r5,Hash_mask@l(r5) /* hash mask */
444 slwi r5,r5,6 /* << 6 */
445 and r7,r7,r5
446 add r6,r6,r7 /* address of primary PTEG */
447 li r8,8
448 mtctr r8
449 addi r7,r6,-8
450 1: lwzu r0,8(r7) /* get next PTE */
451 cmpw 0,r0,r3 /* see if tag matches */
452 bdnzf 2,1b /* while --ctr != 0 && !cr0.eq */
453 beq 3f /* if we found it */
454 ori r3,r3,0x40 /* set H (alt. hash) bit */
455 xor r6,r6,r5 /* address of secondary PTEG */
456 mtctr r8
457 addi r7,r6,-8
458 2: lwzu r0,8(r7) /* get next PTE */
459 cmpw 0,r0,r3 /* see if tag matches */
460 bdnzf 2,2b /* while --ctr != 0 && !cr0.eq */
461 bne 4f /* if we didn't find it */
462 3: li r0,0
463 stw r0,0(r7) /* invalidate entry */
464 4: sync
465 tlbie r4 /* in hw tlb too */
466 sync
467 #ifdef __SMP__
468 tlbsync
469 sync
470 lis r3,hash_table_lock@h
471 li r0,0
472 stw r0,hash_table_lock@l(r3)
473 mtmsr r10
474 SYNC
475 #endif
476 blr