| blob | 74f00ce10870aa1cc4acf503a100b291bff72a14 |
1 /*
2 * arch/ppc/kernel/hashtable.S
3 *
4 * $Id: hashtable.S,v 1.3 1999/09/05 11:56:27 paulus Exp $
5 *
6 * PowerPC version
7 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
8 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
9 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
10 * Adapted for Power Macintosh by Paul Mackerras.
11 * Low-level exception handlers and MMU support
12 * rewritten by Paul Mackerras.
13 * Copyright (C) 1996 Paul Mackerras.
14 *
15 * This file contains low-level assembler routines for managing
16 * the PowerPC MMU hash table. (PPC 8xx processors don't use a
17 * hash table, so this file is not used on them.)
18 *
19 * This program is free software; you can redistribute it and/or
20 * modify it under the terms of the GNU General Public License
21 * as published by the Free Software Foundation; either version
22 * 2 of the License, or (at your option) any later version.
23 *
24 */
26 #include "ppc_asm.h"
27 #include <asm/processor.h>
28 #include <asm/page.h>
29 #include <linux/config.h>
31 /*
32 * Load a PTE into the hash table, if possible.
33 * The address is in r3, and r4 contains access flags:
34 * _PAGE_USER (4) if a user-mode access, ored with
35 * _PAGE_RW (2) if a write. r20 contains DSISR or SRR1,
36 * so bit 1 (0x40000000) is set if the exception was due
37 * to no matching PTE being found in the hash table.
38 * r5 contains the physical address of the current task's thread.
39 *
40 * Returns to the caller if the access is illegal or there is no
41 * mapping for the address. Otherwise it places an appropriate PTE
42 * in the hash table and returns from the exception.
43 * Uses r0, r2 - r6, ctr, lr.
44 *
45 * For speed, 4 of the instructions get patched once the size and
46 * physical address of the hash table are known. These definitions
47 * of Hash_base and Hash_bits below are just an example.
48 */
49 Hash_base = 0x180000
50 Hash_bits = 12 /* e.g. 256kB hash table */
51 Hash_msk = (((1 << Hash_bits) - 1) * 64)
53 .globl hash_page
54 hash_page:
55 #ifdef __SMP__
56 eieio
57 lis r2,hash_table_lock@h
58 ori r2,r2,hash_table_lock@l
59 tophys(r2,r2)
60 lis r6,100000000@h
61 mtctr r6
62 lwz r0,PROCESSOR-THREAD(r5)
63 or r0,r0,r6
64 10: lwarx r6,0,r2
65 cmpi 0,r6,0
66 bne- 12f
67 stwcx. r0,0,r2
68 beq+ 11f
69 12: cmpw r6,r0
70 bdnzf 2,10b
71 tw 31,31,31
72 11: eieio
73 #endif
74 /* Get PTE (linux-style) and check access */
75 mfspr r2,SPRG3 /* current task's THREAD (phys) */
76 lwz r5,PGDIR(r2) /* virt page-table root */
77 tophys(r5,r5) /* convert to phys addr */
78 rlwimi r5,r3,12,20,29 /* insert top 10 bits of address */
79 lwz r5,0(r5) /* get pmd entry */
80 rlwinm. r5,r5,0,0,19 /* extract address of pte page */
81 #ifdef __SMP__
82 beq- hash_page_out /* return if no mapping */
83 #else
84 /* XXX it seems like the 601 will give a machine fault on the
85 rfi if its alignment is wrong (bottom 4 bits of address are
86 8 or 0xc) and we have had a not-taken conditional branch
87 to the address following the rfi. */
88 beqlr-
89 #endif
90 tophys(r2,r5)
91 rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */
92 lwz r6,0(r2) /* get linux-style pte */
93 ori r4,r4,1 /* set _PAGE_PRESENT bit in access */
94 andc. r0,r4,r6 /* check access & ~permission */
95 #ifdef __SMP__
96 bne- hash_page_out /* return if access not permitted */
97 #else
98 bnelr-
99 #endif
101 ori r6,r6,0x100 /* set _PAGE_ACCESSED in pte */
102 rlwinm r5,r4,5,24,24 /* _PAGE_RW access -> _PAGE_DIRTY */
103 rlwimi r5,r4,7,22,22 /* _PAGE_RW -> _PAGE_HWWRITE */
104 or r6,r6,r5
105 stw r6,0(r2) /* update PTE (accessed/dirty bits) */
107 /* Convert linux-style PTE to low word of PPC-style PTE */
108 #ifdef CONFIG_PPC64
109 /* clear the high 32 bits just in case */
110 clrldi r6,r6,32
111 clrldi r4,r4,32
112 #endif /* CONFIG_PPC64 */
113 rlwinm r4,r6,32-9,31,31 /* _PAGE_HWWRITE -> PP lsb */
114 rlwimi r6,r6,32-1,31,31 /* _PAGE_USER -> PP (both bits now) */
115 ori r4,r4,0xe04 /* clear out reserved bits */
116 andc r6,r6,r4 /* PP=2 or 0, when _PAGE_HWWRITE */
118 /* Construct the high word of the PPC-style PTE */
119 mfsrin r5,r3 /* get segment reg for segment */
120 #ifdef CONFIG_PPC64
121 sldi r5,r5,12
122 #else /* CONFIG_PPC64 */
123 rlwinm r5,r5,7,1,24 /* put VSID in 0x7fffff80 bits */
124 #endif /* CONFIG_PPC64 */
126 #ifndef __SMP__ /* do this later for SMP */
127 #ifdef CONFIG_PPC64
128 ori r5,r5,1 /* set V (valid) bit */
129 #else /* CONFIG_PPC64 */
130 oris r5,r5,0x8000 /* set V (valid) bit */
131 #endif /* CONFIG_PPC64 */
132 #endif
134 #ifdef CONFIG_PPC64
135 /* XXX: does this insert the api correctly? -- Cort */
136 rlwimi r5,r3,17,21,25 /* put in API (abbrev page index) */
137 #else /* CONFIG_PPC64 */
138 rlwimi r5,r3,10,26,31 /* put in API (abbrev page index) */
139 #endif /* CONFIG_PPC64 */
140 /* Get the address of the primary PTE group in the hash table */
141 .globl hash_page_patch_A
142 hash_page_patch_A:
143 lis r4,Hash_base@h /* base address of hash table */
144 #ifdef CONFIG_PPC64
145 /* just in case */
146 clrldi r4,r4,32
147 #endif
148 rlwimi r4,r5,32-1,26-Hash_bits,25 /* (VSID & hash_mask) << 6 */
149 rlwinm r0,r3,32-6,26-Hash_bits,25 /* (PI & hash_mask) << 6 */
150 xor r4,r4,r0 /* make primary hash */
152 /* See whether it was a PTE not found exception or a
153 protection violation. */
154 andis. r0,r20,0x4000
155 li r2,8 /* PTEs/group */
156 bne 10f /* no PTE: go look for an empty slot */
157 tlbie r3 /* invalidate TLB entry */
159 /* Search the primary PTEG for a PTE whose 1st word matches r5 */
160 mtctr r2
161 addi r3,r4,-8
162 1: lwzu r0,8(r3) /* get next PTE */
163 cmp 0,r0,r5
164 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
165 beq+ found_slot
167 /* Search the secondary PTEG for a matching PTE */
168 ori r5,r5,0x40 /* set H (secondary hash) bit */
169 .globl hash_page_patch_B
170 hash_page_patch_B:
171 xoris r3,r4,Hash_msk>>16 /* compute secondary hash */
172 xori r3,r3,0xffc0
173 addi r3,r3,-8
174 mtctr r2
175 2: lwzu r0,8(r3)
176 cmp 0,r0,r5
177 bdnzf 2,2b
178 beq+ found_slot
179 xori r5,r5,0x40 /* clear H bit again */
181 /* Search the primary PTEG for an empty slot */
182 10: mtctr r2
183 addi r3,r4,-8 /* search primary PTEG */
184 1: lwzu r0,8(r3) /* get next PTE */
185 srwi. r0,r0,31 /* only want to check valid bit */
186 bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
187 beq+ found_empty
189 /* Search the secondary PTEG for an empty slot */
190 ori r5,r5,0x40 /* set H (secondary hash) bit */
191 .globl hash_page_patch_C
192 hash_page_patch_C:
193 xoris r3,r4,Hash_msk>>16 /* compute secondary hash */
194 xori r3,r3,0xffc0
195 addi r3,r3,-8
196 mtctr r2
197 2: lwzu r0,8(r3)
198 srwi. r0,r0,31 /* only want to check valid bit */
199 bdnzf 2,2b
200 beq+ found_empty
202 /*
203 * Choose an arbitrary slot in the primary PTEG to overwrite.
204 * Since both the primary and secondary PTEGs are full, and we
205 * have no information that the PTEs in the primary PTEG are
206 * more important or useful than those in the secondary PTEG,
207 * and we know there is a definite (although small) speed
208 * advantage to putting the PTE in the primary PTEG, we always
209 * put the PTE in the primary PTEG.
210 */
211 xori r5,r5,0x40 /* clear H bit again */
212 lwz r2,next_slot@l(0)
213 addi r2,r2,8
214 andi. r2,r2,0x38
215 stw r2,next_slot@l(0)
216 add r3,r4,r2
217 11:
218 /* update counter of evicted pages */
219 lis r2,htab_evicts@h
220 ori r2,r2,htab_evicts@l
221 tophys(r2,r2)
222 lwz r4,0(r2)
223 addi r4,r4,1
224 stw r4,0(r2)
226 #ifndef __SMP__
227 /* Store PTE in PTEG */
228 found_empty:
229 stw r5,0(r3)
230 found_slot:
231 stw r6,4(r3)
232 sync
234 #else /* __SMP__ */
235 /*
236 * Between the tlbie above and updating the hash table entry below,
237 * another CPU could read the hash table entry and put it in its TLB.
238 * There are 3 cases:
239 * 1. using an empty slot
240 * 2. updating an earlier entry to change permissions (i.e. enable write)
241 * 3. taking over the PTE for an unrelated address
242 *
243 * In each case it doesn't really matter if the other CPUs have the old
244 * PTE in their TLB. So we don't need to bother with another tlbie here,
245 * which is convenient as we've overwritten the register that had the
246 * address. :-) The tlbie above is mainly to make sure that this CPU comes
247 * and gets the new PTE from the hash table.
248 *
249 * We do however have to make sure that the PTE is never in an invalid
250 * state with the V bit set.
251 */
252 found_empty:
253 found_slot:
254 stw r5,0(r3) /* clear V (valid) bit in PTE */
255 sync
256 tlbsync
257 sync
258 stw r6,4(r3) /* put in correct RPN, WIMG, PP bits */
259 sync
260 oris r5,r5,0x8000
261 stw r5,0(r3) /* finally set V bit in PTE */
262 #endif /* __SMP__ */
264 /*
265 * Update the hash table miss count. We only want misses here
266 * that _are_ valid addresses and have a pte otherwise we don't
267 * count it as a reload. do_page_fault() takes care of bad addrs
268 * and entries that need linux-style pte's created.
269 *
270 * safe to use r2 here since we're not using it as current yet
271 * update the htab misses count
272 * -- Cort
273 */
274 lis r2,htab_reloads@h
275 ori r2,r2,htab_reloads@l
276 tophys(r2,r2)
277 lwz r3,0(r2)
278 addi r3,r3,1
279 stw r3,0(r2)
281 #ifdef __SMP__
282 lis r2,hash_table_lock@ha
283 tophys(r2,r2)
284 li r0,0
285 stw r0,hash_table_lock@l(r2)
286 eieio
287 #endif
289 /* Return from the exception */
290 lwz r3,_CCR(r21)
291 lwz r4,_LINK(r21)
292 lwz r5,_CTR(r21)
293 mtcrf 0xff,r3
294 mtlr r4
295 mtctr r5
296 lwz r0,GPR0(r21)
297 lwz r1,GPR1(r21)
298 lwz r2,GPR2(r21)
299 lwz r3,GPR3(r21)
300 lwz r4,GPR4(r21)
301 lwz r5,GPR5(r21)
302 lwz r6,GPR6(r21)
303 /* we haven't used xer */
304 mtspr SRR1,r23
305 mtspr SRR0,r22
306 lwz r20,GPR20(r21)
307 lwz r22,GPR22(r21)
308 lwz r23,GPR23(r21)
309 lwz r21,GPR21(r21)
310 rfi
312 #ifdef __SMP__
313 hash_page_out:
314 lis r2,hash_table_lock@ha
315 tophys(r2,r2)
316 li r0,0
317 stw r0,hash_table_lock@l(r2)
318 eieio
319 blr
321 .data
322 .globl hash_table_lock
323 hash_table_lock:
324 .long 0
325 .text
326 #endif /* __SMP__ */
328 /* next_slot is assumed to be within the first 32kB of physical RAM */
329 next_slot:
330 .long 0
332 /*
333 * Flush entries from the hash table with VSIDs in the range
334 * given.
335 */
336 _GLOBAL(flush_hash_segments)
337 lis r5,Hash@ha
338 lwz r5,Hash@l(r5) /* base of hash table */
339 cmpwi 0,r5,0
340 bne+ 99f
341 tlbia
342 sync
343 #ifdef __SMP__
344 tlbsync
345 sync
346 #endif
347 blr
348 99:
349 #ifdef __SMP__
350 /* Note - we had better not do anything which could generate
351 a hash table miss while we have the hash table locked,
352 or we'll get a deadlock. -paulus */
353 mfmsr r10
354 sync
355 rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
356 mtmsr r0
357 SYNC
358 lis r9,hash_table_lock@h
359 ori r9,r9,hash_table_lock@l
360 lwz r8,PROCESSOR(r2)
361 oris r8,r8,8
362 10: lwarx r6,0,r9
363 cmpi 0,r6,0
364 bne- 10b
365 stwcx. r8,0,r9
366 bne- 10b
367 eieio
368 #endif
369 rlwinm r3,r3,7,1,24 /* put VSID lower limit in position */
370 oris r3,r3,0x8000 /* set V bit */
371 rlwinm r4,r4,7,1,24 /* put VSID upper limit in position */
372 oris r4,r4,0x8000
373 ori r4,r4,0x7f
374 lis r6,Hash_size@ha
375 lwz r6,Hash_size@l(r6) /* size in bytes */
376 srwi r6,r6,3 /* # PTEs */
377 mtctr r6
378 addi r5,r5,-8
379 li r0,0
380 1: lwzu r6,8(r5) /* get next tag word */
381 cmplw 0,r6,r3
382 cmplw 1,r6,r4
383 cror 0,0,5 /* set cr0.lt if out of range */
384 blt 2f /* branch if out of range */
385 stw r0,0(r5) /* invalidate entry */
386 2: bdnz 1b /* continue with loop */
387 sync
388 tlbia
389 sync
390 #ifdef __SMP__
391 tlbsync
392 sync
393 lis r3,hash_table_lock@ha
394 stw r0,hash_table_lock@l(r3)
395 mtmsr r10
396 SYNC
397 #endif
398 blr
400 /*
401 * Flush the entry for a particular page from the hash table.
402 *
403 * flush_hash_page(unsigned context, unsigned long va)
404 */
405 _GLOBAL(flush_hash_page)
406 lis r6,Hash@ha
407 lwz r6,Hash@l(r6) /* hash table base */
408 cmpwi 0,r6,0 /* hash table in use? */
409 bne+ 99f
410 tlbie r4 /* in hw tlb too */
411 sync
412 #ifdef __SMP__
413 tlbsync
414 sync
415 #endif
416 blr
417 99:
418 #ifdef __SMP__
419 /* Note - we had better not do anything which could generate
420 a hash table miss while we have the hash table locked,
421 or we'll get a deadlock. -paulus */
422 mfmsr r10
423 sync
424 rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
425 mtmsr r0
426 SYNC
427 lis r9,hash_table_lock@h
428 ori r9,r9,hash_table_lock@l
429 lwz r8,PROCESSOR(r2)
430 oris r8,r8,9
431 10: lwarx r7,0,r9
432 cmpi 0,r7,0
433 bne- 10b
434 stwcx. r8,0,r9
435 bne- 10b
436 eieio
437 #endif
438 rlwinm r3,r3,11,1,20 /* put context into vsid */
439 rlwimi r3,r4,11,21,24 /* put top 4 bits of va into vsid */
440 oris r3,r3,0x8000 /* set V (valid) bit */
441 rlwimi r3,r4,10,26,31 /* put in API (abbrev page index) */
442 rlwinm r7,r4,32-6,10,25 /* get page index << 6 */
443 rlwinm r5,r3,32-1,7,25 /* vsid << 6 */
444 xor r7,r7,r5 /* primary hash << 6 */
445 lis r5,Hash_mask@ha
446 lwz r5,Hash_mask@l(r5) /* hash mask */
447 slwi r5,r5,6 /* << 6 */
448 and r7,r7,r5
449 add r6,r6,r7 /* address of primary PTEG */
450 li r8,8
451 mtctr r8
452 addi r7,r6,-8
453 1: lwzu r0,8(r7) /* get next PTE */
454 cmpw 0,r0,r3 /* see if tag matches */
455 bdnzf 2,1b /* while --ctr != 0 && !cr0.eq */
456 beq 3f /* if we found it */
457 ori r3,r3,0x40 /* set H (alt. hash) bit */
458 xor r6,r6,r5 /* address of secondary PTEG */
459 mtctr r8
460 addi r7,r6,-8
461 2: lwzu r0,8(r7) /* get next PTE */
462 cmpw 0,r0,r3 /* see if tag matches */
463 bdnzf 2,2b /* while --ctr != 0 && !cr0.eq */
464 bne 4f /* if we didn't find it */
465 3: li r0,0
466 stw r0,0(r7) /* invalidate entry */
467 4: sync
468 tlbie r4 /* in hw tlb too */
469 sync
470 #ifdef __SMP__
471 tlbsync
472 sync
473 li r0,0
474 stw r0,0(r9) /* clear hash_table_lock */
475 mtmsr r10
476 SYNC
477 #endif
478 blr