2 * native hashtable management.
4 * SMP scalability work:
5 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
12 #include <linux/spinlock.h>
13 #include <linux/bitops.h>
14 #include <linux/threads.h>
15 #include <linux/smp.h>
17 #include <asm/abs_addr.h>
18 #include <asm/machdep.h>
20 #include <asm/mmu_context.h>
21 #include <asm/pgtable.h>
22 #include <asm/tlbflush.h>
24 #include <asm/cputable.h>
26 #define HPTE_LOCK_BIT 3
28 static DEFINE_SPINLOCK(native_tlbie_lock
);
30 static inline void native_lock_hpte(HPTE
*hptep
)
32 unsigned long *word
= &hptep
->dw0
.dword0
;
35 if (!test_and_set_bit(HPTE_LOCK_BIT
, word
))
37 while(test_bit(HPTE_LOCK_BIT
, word
))
42 static inline void native_unlock_hpte(HPTE
*hptep
)
44 unsigned long *word
= &hptep
->dw0
.dword0
;
46 asm volatile("lwsync":::"memory");
47 clear_bit(HPTE_LOCK_BIT
, word
);
50 long native_hpte_insert(unsigned long hpte_group
, unsigned long va
,
51 unsigned long prpn
, int secondary
,
52 unsigned long hpteflags
, int bolted
, int large
)
54 unsigned long arpn
= physRpn_to_absRpn(prpn
);
55 HPTE
*hptep
= htab_address
+ hpte_group
;
60 for (i
= 0; i
< HPTES_PER_GROUP
; i
++) {
64 /* retry with lock held */
65 native_lock_hpte(hptep
);
69 native_unlock_hpte(hptep
);
75 if (i
== HPTES_PER_GROUP
)
79 lhpte
.dw1
.dw1
.rpn
= arpn
;
80 lhpte
.dw1
.flags
.flags
= hpteflags
;
83 lhpte
.dw0
.dw0
.avpn
= va
>> 23;
84 lhpte
.dw0
.dw0
.h
= secondary
;
85 lhpte
.dw0
.dw0
.bolted
= bolted
;
90 lhpte
.dw0
.dw0
.avpn
&= ~0x1UL
;
93 hptep
->dw1
.dword1
= lhpte
.dw1
.dword1
;
95 /* Guarantee the second dword is visible before the valid bit */
96 __asm__
__volatile__ ("eieio" : : : "memory");
99 * Now set the first dword including the valid bit
100 * NOTE: this also unlocks the hpte
102 hptep
->dw0
.dword0
= lhpte
.dw0
.dword0
;
104 __asm__
__volatile__ ("ptesync" : : : "memory");
106 return i
| (secondary
<< 3);
109 static long native_hpte_remove(unsigned long hpte_group
)
116 /* pick a random entry to start at */
117 slot_offset
= mftb() & 0x7;
119 for (i
= 0; i
< HPTES_PER_GROUP
; i
++) {
120 hptep
= htab_address
+ hpte_group
+ slot_offset
;
121 dw0
= hptep
->dw0
.dw0
;
123 if (dw0
.v
&& !dw0
.bolted
) {
124 /* retry with lock held */
125 native_lock_hpte(hptep
);
126 dw0
= hptep
->dw0
.dw0
;
127 if (dw0
.v
&& !dw0
.bolted
)
129 native_unlock_hpte(hptep
);
136 if (i
== HPTES_PER_GROUP
)
139 /* Invalidate the hpte. NOTE: this also unlocks it */
140 hptep
->dw0
.dword0
= 0;
145 static inline void set_pp_bit(unsigned long pp
, HPTE
*addr
)
148 unsigned long *p
= &addr
->dw1
.dword1
;
150 __asm__
__volatile__(
155 : "=&r" (old
), "=m" (*p
)
156 : "r" (pp
), "r" (p
), "m" (*p
)
161 * Only works on small pages. Yes its ugly to have to check each slot in
162 * the group but we only use this during bootup.
164 static long native_hpte_find(unsigned long vpn
)
172 hash
= hpt_hash(vpn
, 0);
174 for (j
= 0; j
< 2; j
++) {
175 slot
= (hash
& htab_hash_mask
) * HPTES_PER_GROUP
;
176 for (i
= 0; i
< HPTES_PER_GROUP
; i
++) {
177 hptep
= htab_address
+ slot
;
178 dw0
= hptep
->dw0
.dw0
;
180 if ((dw0
.avpn
== (vpn
>> 11)) && dw0
.v
&&
195 static long native_hpte_updatepp(unsigned long slot
, unsigned long newpp
,
196 unsigned long va
, int large
, int local
)
198 HPTE
*hptep
= htab_address
+ slot
;
200 unsigned long avpn
= va
>> 23;
206 native_lock_hpte(hptep
);
208 dw0
= hptep
->dw0
.dw0
;
210 /* Even if we miss, we need to invalidate the TLB */
211 if ((dw0
.avpn
!= avpn
) || !dw0
.v
) {
212 native_unlock_hpte(hptep
);
215 set_pp_bit(newpp
, hptep
);
216 native_unlock_hpte(hptep
);
219 /* Ensure it is out of the tlb too */
220 if (cpu_has_feature(CPU_FTR_TLBIEL
) && !large
&& local
) {
223 int lock_tlbie
= !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE
);
226 spin_lock(&native_tlbie_lock
);
229 spin_unlock(&native_tlbie_lock
);
236 * Update the page protection bits. Intended to be used to create
237 * guard pages for kernel data structures on pages which are bolted
238 * in the HPT. Assumes pages being operated on will not be stolen.
239 * Does not work on large pages.
241 * No need to lock here because we should be the only user.
243 static void native_hpte_updateboltedpp(unsigned long newpp
, unsigned long ea
)
245 unsigned long vsid
, va
, vpn
, flags
= 0;
248 int lock_tlbie
= !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE
);
250 vsid
= get_kernel_vsid(ea
);
251 va
= (vsid
<< 28) | (ea
& 0x0fffffff);
252 vpn
= va
>> PAGE_SHIFT
;
254 slot
= native_hpte_find(vpn
);
256 panic("could not find page to bolt\n");
257 hptep
= htab_address
+ slot
;
259 set_pp_bit(newpp
, hptep
);
261 /* Ensure it is out of the tlb too */
263 spin_lock_irqsave(&native_tlbie_lock
, flags
);
266 spin_unlock_irqrestore(&native_tlbie_lock
, flags
);
269 static void native_hpte_invalidate(unsigned long slot
, unsigned long va
,
270 int large
, int local
)
272 HPTE
*hptep
= htab_address
+ slot
;
274 unsigned long avpn
= va
>> 23;
276 int lock_tlbie
= !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE
);
281 local_irq_save(flags
);
282 native_lock_hpte(hptep
);
284 dw0
= hptep
->dw0
.dw0
;
286 /* Even if we miss, we need to invalidate the TLB */
287 if ((dw0
.avpn
!= avpn
) || !dw0
.v
) {
288 native_unlock_hpte(hptep
);
290 /* Invalidate the hpte. NOTE: this also unlocks it */
291 hptep
->dw0
.dword0
= 0;
294 /* Invalidate the tlb */
295 if (cpu_has_feature(CPU_FTR_TLBIEL
) && !large
&& local
) {
299 spin_lock(&native_tlbie_lock
);
302 spin_unlock(&native_tlbie_lock
);
304 local_irq_restore(flags
);
307 static void native_flush_hash_range(unsigned long context
,
308 unsigned long number
, int local
)
310 unsigned long vsid
, vpn
, va
, hash
, secondary
, slot
, flags
, avpn
;
314 struct ppc64_tlb_batch
*batch
= &__get_cpu_var(ppc64_tlb_batch
);
316 /* XXX fix for large ptes */
317 unsigned long large
= 0;
319 local_irq_save(flags
);
322 for (i
= 0; i
< number
; i
++) {
323 if ((batch
->addr
[i
] >= USER_START
) &&
324 (batch
->addr
[i
] <= USER_END
))
325 vsid
= get_vsid(context
, batch
->addr
[i
]);
327 vsid
= get_kernel_vsid(batch
->addr
[i
]);
329 va
= (vsid
<< 28) | (batch
->addr
[i
] & 0x0fffffff);
330 batch
->vaddr
[j
] = va
;
332 vpn
= va
>> HPAGE_SHIFT
;
334 vpn
= va
>> PAGE_SHIFT
;
335 hash
= hpt_hash(vpn
, large
);
336 secondary
= (pte_val(batch
->pte
[i
]) & _PAGE_SECONDARY
) >> 15;
339 slot
= (hash
& htab_hash_mask
) * HPTES_PER_GROUP
;
340 slot
+= (pte_val(batch
->pte
[i
]) & _PAGE_GROUP_IX
) >> 12;
342 hptep
= htab_address
+ slot
;
348 native_lock_hpte(hptep
);
350 dw0
= hptep
->dw0
.dw0
;
352 /* Even if we miss, we need to invalidate the TLB */
353 if ((dw0
.avpn
!= avpn
) || !dw0
.v
) {
354 native_unlock_hpte(hptep
);
356 /* Invalidate the hpte. NOTE: this also unlocks it */
357 hptep
->dw0
.dword0
= 0;
363 if (cpu_has_feature(CPU_FTR_TLBIEL
) && !large
&& local
) {
364 asm volatile("ptesync":::"memory");
366 for (i
= 0; i
< j
; i
++)
367 __tlbiel(batch
->vaddr
[i
]);
369 asm volatile("ptesync":::"memory");
371 int lock_tlbie
= !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE
);
374 spin_lock(&native_tlbie_lock
);
376 asm volatile("ptesync":::"memory");
378 for (i
= 0; i
< j
; i
++)
379 __tlbie(batch
->vaddr
[i
], 0);
381 asm volatile("eieio; tlbsync; ptesync":::"memory");
384 spin_unlock(&native_tlbie_lock
);
387 local_irq_restore(flags
);
390 #ifdef CONFIG_PPC_PSERIES
391 /* Disable TLB batching on nighthawk */
392 static inline int tlb_batching_enabled(void)
394 struct device_node
*root
= of_find_node_by_path("/");
398 const char *model
= get_property(root
, "model", NULL
);
399 if (model
&& !strcmp(model
, "IBM,9076-N81"))
407 static inline int tlb_batching_enabled(void)
413 void hpte_init_native(void)
415 ppc_md
.hpte_invalidate
= native_hpte_invalidate
;
416 ppc_md
.hpte_updatepp
= native_hpte_updatepp
;
417 ppc_md
.hpte_updateboltedpp
= native_hpte_updateboltedpp
;
418 ppc_md
.hpte_insert
= native_hpte_insert
;
419 ppc_md
.hpte_remove
= native_hpte_remove
;
420 if (tlb_batching_enabled())
421 ppc_md
.flush_hash_range
= native_flush_hash_range
;