1 /* arch/sparc64/mm/tsb.c
3 * Copyright (C) 2006, 2008 David S. Miller <davem@davemloft.net>
6 #include <linux/kernel.h>
7 #include <linux/preempt.h>
8 #include <asm/system.h>
10 #include <asm/tlbflush.h>
12 #include <asm/mmu_context.h>
13 #include <asm/pgtable.h>
15 #include <asm/oplib.h>
17 extern struct tsb swapper_tsb
[KERNEL_TSB_NENTRIES
];
19 static inline unsigned long tsb_hash(unsigned long vaddr
, unsigned long hash_shift
, unsigned long nentries
)
22 return vaddr
& (nentries
- 1);
25 static inline int tag_compare(unsigned long tag
, unsigned long vaddr
)
27 return (tag
== (vaddr
>> 22));
30 /* TSB flushes need only occur on the processor initiating the address
31 * space modification, not on each cpu the address space has run on.
32 * Only the TLB flush needs that treatment.
35 void flush_tsb_kernel_range(unsigned long start
, unsigned long end
)
39 for (v
= start
; v
< end
; v
+= PAGE_SIZE
) {
40 unsigned long hash
= tsb_hash(v
, PAGE_SHIFT
,
42 struct tsb
*ent
= &swapper_tsb
[hash
];
44 if (tag_compare(ent
->tag
, v
)) {
45 ent
->tag
= (1UL << TSB_TAG_INVALID_BIT
);
46 membar_storeload_storestore();
51 static void __flush_tsb_one(struct mmu_gather
*mp
, unsigned long hash_shift
, unsigned long tsb
, unsigned long nentries
)
55 for (i
= 0; i
< mp
->tlb_nr
; i
++) {
56 unsigned long v
= mp
->vaddrs
[i
];
57 unsigned long tag
, ent
, hash
;
61 hash
= tsb_hash(v
, hash_shift
, nentries
);
62 ent
= tsb
+ (hash
* sizeof(struct tsb
));
69 void flush_tsb_user(struct mmu_gather
*mp
)
71 struct mm_struct
*mm
= mp
->mm
;
72 unsigned long nentries
, base
, flags
;
74 spin_lock_irqsave(&mm
->context
.lock
, flags
);
76 base
= (unsigned long) mm
->context
.tsb_block
[MM_TSB_BASE
].tsb
;
77 nentries
= mm
->context
.tsb_block
[MM_TSB_BASE
].tsb_nentries
;
78 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
)
80 __flush_tsb_one(mp
, PAGE_SHIFT
, base
, nentries
);
82 #ifdef CONFIG_HUGETLB_PAGE
83 if (mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb
) {
84 base
= (unsigned long) mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb
;
85 nentries
= mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb_nentries
;
86 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
)
88 __flush_tsb_one(mp
, HPAGE_SHIFT
, base
, nentries
);
91 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
94 #if defined(CONFIG_SPARC64_PAGE_SIZE_8KB)
95 #define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K
96 #define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K
97 #elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB)
98 #define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_64K
99 #define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_64K
101 #error Broken base page size setting...
104 #ifdef CONFIG_HUGETLB_PAGE
105 #if defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
106 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_64K
107 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_64K
108 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K)
109 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_512K
110 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_512K
111 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
112 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB
113 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB
115 #error Broken huge page size setting...
119 static void setup_tsb_params(struct mm_struct
*mm
, unsigned long tsb_idx
, unsigned long tsb_bytes
)
121 unsigned long tsb_reg
, base
, tsb_paddr
;
122 unsigned long page_sz
, tte
;
124 mm
->context
.tsb_block
[tsb_idx
].tsb_nentries
=
125 tsb_bytes
/ sizeof(struct tsb
);
128 tte
= pgprot_val(PAGE_KERNEL_LOCKED
);
129 tsb_paddr
= __pa(mm
->context
.tsb_block
[tsb_idx
].tsb
);
130 BUG_ON(tsb_paddr
& (tsb_bytes
- 1UL));
132 /* Use the smallest page size that can map the whole TSB
138 #ifdef DCACHE_ALIASING_POSSIBLE
139 base
+= (tsb_paddr
& 8192);
161 page_sz
= 512 * 1024;
166 page_sz
= 512 * 1024;
171 page_sz
= 512 * 1024;
176 page_sz
= 4 * 1024 * 1024;
180 printk(KERN_ERR
"TSB[%s:%d]: Impossible TSB size %lu, killing process.\n",
181 current
->comm
, current
->pid
, tsb_bytes
);
184 tte
|= pte_sz_bits(page_sz
);
186 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
) {
187 /* Physical mapping, no locked TLB entry for TSB. */
188 tsb_reg
|= tsb_paddr
;
190 mm
->context
.tsb_block
[tsb_idx
].tsb_reg_val
= tsb_reg
;
191 mm
->context
.tsb_block
[tsb_idx
].tsb_map_vaddr
= 0;
192 mm
->context
.tsb_block
[tsb_idx
].tsb_map_pte
= 0;
195 tsb_reg
|= (tsb_paddr
& (page_sz
- 1UL));
196 tte
|= (tsb_paddr
& ~(page_sz
- 1UL));
198 mm
->context
.tsb_block
[tsb_idx
].tsb_reg_val
= tsb_reg
;
199 mm
->context
.tsb_block
[tsb_idx
].tsb_map_vaddr
= base
;
200 mm
->context
.tsb_block
[tsb_idx
].tsb_map_pte
= tte
;
203 /* Setup the Hypervisor TSB descriptor. */
204 if (tlb_type
== hypervisor
) {
205 struct hv_tsb_descr
*hp
= &mm
->context
.tsb_descr
[tsb_idx
];
209 hp
->pgsz_idx
= HV_PGSZ_IDX_BASE
;
211 #ifdef CONFIG_HUGETLB_PAGE
213 hp
->pgsz_idx
= HV_PGSZ_IDX_HUGE
;
220 hp
->num_ttes
= tsb_bytes
/ 16;
224 hp
->pgsz_mask
= HV_PGSZ_MASK_BASE
;
226 #ifdef CONFIG_HUGETLB_PAGE
228 hp
->pgsz_mask
= HV_PGSZ_MASK_HUGE
;
234 hp
->tsb_base
= tsb_paddr
;
239 static struct kmem_cache
*tsb_caches
[8] __read_mostly
;
241 static const char *tsb_cache_names
[8] = {
252 void __init
pgtable_cache_init(void)
256 for (i
= 0; i
< 8; i
++) {
257 unsigned long size
= 8192 << i
;
258 const char *name
= tsb_cache_names
[i
];
260 tsb_caches
[i
] = kmem_cache_create(name
,
263 if (!tsb_caches
[i
]) {
264 prom_printf("Could not create %s cache\n", name
);
270 /* When the RSS of an address space exceeds tsb_rss_limit for a TSB,
271 * do_sparc64_fault() invokes this routine to try and grow it.
273 * When we reach the maximum TSB size supported, we stick ~0UL into
274 * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault()
275 * will not trigger any longer.
277 * The TSB can be anywhere from 8K to 1MB in size, in increasing powers
278 * of two. The TSB must be aligned to it's size, so f.e. a 512K TSB
279 * must be 512K aligned. It also must be physically contiguous, so we
280 * cannot use vmalloc().
282 * The idea here is to grow the TSB when the RSS of the process approaches
283 * the number of entries that the current TSB can hold at once. Currently,
284 * we trigger when the RSS hits 3/4 of the TSB capacity.
286 void tsb_grow(struct mm_struct
*mm
, unsigned long tsb_index
, unsigned long rss
)
288 unsigned long max_tsb_size
= 1 * 1024 * 1024;
289 unsigned long new_size
, old_size
, flags
;
290 struct tsb
*old_tsb
, *new_tsb
;
291 unsigned long new_cache_index
, old_cache_index
;
292 unsigned long new_rss_limit
;
295 if (max_tsb_size
> (PAGE_SIZE
<< MAX_ORDER
))
296 max_tsb_size
= (PAGE_SIZE
<< MAX_ORDER
);
299 for (new_size
= 8192; new_size
< max_tsb_size
; new_size
<<= 1UL) {
300 unsigned long n_entries
= new_size
/ sizeof(struct tsb
);
302 n_entries
= (n_entries
* 3) / 4;
309 if (new_size
== max_tsb_size
)
310 new_rss_limit
= ~0UL;
312 new_rss_limit
= ((new_size
/ sizeof(struct tsb
)) * 3) / 4;
315 gfp_flags
= GFP_KERNEL
;
316 if (new_size
> (PAGE_SIZE
* 2))
317 gfp_flags
= __GFP_NOWARN
| __GFP_NORETRY
;
319 new_tsb
= kmem_cache_alloc_node(tsb_caches
[new_cache_index
],
320 gfp_flags
, numa_node_id());
321 if (unlikely(!new_tsb
)) {
322 /* Not being able to fork due to a high-order TSB
323 * allocation failure is very bad behavior. Just back
324 * down to a 0-order allocation and force no TSB
325 * growing for this address space.
327 if (mm
->context
.tsb_block
[tsb_index
].tsb
== NULL
&&
328 new_cache_index
> 0) {
331 new_rss_limit
= ~0UL;
332 goto retry_tsb_alloc
;
335 /* If we failed on a TSB grow, we are under serious
336 * memory pressure so don't try to grow any more.
338 if (mm
->context
.tsb_block
[tsb_index
].tsb
!= NULL
)
339 mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
= ~0UL;
343 /* Mark all tags as invalid. */
344 tsb_init(new_tsb
, new_size
);
346 /* Ok, we are about to commit the changes. If we are
347 * growing an existing TSB the locking is very tricky,
350 * We have to hold mm->context.lock while committing to the
351 * new TSB, this synchronizes us with processors in
352 * flush_tsb_user() and switch_mm() for this address space.
354 * But even with that lock held, processors run asynchronously
355 * accessing the old TSB via TLB miss handling. This is OK
356 * because those actions are just propagating state from the
357 * Linux page tables into the TSB, page table mappings are not
358 * being changed. If a real fault occurs, the processor will
359 * synchronize with us when it hits flush_tsb_user(), this is
360 * also true for the case where vmscan is modifying the page
361 * tables. The only thing we need to be careful with is to
362 * skip any locked TSB entries during copy_tsb().
364 * When we finish committing to the new TSB, we have to drop
365 * the lock and ask all other cpus running this address space
366 * to run tsb_context_switch() to see the new TSB table.
368 spin_lock_irqsave(&mm
->context
.lock
, flags
);
370 old_tsb
= mm
->context
.tsb_block
[tsb_index
].tsb
;
372 (mm
->context
.tsb_block
[tsb_index
].tsb_reg_val
& 0x7UL
);
373 old_size
= (mm
->context
.tsb_block
[tsb_index
].tsb_nentries
*
377 /* Handle multiple threads trying to grow the TSB at the same time.
378 * One will get in here first, and bump the size and the RSS limit.
379 * The others will get in here next and hit this check.
381 if (unlikely(old_tsb
&&
382 (rss
< mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
))) {
383 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
385 kmem_cache_free(tsb_caches
[new_cache_index
], new_tsb
);
389 mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
= new_rss_limit
;
392 extern void copy_tsb(unsigned long old_tsb_base
,
393 unsigned long old_tsb_size
,
394 unsigned long new_tsb_base
,
395 unsigned long new_tsb_size
);
396 unsigned long old_tsb_base
= (unsigned long) old_tsb
;
397 unsigned long new_tsb_base
= (unsigned long) new_tsb
;
399 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
) {
400 old_tsb_base
= __pa(old_tsb_base
);
401 new_tsb_base
= __pa(new_tsb_base
);
403 copy_tsb(old_tsb_base
, old_size
, new_tsb_base
, new_size
);
406 mm
->context
.tsb_block
[tsb_index
].tsb
= new_tsb
;
407 setup_tsb_params(mm
, tsb_index
, new_size
);
409 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
411 /* If old_tsb is NULL, we're being invoked for the first time
412 * from init_new_context().
415 /* Reload it on the local cpu. */
416 tsb_context_switch(mm
);
418 /* Now force other processors to do the same. */
423 /* Now it is safe to free the old tsb. */
424 kmem_cache_free(tsb_caches
[old_cache_index
], old_tsb
);
428 int init_new_context(struct task_struct
*tsk
, struct mm_struct
*mm
)
430 #ifdef CONFIG_HUGETLB_PAGE
431 unsigned long huge_pte_count
;
435 spin_lock_init(&mm
->context
.lock
);
437 mm
->context
.sparc64_ctx_val
= 0UL;
439 #ifdef CONFIG_HUGETLB_PAGE
440 /* We reset it to zero because the fork() page copying
441 * will re-increment the counters as the parent PTEs are
442 * copied into the child address space.
444 huge_pte_count
= mm
->context
.huge_pte_count
;
445 mm
->context
.huge_pte_count
= 0;
448 /* copy_mm() copies over the parent's mm_struct before calling
449 * us, so we need to zero out the TSB pointer or else tsb_grow()
450 * will be confused and think there is an older TSB to free up.
452 for (i
= 0; i
< MM_NUM_TSBS
; i
++)
453 mm
->context
.tsb_block
[i
].tsb
= NULL
;
455 /* If this is fork, inherit the parent's TSB size. We would
456 * grow it to that size on the first page fault anyways.
458 tsb_grow(mm
, MM_TSB_BASE
, get_mm_rss(mm
));
460 #ifdef CONFIG_HUGETLB_PAGE
461 if (unlikely(huge_pte_count
))
462 tsb_grow(mm
, MM_TSB_HUGE
, huge_pte_count
);
465 if (unlikely(!mm
->context
.tsb_block
[MM_TSB_BASE
].tsb
))
471 static void tsb_destroy_one(struct tsb_config
*tp
)
473 unsigned long cache_index
;
477 cache_index
= tp
->tsb_reg_val
& 0x7UL
;
478 kmem_cache_free(tsb_caches
[cache_index
], tp
->tsb
);
480 tp
->tsb_reg_val
= 0UL;
483 void destroy_context(struct mm_struct
*mm
)
485 unsigned long flags
, i
;
487 for (i
= 0; i
< MM_NUM_TSBS
; i
++)
488 tsb_destroy_one(&mm
->context
.tsb_block
[i
]);
490 spin_lock_irqsave(&ctx_alloc_lock
, flags
);
492 if (CTX_VALID(mm
->context
)) {
493 unsigned long nr
= CTX_NRBITS(mm
->context
);
494 mmu_context_bmap
[nr
>>6] &= ~(1UL << (nr
& 63));
497 spin_unlock_irqrestore(&ctx_alloc_lock
, flags
);