1 /* arch/sparc64/mm/tsb.c
3 * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
6 #include <linux/kernel.h>
7 #include <asm/system.h>
9 #include <asm/tlbflush.h>
11 #include <asm/mmu_context.h>
12 #include <asm/pgtable.h>
14 #include <asm/oplib.h>
16 extern struct tsb swapper_tsb
[KERNEL_TSB_NENTRIES
];
18 static inline unsigned long tsb_hash(unsigned long vaddr
, unsigned long hash_shift
, unsigned long nentries
)
21 return vaddr
& (nentries
- 1);
24 static inline int tag_compare(unsigned long tag
, unsigned long vaddr
)
26 return (tag
== (vaddr
>> 22));
29 /* TSB flushes need only occur on the processor initiating the address
30 * space modification, not on each cpu the address space has run on.
31 * Only the TLB flush needs that treatment.
34 void flush_tsb_kernel_range(unsigned long start
, unsigned long end
)
38 for (v
= start
; v
< end
; v
+= PAGE_SIZE
) {
39 unsigned long hash
= tsb_hash(v
, PAGE_SHIFT
,
41 struct tsb
*ent
= &swapper_tsb
[hash
];
43 if (tag_compare(ent
->tag
, v
)) {
44 ent
->tag
= (1UL << TSB_TAG_INVALID_BIT
);
45 membar_storeload_storestore();
50 static void __flush_tsb_one(struct mmu_gather
*mp
, unsigned long hash_shift
, unsigned long tsb
, unsigned long nentries
)
54 for (i
= 0; i
< mp
->tlb_nr
; i
++) {
55 unsigned long v
= mp
->vaddrs
[i
];
56 unsigned long tag
, ent
, hash
;
60 hash
= tsb_hash(v
, hash_shift
, nentries
);
61 ent
= tsb
+ (hash
* sizeof(struct tsb
));
68 void flush_tsb_user(struct mmu_gather
*mp
)
70 struct mm_struct
*mm
= mp
->mm
;
71 unsigned long nentries
, base
, flags
;
73 spin_lock_irqsave(&mm
->context
.lock
, flags
);
75 base
= (unsigned long) mm
->context
.tsb_block
[MM_TSB_BASE
].tsb
;
76 nentries
= mm
->context
.tsb_block
[MM_TSB_BASE
].tsb_nentries
;
77 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
)
79 __flush_tsb_one(mp
, PAGE_SHIFT
, base
, nentries
);
81 #ifdef CONFIG_HUGETLB_PAGE
82 if (mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb
) {
83 base
= (unsigned long) mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb
;
84 nentries
= mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb_nentries
;
85 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
)
87 __flush_tsb_one(mp
, HPAGE_SHIFT
, base
, nentries
);
90 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
93 #if defined(CONFIG_SPARC64_PAGE_SIZE_8KB)
94 #define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K
95 #define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K
96 #elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB)
97 #define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_64K
98 #define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_64K
100 #error Broken base page size setting...
103 #ifdef CONFIG_HUGETLB_PAGE
104 #if defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
105 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_64K
106 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_64K
107 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K)
108 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_512K
109 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_512K
110 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
111 #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB
112 #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB
114 #error Broken huge page size setting...
118 static void setup_tsb_params(struct mm_struct
*mm
, unsigned long tsb_idx
, unsigned long tsb_bytes
)
120 unsigned long tsb_reg
, base
, tsb_paddr
;
121 unsigned long page_sz
, tte
;
123 mm
->context
.tsb_block
[tsb_idx
].tsb_nentries
=
124 tsb_bytes
/ sizeof(struct tsb
);
127 tte
= pgprot_val(PAGE_KERNEL_LOCKED
);
128 tsb_paddr
= __pa(mm
->context
.tsb_block
[tsb_idx
].tsb
);
129 BUG_ON(tsb_paddr
& (tsb_bytes
- 1UL));
131 /* Use the smallest page size that can map the whole TSB
137 #ifdef DCACHE_ALIASING_POSSIBLE
138 base
+= (tsb_paddr
& 8192);
160 page_sz
= 512 * 1024;
165 page_sz
= 512 * 1024;
170 page_sz
= 512 * 1024;
175 page_sz
= 4 * 1024 * 1024;
179 printk(KERN_ERR
"TSB[%s:%d]: Impossible TSB size %lu, killing process.\n",
180 current
->comm
, current
->pid
, tsb_bytes
);
183 tte
|= pte_sz_bits(page_sz
);
185 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
) {
186 /* Physical mapping, no locked TLB entry for TSB. */
187 tsb_reg
|= tsb_paddr
;
189 mm
->context
.tsb_block
[tsb_idx
].tsb_reg_val
= tsb_reg
;
190 mm
->context
.tsb_block
[tsb_idx
].tsb_map_vaddr
= 0;
191 mm
->context
.tsb_block
[tsb_idx
].tsb_map_pte
= 0;
194 tsb_reg
|= (tsb_paddr
& (page_sz
- 1UL));
195 tte
|= (tsb_paddr
& ~(page_sz
- 1UL));
197 mm
->context
.tsb_block
[tsb_idx
].tsb_reg_val
= tsb_reg
;
198 mm
->context
.tsb_block
[tsb_idx
].tsb_map_vaddr
= base
;
199 mm
->context
.tsb_block
[tsb_idx
].tsb_map_pte
= tte
;
202 /* Setup the Hypervisor TSB descriptor. */
203 if (tlb_type
== hypervisor
) {
204 struct hv_tsb_descr
*hp
= &mm
->context
.tsb_descr
[tsb_idx
];
208 hp
->pgsz_idx
= HV_PGSZ_IDX_BASE
;
210 #ifdef CONFIG_HUGETLB_PAGE
212 hp
->pgsz_idx
= HV_PGSZ_IDX_HUGE
;
219 hp
->num_ttes
= tsb_bytes
/ 16;
223 hp
->pgsz_mask
= HV_PGSZ_MASK_BASE
;
225 #ifdef CONFIG_HUGETLB_PAGE
227 hp
->pgsz_mask
= HV_PGSZ_MASK_HUGE
;
233 hp
->tsb_base
= tsb_paddr
;
238 static struct kmem_cache
*tsb_caches
[8] __read_mostly
;
240 static const char *tsb_cache_names
[8] = {
251 void __init
pgtable_cache_init(void)
255 for (i
= 0; i
< 8; i
++) {
256 unsigned long size
= 8192 << i
;
257 const char *name
= tsb_cache_names
[i
];
259 tsb_caches
[i
] = kmem_cache_create(name
,
262 if (!tsb_caches
[i
]) {
263 prom_printf("Could not create %s cache\n", name
);
269 /* When the RSS of an address space exceeds tsb_rss_limit for a TSB,
270 * do_sparc64_fault() invokes this routine to try and grow it.
272 * When we reach the maximum TSB size supported, we stick ~0UL into
273 * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault()
274 * will not trigger any longer.
276 * The TSB can be anywhere from 8K to 1MB in size, in increasing powers
277 * of two. The TSB must be aligned to it's size, so f.e. a 512K TSB
278 * must be 512K aligned. It also must be physically contiguous, so we
279 * cannot use vmalloc().
281 * The idea here is to grow the TSB when the RSS of the process approaches
282 * the number of entries that the current TSB can hold at once. Currently,
283 * we trigger when the RSS hits 3/4 of the TSB capacity.
285 void tsb_grow(struct mm_struct
*mm
, unsigned long tsb_index
, unsigned long rss
)
287 unsigned long max_tsb_size
= 1 * 1024 * 1024;
288 unsigned long new_size
, old_size
, flags
;
289 struct tsb
*old_tsb
, *new_tsb
;
290 unsigned long new_cache_index
, old_cache_index
;
291 unsigned long new_rss_limit
;
294 if (max_tsb_size
> (PAGE_SIZE
<< MAX_ORDER
))
295 max_tsb_size
= (PAGE_SIZE
<< MAX_ORDER
);
298 for (new_size
= 8192; new_size
< max_tsb_size
; new_size
<<= 1UL) {
299 unsigned long n_entries
= new_size
/ sizeof(struct tsb
);
301 n_entries
= (n_entries
* 3) / 4;
308 if (new_size
== max_tsb_size
)
309 new_rss_limit
= ~0UL;
311 new_rss_limit
= ((new_size
/ sizeof(struct tsb
)) * 3) / 4;
314 gfp_flags
= GFP_KERNEL
;
315 if (new_size
> (PAGE_SIZE
* 2))
316 gfp_flags
= __GFP_NOWARN
| __GFP_NORETRY
;
318 new_tsb
= kmem_cache_alloc_node(tsb_caches
[new_cache_index
],
319 gfp_flags
, numa_node_id());
320 if (unlikely(!new_tsb
)) {
321 /* Not being able to fork due to a high-order TSB
322 * allocation failure is very bad behavior. Just back
323 * down to a 0-order allocation and force no TSB
324 * growing for this address space.
326 if (mm
->context
.tsb_block
[tsb_index
].tsb
== NULL
&&
327 new_cache_index
> 0) {
330 new_rss_limit
= ~0UL;
331 goto retry_tsb_alloc
;
334 /* If we failed on a TSB grow, we are under serious
335 * memory pressure so don't try to grow any more.
337 if (mm
->context
.tsb_block
[tsb_index
].tsb
!= NULL
)
338 mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
= ~0UL;
342 /* Mark all tags as invalid. */
343 tsb_init(new_tsb
, new_size
);
345 /* Ok, we are about to commit the changes. If we are
346 * growing an existing TSB the locking is very tricky,
349 * We have to hold mm->context.lock while committing to the
350 * new TSB, this synchronizes us with processors in
351 * flush_tsb_user() and switch_mm() for this address space.
353 * But even with that lock held, processors run asynchronously
354 * accessing the old TSB via TLB miss handling. This is OK
355 * because those actions are just propagating state from the
356 * Linux page tables into the TSB, page table mappings are not
357 * being changed. If a real fault occurs, the processor will
358 * synchronize with us when it hits flush_tsb_user(), this is
359 * also true for the case where vmscan is modifying the page
360 * tables. The only thing we need to be careful with is to
361 * skip any locked TSB entries during copy_tsb().
363 * When we finish committing to the new TSB, we have to drop
364 * the lock and ask all other cpus running this address space
365 * to run tsb_context_switch() to see the new TSB table.
367 spin_lock_irqsave(&mm
->context
.lock
, flags
);
369 old_tsb
= mm
->context
.tsb_block
[tsb_index
].tsb
;
371 (mm
->context
.tsb_block
[tsb_index
].tsb_reg_val
& 0x7UL
);
372 old_size
= (mm
->context
.tsb_block
[tsb_index
].tsb_nentries
*
376 /* Handle multiple threads trying to grow the TSB at the same time.
377 * One will get in here first, and bump the size and the RSS limit.
378 * The others will get in here next and hit this check.
380 if (unlikely(old_tsb
&&
381 (rss
< mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
))) {
382 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
384 kmem_cache_free(tsb_caches
[new_cache_index
], new_tsb
);
388 mm
->context
.tsb_block
[tsb_index
].tsb_rss_limit
= new_rss_limit
;
391 extern void copy_tsb(unsigned long old_tsb_base
,
392 unsigned long old_tsb_size
,
393 unsigned long new_tsb_base
,
394 unsigned long new_tsb_size
);
395 unsigned long old_tsb_base
= (unsigned long) old_tsb
;
396 unsigned long new_tsb_base
= (unsigned long) new_tsb
;
398 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
) {
399 old_tsb_base
= __pa(old_tsb_base
);
400 new_tsb_base
= __pa(new_tsb_base
);
402 copy_tsb(old_tsb_base
, old_size
, new_tsb_base
, new_size
);
405 mm
->context
.tsb_block
[tsb_index
].tsb
= new_tsb
;
406 setup_tsb_params(mm
, tsb_index
, new_size
);
408 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
410 /* If old_tsb is NULL, we're being invoked for the first time
411 * from init_new_context().
414 /* Reload it on the local cpu. */
415 tsb_context_switch(mm
);
417 /* Now force other processors to do the same. */
420 /* Now it is safe to free the old tsb. */
421 kmem_cache_free(tsb_caches
[old_cache_index
], old_tsb
);
425 int init_new_context(struct task_struct
*tsk
, struct mm_struct
*mm
)
427 #ifdef CONFIG_HUGETLB_PAGE
428 unsigned long huge_pte_count
;
432 spin_lock_init(&mm
->context
.lock
);
434 mm
->context
.sparc64_ctx_val
= 0UL;
436 #ifdef CONFIG_HUGETLB_PAGE
437 /* We reset it to zero because the fork() page copying
438 * will re-increment the counters as the parent PTEs are
439 * copied into the child address space.
441 huge_pte_count
= mm
->context
.huge_pte_count
;
442 mm
->context
.huge_pte_count
= 0;
445 /* copy_mm() copies over the parent's mm_struct before calling
446 * us, so we need to zero out the TSB pointer or else tsb_grow()
447 * will be confused and think there is an older TSB to free up.
449 for (i
= 0; i
< MM_NUM_TSBS
; i
++)
450 mm
->context
.tsb_block
[i
].tsb
= NULL
;
452 /* If this is fork, inherit the parent's TSB size. We would
453 * grow it to that size on the first page fault anyways.
455 tsb_grow(mm
, MM_TSB_BASE
, get_mm_rss(mm
));
457 #ifdef CONFIG_HUGETLB_PAGE
458 if (unlikely(huge_pte_count
))
459 tsb_grow(mm
, MM_TSB_HUGE
, huge_pte_count
);
462 if (unlikely(!mm
->context
.tsb_block
[MM_TSB_BASE
].tsb
))
468 static void tsb_destroy_one(struct tsb_config
*tp
)
470 unsigned long cache_index
;
474 cache_index
= tp
->tsb_reg_val
& 0x7UL
;
475 kmem_cache_free(tsb_caches
[cache_index
], tp
->tsb
);
477 tp
->tsb_reg_val
= 0UL;
480 void destroy_context(struct mm_struct
*mm
)
482 unsigned long flags
, i
;
484 for (i
= 0; i
< MM_NUM_TSBS
; i
++)
485 tsb_destroy_one(&mm
->context
.tsb_block
[i
]);
487 spin_lock_irqsave(&ctx_alloc_lock
, flags
);
489 if (CTX_VALID(mm
->context
)) {
490 unsigned long nr
= CTX_NRBITS(mm
->context
);
491 mmu_context_bmap
[nr
>>6] &= ~(1UL << (nr
& 63));
494 spin_unlock_irqrestore(&ctx_alloc_lock
, flags
);