2 * SN Platform GRU Driver
4 * FAULT HANDLER FOR GRU DETECTED TLB MISSES
6 * This file contains code that handles TLB misses within the GRU.
7 * These misses are reported either via interrupts or user polling of
10 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 #include <linux/kernel.h>
28 #include <linux/errno.h>
29 #include <linux/spinlock.h>
31 #include <linux/hugetlb.h>
32 #include <linux/device.h>
34 #include <linux/uaccess.h>
35 #include <linux/security.h>
36 #include <asm/pgtable.h>
38 #include "grutables.h"
40 #include "gru_instructions.h"
41 #include <asm/uv/uv_hub.h>
44 * Test if a physical address is a valid GRU GSEG address
46 static inline int is_gru_paddr(unsigned long paddr
)
48 return paddr
>= gru_start_paddr
&& paddr
< gru_end_paddr
;
52 * Find the vma of a GRU segment. Caller must hold mmap_sem.
54 struct vm_area_struct
*gru_find_vma(unsigned long vaddr
)
56 struct vm_area_struct
*vma
;
58 vma
= find_vma(current
->mm
, vaddr
);
59 if (vma
&& vma
->vm_start
<= vaddr
&& vma
->vm_ops
== &gru_vm_ops
)
65 * Find and lock the gts that contains the specified user vaddr.
68 * - *gts with the mmap_sem locked for read and the GTS locked.
69 * - NULL if vaddr invalid OR is not a valid GSEG vaddr.
72 static struct gru_thread_state
*gru_find_lock_gts(unsigned long vaddr
)
74 struct mm_struct
*mm
= current
->mm
;
75 struct vm_area_struct
*vma
;
76 struct gru_thread_state
*gts
= NULL
;
78 down_read(&mm
->mmap_sem
);
79 vma
= gru_find_vma(vaddr
);
81 gts
= gru_find_thread_state(vma
, TSID(vaddr
, vma
));
83 mutex_lock(>s
->ts_ctxlock
);
85 up_read(&mm
->mmap_sem
);
89 static struct gru_thread_state
*gru_alloc_locked_gts(unsigned long vaddr
)
91 struct mm_struct
*mm
= current
->mm
;
92 struct vm_area_struct
*vma
;
93 struct gru_thread_state
*gts
= NULL
;
95 down_write(&mm
->mmap_sem
);
96 vma
= gru_find_vma(vaddr
);
98 gts
= gru_alloc_thread_state(vma
, TSID(vaddr
, vma
));
100 mutex_lock(>s
->ts_ctxlock
);
101 downgrade_write(&mm
->mmap_sem
);
103 up_write(&mm
->mmap_sem
);
110 * Unlock a GTS that was previously locked with gru_find_lock_gts().
112 static void gru_unlock_gts(struct gru_thread_state
*gts
)
114 mutex_unlock(>s
->ts_ctxlock
);
115 up_read(¤t
->mm
->mmap_sem
);
119 * Set a CB.istatus to active using a user virtual address. This must be done
120 * just prior to a TFH RESTART. The new cb.istatus is an in-cache status ONLY.
121 * If the line is evicted, the status may be lost. The in-cache update
122 * is necessary to prevent the user from seeing a stale cb.istatus that will
123 * change as soon as the TFH restart is complete. Races may cause an
124 * occasional failure to clear the cb.istatus, but that is ok.
126 static void gru_cb_set_istatus_active(struct gru_instruction_bits
*cbk
)
129 cbk
->istatus
= CBS_ACTIVE
;
134 * Convert a interrupt IRQ to a pointer to the GRU GTS that caused the
135 * interrupt. Interrupts are always sent to a cpu on the blade that contains the
136 * GRU (except for headless blades which are not currently supported). A blade
137 * has N grus; a block of N consecutive IRQs is assigned to the GRUs. The IRQ
138 * number uniquely identifies the GRU chiplet on the local blade that caused the
139 * interrupt. Always called in interrupt context.
141 static inline struct gru_state
*irq_to_gru(int irq
)
143 return &gru_base
[uv_numa_blade_id()]->bs_grus
[irq
- IRQ_GRU
];
149 * The GRU has an array of fault maps. A map is private to a cpu
150 * Only one cpu will be accessing a cpu's fault map.
152 * This function scans the cpu-private fault map & clears all bits that
153 * are set. The function returns a bitmap that indicates the bits that
154 * were cleared. Note that sense the maps may be updated asynchronously by
155 * the GRU, atomic operations must be used to clear bits.
157 static void get_clear_fault_map(struct gru_state
*gru
,
158 struct gru_tlb_fault_map
*imap
,
159 struct gru_tlb_fault_map
*dmap
)
162 struct gru_tlb_fault_map
*tfm
;
164 tfm
= get_tfm_for_cpu(gru
, gru_cpu_fault_map_id());
165 prefetchw(tfm
); /* Helps on hardware, required for emulator */
166 for (i
= 0; i
< BITS_TO_LONGS(GRU_NUM_CBE
); i
++) {
167 k
= tfm
->fault_bits
[i
];
169 k
= xchg(&tfm
->fault_bits
[i
], 0UL);
170 imap
->fault_bits
[i
] = k
;
171 k
= tfm
->done_bits
[i
];
173 k
= xchg(&tfm
->done_bits
[i
], 0UL);
174 dmap
->fault_bits
[i
] = k
;
178 * Not functionally required but helps performance. (Required
181 gru_flush_cache(tfm
);
185 * Atomic (interrupt context) & non-atomic (user context) functions to
186 * convert a vaddr into a physical address. The size of the page
187 * is returned in pageshift.
191 * 1 - (atomic only) try again in non-atomic context
193 static int non_atomic_pte_lookup(struct vm_area_struct
*vma
,
194 unsigned long vaddr
, int write
,
195 unsigned long *paddr
, int *pageshift
)
199 /* ZZZ Need to handle HUGE pages */
200 if (is_vm_hugetlb_page(vma
))
202 *pageshift
= PAGE_SHIFT
;
204 (current
, current
->mm
, vaddr
, 1, write
, 0, &page
, NULL
) <= 0)
206 *paddr
= page_to_phys(page
);
214 * Convert a user virtual address to a physical address
215 * Only supports Intel large pages (2MB only) on x86_64.
216 * ZZZ - hugepage support is incomplete
218 * NOTE: mmap_sem is already held on entry to this function. This
219 * guarantees existence of the page tables.
221 static int atomic_pte_lookup(struct vm_area_struct
*vma
, unsigned long vaddr
,
222 int write
, unsigned long *paddr
, int *pageshift
)
229 pgdp
= pgd_offset(vma
->vm_mm
, vaddr
);
230 if (unlikely(pgd_none(*pgdp
)))
233 pudp
= pud_offset(pgdp
, vaddr
);
234 if (unlikely(pud_none(*pudp
)))
237 pmdp
= pmd_offset(pudp
, vaddr
);
238 if (unlikely(pmd_none(*pmdp
)))
241 if (unlikely(pmd_large(*pmdp
)))
242 pte
= *(pte_t
*) pmdp
;
245 pte
= *pte_offset_kernel(pmdp
, vaddr
);
247 if (unlikely(!pte_present(pte
) ||
248 (write
&& (!pte_write(pte
) || !pte_dirty(pte
)))))
251 *paddr
= pte_pfn(pte
) << PAGE_SHIFT
;
252 #ifdef CONFIG_HUGETLB_PAGE
253 *pageshift
= is_vm_hugetlb_page(vma
) ? HPAGE_SHIFT
: PAGE_SHIFT
;
255 *pageshift
= PAGE_SHIFT
;
264 static int gru_vtop(struct gru_thread_state
*gts
, unsigned long vaddr
,
265 int write
, int atomic
, unsigned long *gpa
, int *pageshift
)
267 struct mm_struct
*mm
= gts
->ts_mm
;
268 struct vm_area_struct
*vma
;
272 vma
= find_vma(mm
, vaddr
);
277 * Atomic lookup is faster & usually works even if called in non-atomic
280 rmb(); /* Must/check ms_range_active before loading PTEs */
281 ret
= atomic_pte_lookup(vma
, vaddr
, write
, &paddr
, &ps
);
285 if (non_atomic_pte_lookup(vma
, vaddr
, write
, &paddr
, &ps
))
288 if (is_gru_paddr(paddr
))
290 paddr
= paddr
& ~((1UL << ps
) - 1);
291 *gpa
= uv_soc_phys_ram_to_gpa(paddr
);
303 * Drop a TLB entry into the GRU. The fault is described by info in an TFH.
305 * cb Address of user CBR. Null if not running in user context
307 * 0 = dropin, exception, or switch to UPM successful
308 * 1 = range invalidate active
312 static int gru_try_dropin(struct gru_thread_state
*gts
,
313 struct gru_tlb_fault_handle
*tfh
,
314 struct gru_instruction_bits
*cbk
)
316 int pageshift
= 0, asid
, write
, ret
, atomic
= !cbk
;
317 unsigned long gpa
= 0, vaddr
= 0;
320 * NOTE: The GRU contains magic hardware that eliminates races between
321 * TLB invalidates and TLB dropins. If an invalidate occurs
322 * in the window between reading the TFH and the subsequent TLB dropin,
323 * the dropin is ignored. This eliminates the need for additional locks.
327 * Error if TFH state is IDLE or FMM mode & the user issuing a UPM call.
328 * Might be a hardware race OR a stupid user. Ignore FMM because FMM
329 * is a transient state.
331 if (tfh
->status
!= TFHSTATUS_EXCEPTION
) {
332 gru_flush_cache(tfh
);
333 if (tfh
->status
!= TFHSTATUS_EXCEPTION
)
334 goto failnoexception
;
335 STAT(tfh_stale_on_fault
);
337 if (tfh
->state
== TFHSTATE_IDLE
)
339 if (tfh
->state
== TFHSTATE_MISS_FMM
&& cbk
)
342 write
= (tfh
->cause
& TFHCAUSE_TLB_MOD
) != 0;
343 vaddr
= tfh
->missvaddr
;
344 asid
= tfh
->missasid
;
348 rmb(); /* TFH must be cache resident before reading ms_range_active */
351 * TFH is cache resident - at least briefly. Fail the dropin
352 * if a range invalidate is active.
354 if (atomic_read(>s
->ts_gms
->ms_range_active
))
357 ret
= gru_vtop(gts
, vaddr
, write
, atomic
, &gpa
, &pageshift
);
363 if (!(gts
->ts_sizeavail
& GRU_SIZEAVAIL(pageshift
))) {
364 gts
->ts_sizeavail
|= GRU_SIZEAVAIL(pageshift
);
365 if (atomic
|| !gru_update_cch(gts
, 0)) {
366 gts
->ts_force_cch_reload
= 1;
370 gru_cb_set_istatus_active(cbk
);
371 tfh_write_restart(tfh
, gpa
, GAA_RAM
, vaddr
, asid
, write
,
372 GRU_PAGESIZE(pageshift
));
375 "%s: tfh 0x%p, vaddr 0x%lx, asid 0x%x, ps %d, gpa 0x%lx\n",
376 ret
? "non-atomic" : "atomic", tfh
, vaddr
, asid
,
381 /* No asid (delayed unload). */
382 STAT(tlb_dropin_fail_no_asid
);
383 gru_dbg(grudev
, "FAILED no_asid tfh: 0x%p, vaddr 0x%lx\n", tfh
, vaddr
);
385 tfh_user_polling_mode(tfh
);
387 gru_flush_cache(tfh
);
391 /* Atomic failure switch CBR to UPM */
392 tfh_user_polling_mode(tfh
);
393 STAT(tlb_dropin_fail_upm
);
394 gru_dbg(grudev
, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh
, vaddr
);
398 /* FMM state on UPM call */
399 gru_flush_cache(tfh
);
400 STAT(tlb_dropin_fail_fmm
);
401 gru_dbg(grudev
, "FAILED fmm tfh: 0x%p, state %d\n", tfh
, tfh
->state
);
405 /* TFH status did not show exception pending */
406 gru_flush_cache(tfh
);
408 gru_flush_cache(cbk
);
409 STAT(tlb_dropin_fail_no_exception
);
410 gru_dbg(grudev
, "FAILED non-exception tfh: 0x%p, status %d, state %d\n",
411 tfh
, tfh
->status
, tfh
->state
);
415 /* TFH state was idle - no miss pending */
416 gru_flush_cache(tfh
);
418 gru_flush_cache(cbk
);
419 STAT(tlb_dropin_fail_idle
);
420 gru_dbg(grudev
, "FAILED idle tfh: 0x%p, state %d\n", tfh
, tfh
->state
);
424 /* All errors (atomic & non-atomic) switch CBR to EXCEPTION state */
426 STAT(tlb_dropin_fail_invalid
);
427 gru_dbg(grudev
, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh
, vaddr
);
431 /* Range invalidate active. Switch to UPM iff atomic */
433 tfh_user_polling_mode(tfh
);
435 gru_flush_cache(tfh
);
436 STAT(tlb_dropin_fail_range_active
);
437 gru_dbg(grudev
, "FAILED range active: tfh 0x%p, vaddr 0x%lx\n",
443 * Process an external interrupt from the GRU. This interrupt is
444 * caused by a TLB miss.
445 * Note that this is the interrupt handler that is registered with linux
446 * interrupt handlers.
448 irqreturn_t
gru_intr(int irq
, void *dev_id
)
450 struct gru_state
*gru
;
451 struct gru_tlb_fault_map imap
, dmap
;
452 struct gru_thread_state
*gts
;
453 struct gru_tlb_fault_handle
*tfh
= NULL
;
458 gru
= irq_to_gru(irq
);
460 dev_err(grudev
, "GRU: invalid interrupt: cpu %d, irq %d\n",
461 raw_smp_processor_id(), irq
);
464 get_clear_fault_map(gru
, &imap
, &dmap
);
466 for_each_cbr_in_tfm(cbrnum
, dmap
.fault_bits
) {
467 complete(gru
->gs_blade
->bs_async_wq
);
468 gru_dbg(grudev
, "gid %d, cbr_done %d, done %d\n",
469 gru
->gs_gid
, cbrnum
, gru
->gs_blade
->bs_async_wq
->done
);
472 for_each_cbr_in_tfm(cbrnum
, imap
.fault_bits
) {
473 tfh
= get_tfh_by_index(gru
, cbrnum
);
474 prefetchw(tfh
); /* Helps on hdw, required for emulator */
477 * When hardware sets a bit in the faultmap, it implicitly
478 * locks the GRU context so that it cannot be unloaded.
479 * The gts cannot change until a TFH start/writestart command
482 ctxnum
= tfh
->ctxnum
;
483 gts
= gru
->gs_gts
[ctxnum
];
486 * This is running in interrupt context. Trylock the mmap_sem.
487 * If it fails, retry the fault in user context.
489 if (!gts
->ts_force_cch_reload
&&
490 down_read_trylock(>s
->ts_mm
->mmap_sem
)) {
491 gts
->ustats
.fmm_tlbdropin
++;
492 gru_try_dropin(gts
, tfh
, NULL
);
493 up_read(>s
->ts_mm
->mmap_sem
);
495 tfh_user_polling_mode(tfh
);
496 STAT(intr_mm_lock_failed
);
503 static int gru_user_dropin(struct gru_thread_state
*gts
,
504 struct gru_tlb_fault_handle
*tfh
,
507 struct gru_mm_struct
*gms
= gts
->ts_gms
;
510 gts
->ustats
.upm_tlbdropin
++;
512 wait_event(gms
->ms_wait_queue
,
513 atomic_read(&gms
->ms_range_active
) == 0);
514 prefetchw(tfh
); /* Helps on hdw, required for emulator */
515 ret
= gru_try_dropin(gts
, tfh
, cb
);
518 STAT(call_os_wait_queue
);
523 * This interface is called as a result of a user detecting a "call OS" bit
524 * in a user CB. Normally means that a TLB fault has occurred.
525 * cb - user virtual address of the CB
527 int gru_handle_user_call_os(unsigned long cb
)
529 struct gru_tlb_fault_handle
*tfh
;
530 struct gru_thread_state
*gts
;
532 int ucbnum
, cbrnum
, ret
= -EINVAL
;
535 gru_dbg(grudev
, "address 0x%lx\n", cb
);
537 /* sanity check the cb pointer */
538 ucbnum
= get_cb_number((void *)cb
);
539 if ((cb
& (GRU_HANDLE_STRIDE
- 1)) || ucbnum
>= GRU_NUM_CB
)
542 gts
= gru_find_lock_gts(cb
);
546 if (ucbnum
>= gts
->ts_cbr_au_count
* GRU_CBR_AU_SIZE
)
550 * If force_unload is set, the UPM TLB fault is phony. The task
551 * has migrated to another node and the GSEG must be moved. Just
552 * unload the context. The task will page fault and assign a new
555 if (gts
->ts_tgid_owner
== current
->tgid
&& gts
->ts_blade
>= 0 &&
556 gts
->ts_blade
!= uv_numa_blade_id()) {
557 STAT(call_os_offnode_reference
);
558 gts
->ts_force_unload
= 1;
562 * CCH may contain stale data if ts_force_cch_reload is set.
564 if (gts
->ts_gru
&& gts
->ts_force_cch_reload
) {
565 gts
->ts_force_cch_reload
= 0;
566 gru_update_cch(gts
, 0);
570 cbrnum
= thread_cbr_number(gts
, ucbnum
);
571 if (gts
->ts_force_unload
) {
572 gru_unload_context(gts
, 1);
573 } else if (gts
->ts_gru
) {
574 tfh
= get_tfh_by_index(gts
->ts_gru
, cbrnum
);
575 cbk
= get_gseg_base_address_cb(gts
->ts_gru
->gs_gru_base_vaddr
,
576 gts
->ts_ctxnum
, ucbnum
);
577 ret
= gru_user_dropin(gts
, tfh
, cbk
);
585 * Fetch the exception detail information for a CB that terminated with
588 int gru_get_exception_detail(unsigned long arg
)
590 struct control_block_extended_exc_detail excdet
;
591 struct gru_control_block_extended
*cbe
;
592 struct gru_thread_state
*gts
;
593 int ucbnum
, cbrnum
, ret
;
595 STAT(user_exception
);
596 if (copy_from_user(&excdet
, (void __user
*)arg
, sizeof(excdet
)))
599 gru_dbg(grudev
, "address 0x%lx\n", excdet
.cb
);
600 gts
= gru_find_lock_gts(excdet
.cb
);
604 ucbnum
= get_cb_number((void *)excdet
.cb
);
605 if (ucbnum
>= gts
->ts_cbr_au_count
* GRU_CBR_AU_SIZE
) {
607 } else if (gts
->ts_gru
) {
608 cbrnum
= thread_cbr_number(gts
, ucbnum
);
609 cbe
= get_cbe_by_index(gts
->ts_gru
, cbrnum
);
610 gru_flush_cache(cbe
); /* CBE not coherent */
611 excdet
.opc
= cbe
->opccpy
;
612 excdet
.exopc
= cbe
->exopccpy
;
613 excdet
.ecause
= cbe
->ecause
;
614 excdet
.exceptdet0
= cbe
->idef1upd
;
615 excdet
.exceptdet1
= cbe
->idef3upd
;
616 excdet
.cbrstate
= cbe
->cbrstate
;
617 excdet
.cbrexecstatus
= cbe
->cbrexecstatus
;
618 gru_flush_cache(cbe
);
626 "cb 0x%lx, op %d, exopc %d, cbrstate %d, cbrexecstatus 0x%x, ecause 0x%x, "
627 "exdet0 0x%lx, exdet1 0x%x\n",
628 excdet
.cb
, excdet
.opc
, excdet
.exopc
, excdet
.cbrstate
, excdet
.cbrexecstatus
,
629 excdet
.ecause
, excdet
.exceptdet0
, excdet
.exceptdet1
);
630 if (!ret
&& copy_to_user((void __user
*)arg
, &excdet
, sizeof(excdet
)))
636 * User request to unload a context. Content is saved for possible reload.
638 static int gru_unload_all_contexts(void)
640 struct gru_thread_state
*gts
;
641 struct gru_state
*gru
;
644 if (!capable(CAP_SYS_ADMIN
))
647 gru
= GID_TO_GRU(gid
);
648 spin_lock(&gru
->gs_lock
);
649 for (ctxnum
= 0; ctxnum
< GRU_NUM_CCH
; ctxnum
++) {
650 gts
= gru
->gs_gts
[ctxnum
];
651 if (gts
&& mutex_trylock(>s
->ts_ctxlock
)) {
652 spin_unlock(&gru
->gs_lock
);
653 gru_unload_context(gts
, 1);
654 mutex_unlock(>s
->ts_ctxlock
);
655 spin_lock(&gru
->gs_lock
);
658 spin_unlock(&gru
->gs_lock
);
663 int gru_user_unload_context(unsigned long arg
)
665 struct gru_thread_state
*gts
;
666 struct gru_unload_context_req req
;
668 STAT(user_unload_context
);
669 if (copy_from_user(&req
, (void __user
*)arg
, sizeof(req
)))
672 gru_dbg(grudev
, "gseg 0x%lx\n", req
.gseg
);
675 return gru_unload_all_contexts();
677 gts
= gru_find_lock_gts(req
.gseg
);
682 gru_unload_context(gts
, 1);
689 * User request to flush a range of virtual addresses from the GRU TLB
690 * (Mainly for testing).
692 int gru_user_flush_tlb(unsigned long arg
)
694 struct gru_thread_state
*gts
;
695 struct gru_flush_tlb_req req
;
696 struct gru_mm_struct
*gms
;
698 STAT(user_flush_tlb
);
699 if (copy_from_user(&req
, (void __user
*)arg
, sizeof(req
)))
702 gru_dbg(grudev
, "gseg 0x%lx, vaddr 0x%lx, len 0x%lx\n", req
.gseg
,
705 gts
= gru_find_lock_gts(req
.gseg
);
711 gru_flush_tlb_range(gms
, req
.vaddr
, req
.len
);
717 * Fetch GSEG statisticss
719 long gru_get_gseg_statistics(unsigned long arg
)
721 struct gru_thread_state
*gts
;
722 struct gru_get_gseg_statistics_req req
;
724 if (copy_from_user(&req
, (void __user
*)arg
, sizeof(req
)))
728 * The library creates arrays of contexts for threaded programs.
729 * If no gts exists in the array, the context has never been used & all
730 * statistics are implicitly 0.
732 gts
= gru_find_lock_gts(req
.gseg
);
734 memcpy(&req
.stats
, >s
->ustats
, sizeof(gts
->ustats
));
737 memset(&req
.stats
, 0, sizeof(gts
->ustats
));
740 if (copy_to_user((void __user
*)arg
, &req
, sizeof(req
)))
747 * Register the current task as the user of the GSEG slice.
748 * Needed for TLB fault interrupt targeting.
750 int gru_set_context_option(unsigned long arg
)
752 struct gru_thread_state
*gts
;
753 struct gru_set_context_option_req req
;
756 STAT(set_context_option
);
757 if (copy_from_user(&req
, (void __user
*)arg
, sizeof(req
)))
759 gru_dbg(grudev
, "op %d, gseg 0x%lx, value1 0x%lx\n", req
.op
, req
.gseg
, req
.val1
);
761 gts
= gru_alloc_locked_gts(req
.gseg
);
767 /* Register the current task as the GSEG owner */
768 gts
->ts_tgid_owner
= current
->tgid
;
770 case sco_cch_req_slice
:
771 /* Set the CCH slice option */
772 gts
->ts_cch_req_slice
= req
.val1
& 3;