2 * SN Platform GRU Driver
4 * DRIVER TABLE MANAGER + GRU CONTEXT LOAD/UNLOAD
6 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 #include <linux/kernel.h>
24 #include <linux/slab.h>
26 #include <linux/spinlock.h>
27 #include <linux/sched.h>
28 #include <linux/device.h>
29 #include <linux/list.h>
30 #include <linux/err.h>
31 #include <asm/uv/uv_hub.h>
33 #include "grutables.h"
34 #include "gruhandles.h"
36 unsigned long gru_options __read_mostly
;
38 static struct device_driver gru_driver
= {
42 static struct device gru_device
= {
44 .driver
= &gru_driver
,
47 struct device
*grudev
= &gru_device
;
50 * Select a gru fault map to be used by the current cpu. Note that
51 * multiple cpus may be using the same map.
52 * ZZZ should be inline but did not work on emulator
54 int gru_cpu_fault_map_id(void)
56 int cpu
= smp_processor_id();
59 core
= uv_cpu_core_number(cpu
);
60 id
= core
+ UV_MAX_INT_CORES
* uv_cpu_socket_number(cpu
);
64 /*--------- ASID Management -------------------------------------------
66 * Initially, assign asids sequentially from MIN_ASID .. MAX_ASID.
67 * Once MAX is reached, flush the TLB & start over. However,
68 * some asids may still be in use. There won't be many (percentage wise) still
69 * in use. Search active contexts & determine the value of the first
70 * asid in use ("x"s below). Set "limit" to this value.
71 * This defines a block of assignable asids.
73 * When "limit" is reached, search forward from limit+1 and determine the
74 * next block of assignable asids.
76 * Repeat until MAX_ASID is reached, then start over again.
78 * Each time MAX_ASID is reached, increment the asid generation. Since
79 * the search for in-use asids only checks contexts with GRUs currently
80 * assigned, asids in some contexts will be missed. Prior to loading
81 * a context, the asid generation of the GTS asid is rechecked. If it
82 * doesn't match the current generation, a new asid will be assigned.
84 * 0---------------x------------x---------------------x----|
85 * ^-next ^-limit ^-MAX_ASID
87 * All asid manipulation & context loading/unloading is protected by the
91 /* Hit the asid limit. Start over */
92 static int gru_wrap_asid(struct gru_state
*gru
)
94 gru_dbg(grudev
, "gid %d\n", gru
->gs_gid
);
100 /* Find the next chunk of unused asids */
101 static int gru_reset_asid_limit(struct gru_state
*gru
, int asid
)
103 int i
, gid
, inuse_asid
, limit
;
105 gru_dbg(grudev
, "gid %d, asid 0x%x\n", gru
->gs_gid
, asid
);
109 asid
= gru_wrap_asid(gru
);
110 gru_flush_all_tlb(gru
);
113 for (i
= 0; i
< GRU_NUM_CCH
; i
++) {
114 if (!gru
->gs_gts
[i
] || is_kernel_context(gru
->gs_gts
[i
]))
116 inuse_asid
= gru
->gs_gts
[i
]->ts_gms
->ms_asids
[gid
].mt_asid
;
117 gru_dbg(grudev
, "gid %d, gts %p, gms %p, inuse 0x%x, cxt %d\n",
118 gru
->gs_gid
, gru
->gs_gts
[i
], gru
->gs_gts
[i
]->ts_gms
,
120 if (inuse_asid
== asid
) {
124 * empty range: reset the range limit and
128 if (asid
>= MAX_ASID
)
129 asid
= gru_wrap_asid(gru
);
134 if ((inuse_asid
> asid
) && (inuse_asid
< limit
))
137 gru
->gs_asid_limit
= limit
;
139 gru_dbg(grudev
, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru
->gs_gid
,
144 /* Assign a new ASID to a thread context. */
145 static int gru_assign_asid(struct gru_state
*gru
)
149 gru
->gs_asid
+= ASID_INC
;
151 if (asid
>= gru
->gs_asid_limit
)
152 asid
= gru_reset_asid_limit(gru
, asid
);
154 gru_dbg(grudev
, "gid %d, asid 0x%x\n", gru
->gs_gid
, asid
);
159 * Clear n bits in a word. Return a word indicating the bits that were cleared.
160 * Optionally, build an array of chars that contain the bit numbers allocated.
162 static unsigned long reserve_resources(unsigned long *p
, int n
, int mmax
,
165 unsigned long bits
= 0;
169 i
= find_first_bit(p
, mmax
);
180 unsigned long gru_reserve_cb_resources(struct gru_state
*gru
, int cbr_au_count
,
183 return reserve_resources(&gru
->gs_cbr_map
, cbr_au_count
, GRU_CBR_AU
,
187 unsigned long gru_reserve_ds_resources(struct gru_state
*gru
, int dsr_au_count
,
190 return reserve_resources(&gru
->gs_dsr_map
, dsr_au_count
, GRU_DSR_AU
,
194 static void reserve_gru_resources(struct gru_state
*gru
,
195 struct gru_thread_state
*gts
)
197 gru
->gs_active_contexts
++;
199 gru_reserve_cb_resources(gru
, gts
->ts_cbr_au_count
,
202 gru_reserve_ds_resources(gru
, gts
->ts_dsr_au_count
, NULL
);
205 static void free_gru_resources(struct gru_state
*gru
,
206 struct gru_thread_state
*gts
)
208 gru
->gs_active_contexts
--;
209 gru
->gs_cbr_map
|= gts
->ts_cbr_map
;
210 gru
->gs_dsr_map
|= gts
->ts_dsr_map
;
214 * Check if a GRU has sufficient free resources to satisfy an allocation
215 * request. Note: GRU locks may or may not be held when this is called. If
216 * not held, recheck after acquiring the appropriate locks.
218 * Returns 1 if sufficient resources, 0 if not
220 static int check_gru_resources(struct gru_state
*gru
, int cbr_au_count
,
221 int dsr_au_count
, int max_active_contexts
)
223 return hweight64(gru
->gs_cbr_map
) >= cbr_au_count
224 && hweight64(gru
->gs_dsr_map
) >= dsr_au_count
225 && gru
->gs_active_contexts
< max_active_contexts
;
229 * TLB manangment requires tracking all GRU chiplets that have loaded a GSEG
232 static int gru_load_mm_tracker(struct gru_state
*gru
,
233 struct gru_thread_state
*gts
)
235 struct gru_mm_struct
*gms
= gts
->ts_gms
;
236 struct gru_mm_tracker
*asids
= &gms
->ms_asids
[gru
->gs_gid
];
237 unsigned short ctxbitmap
= (1 << gts
->ts_ctxnum
);
240 spin_lock(&gms
->ms_asid_lock
);
241 asid
= asids
->mt_asid
;
243 spin_lock(&gru
->gs_asid_lock
);
244 if (asid
== 0 || (asids
->mt_ctxbitmap
== 0 && asids
->mt_asid_gen
!=
246 asid
= gru_assign_asid(gru
);
247 asids
->mt_asid
= asid
;
248 asids
->mt_asid_gen
= gru
->gs_asid_gen
;
253 spin_unlock(&gru
->gs_asid_lock
);
255 BUG_ON(asids
->mt_ctxbitmap
& ctxbitmap
);
256 asids
->mt_ctxbitmap
|= ctxbitmap
;
257 if (!test_bit(gru
->gs_gid
, gms
->ms_asidmap
))
258 __set_bit(gru
->gs_gid
, gms
->ms_asidmap
);
259 spin_unlock(&gms
->ms_asid_lock
);
262 "gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
263 gru
->gs_gid
, gts
, gms
, gts
->ts_ctxnum
, asid
,
268 static void gru_unload_mm_tracker(struct gru_state
*gru
,
269 struct gru_thread_state
*gts
)
271 struct gru_mm_struct
*gms
= gts
->ts_gms
;
272 struct gru_mm_tracker
*asids
;
273 unsigned short ctxbitmap
;
275 asids
= &gms
->ms_asids
[gru
->gs_gid
];
276 ctxbitmap
= (1 << gts
->ts_ctxnum
);
277 spin_lock(&gms
->ms_asid_lock
);
278 spin_lock(&gru
->gs_asid_lock
);
279 BUG_ON((asids
->mt_ctxbitmap
& ctxbitmap
) != ctxbitmap
);
280 asids
->mt_ctxbitmap
^= ctxbitmap
;
281 gru_dbg(grudev
, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
282 gru
->gs_gid
, gts
, gms
, gts
->ts_ctxnum
, gms
->ms_asidmap
[0]);
283 spin_unlock(&gru
->gs_asid_lock
);
284 spin_unlock(&gms
->ms_asid_lock
);
288 * Decrement the reference count on a GTS structure. Free the structure
289 * if the reference count goes to zero.
291 void gts_drop(struct gru_thread_state
*gts
)
293 if (gts
&& atomic_dec_return(>s
->ts_refcnt
) == 0) {
295 gru_drop_mmu_notifier(gts
->ts_gms
);
302 * Locate the GTS structure for the current thread.
304 static struct gru_thread_state
*gru_find_current_gts_nolock(struct gru_vma_data
307 struct gru_thread_state
*gts
;
309 list_for_each_entry(gts
, &vdata
->vd_head
, ts_next
)
310 if (gts
->ts_tsid
== tsid
)
316 * Allocate a thread state structure.
318 struct gru_thread_state
*gru_alloc_gts(struct vm_area_struct
*vma
,
319 int cbr_au_count
, int dsr_au_count
, int options
, int tsid
)
321 struct gru_thread_state
*gts
;
322 struct gru_mm_struct
*gms
;
325 bytes
= DSR_BYTES(dsr_au_count
) + CBR_BYTES(cbr_au_count
);
326 bytes
+= sizeof(struct gru_thread_state
);
327 gts
= kmalloc(bytes
, GFP_KERNEL
);
329 return ERR_PTR(-ENOMEM
);
332 memset(gts
, 0, sizeof(struct gru_thread_state
)); /* zero out header */
333 atomic_set(>s
->ts_refcnt
, 1);
334 mutex_init(>s
->ts_ctxlock
);
335 gts
->ts_cbr_au_count
= cbr_au_count
;
336 gts
->ts_dsr_au_count
= dsr_au_count
;
337 gts
->ts_user_options
= options
;
338 gts
->ts_user_blade_id
= -1;
339 gts
->ts_user_chiplet_id
= -1;
341 gts
->ts_ctxnum
= NULLCTX
;
342 gts
->ts_tlb_int_select
= -1;
343 gts
->ts_cch_req_slice
= -1;
344 gts
->ts_sizeavail
= GRU_SIZEAVAIL(PAGE_SHIFT
);
346 gts
->ts_mm
= current
->mm
;
348 gms
= gru_register_mmu_notifier();
354 gru_dbg(grudev
, "alloc gts %p\n", gts
);
359 return ERR_CAST(gms
);
363 * Allocate a vma private data structure.
365 struct gru_vma_data
*gru_alloc_vma_data(struct vm_area_struct
*vma
, int tsid
)
367 struct gru_vma_data
*vdata
= NULL
;
369 vdata
= kmalloc(sizeof(*vdata
), GFP_KERNEL
);
374 INIT_LIST_HEAD(&vdata
->vd_head
);
375 spin_lock_init(&vdata
->vd_lock
);
376 gru_dbg(grudev
, "alloc vdata %p\n", vdata
);
381 * Find the thread state structure for the current thread.
383 struct gru_thread_state
*gru_find_thread_state(struct vm_area_struct
*vma
,
386 struct gru_vma_data
*vdata
= vma
->vm_private_data
;
387 struct gru_thread_state
*gts
;
389 spin_lock(&vdata
->vd_lock
);
390 gts
= gru_find_current_gts_nolock(vdata
, tsid
);
391 spin_unlock(&vdata
->vd_lock
);
392 gru_dbg(grudev
, "vma %p, gts %p\n", vma
, gts
);
397 * Allocate a new thread state for a GSEG. Note that races may allow
398 * another thread to race to create a gts.
400 struct gru_thread_state
*gru_alloc_thread_state(struct vm_area_struct
*vma
,
403 struct gru_vma_data
*vdata
= vma
->vm_private_data
;
404 struct gru_thread_state
*gts
, *ngts
;
406 gts
= gru_alloc_gts(vma
, vdata
->vd_cbr_au_count
, vdata
->vd_dsr_au_count
,
407 vdata
->vd_user_options
, tsid
);
411 spin_lock(&vdata
->vd_lock
);
412 ngts
= gru_find_current_gts_nolock(vdata
, tsid
);
416 STAT(gts_double_allocate
);
418 list_add(>s
->ts_next
, &vdata
->vd_head
);
420 spin_unlock(&vdata
->vd_lock
);
421 gru_dbg(grudev
, "vma %p, gts %p\n", vma
, gts
);
426 * Free the GRU context assigned to the thread state.
428 static void gru_free_gru_context(struct gru_thread_state
*gts
)
430 struct gru_state
*gru
;
433 gru_dbg(grudev
, "gts %p, gid %d\n", gts
, gru
->gs_gid
);
435 spin_lock(&gru
->gs_lock
);
436 gru
->gs_gts
[gts
->ts_ctxnum
] = NULL
;
437 free_gru_resources(gru
, gts
);
438 BUG_ON(test_bit(gts
->ts_ctxnum
, &gru
->gs_context_map
) == 0);
439 __clear_bit(gts
->ts_ctxnum
, &gru
->gs_context_map
);
440 gts
->ts_ctxnum
= NULLCTX
;
443 spin_unlock(&gru
->gs_lock
);
450 * Prefetching cachelines help hardware performance.
451 * (Strictly a performance enhancement. Not functionally required).
453 static void prefetch_data(void *p
, int num
, int stride
)
461 static inline long gru_copy_handle(void *d
, void *s
)
463 memcpy(d
, s
, GRU_HANDLE_BYTES
);
464 return GRU_HANDLE_BYTES
;
467 static void gru_prefetch_context(void *gseg
, void *cb
, void *cbe
,
468 unsigned long cbrmap
, unsigned long length
)
472 prefetch_data(gseg
+ GRU_DS_BASE
, length
/ GRU_CACHE_LINE_BYTES
,
473 GRU_CACHE_LINE_BYTES
);
475 for_each_cbr_in_allocation_map(i
, &cbrmap
, scr
) {
476 prefetch_data(cb
, 1, GRU_CACHE_LINE_BYTES
);
477 prefetch_data(cbe
+ i
* GRU_HANDLE_STRIDE
, 1,
478 GRU_CACHE_LINE_BYTES
);
479 cb
+= GRU_HANDLE_STRIDE
;
483 static void gru_load_context_data(void *save
, void *grubase
, int ctxnum
,
484 unsigned long cbrmap
, unsigned long dsrmap
,
487 void *gseg
, *cb
, *cbe
;
488 unsigned long length
;
491 gseg
= grubase
+ ctxnum
* GRU_GSEG_STRIDE
;
492 cb
= gseg
+ GRU_CB_BASE
;
493 cbe
= grubase
+ GRU_CBE_BASE
;
494 length
= hweight64(dsrmap
) * GRU_DSR_AU_BYTES
;
495 gru_prefetch_context(gseg
, cb
, cbe
, cbrmap
, length
);
497 for_each_cbr_in_allocation_map(i
, &cbrmap
, scr
) {
499 save
+= gru_copy_handle(cb
, save
);
500 save
+= gru_copy_handle(cbe
+ i
* GRU_HANDLE_STRIDE
,
503 memset(cb
, 0, GRU_CACHE_LINE_BYTES
);
504 memset(cbe
+ i
* GRU_HANDLE_STRIDE
, 0,
505 GRU_CACHE_LINE_BYTES
);
507 /* Flush CBE to hide race in context restart */
509 gru_flush_cache(cbe
+ i
* GRU_HANDLE_STRIDE
);
510 cb
+= GRU_HANDLE_STRIDE
;
514 memcpy(gseg
+ GRU_DS_BASE
, save
, length
);
516 memset(gseg
+ GRU_DS_BASE
, 0, length
);
519 static void gru_unload_context_data(void *save
, void *grubase
, int ctxnum
,
520 unsigned long cbrmap
, unsigned long dsrmap
)
522 void *gseg
, *cb
, *cbe
;
523 unsigned long length
;
526 gseg
= grubase
+ ctxnum
* GRU_GSEG_STRIDE
;
527 cb
= gseg
+ GRU_CB_BASE
;
528 cbe
= grubase
+ GRU_CBE_BASE
;
529 length
= hweight64(dsrmap
) * GRU_DSR_AU_BYTES
;
531 /* CBEs may not be coherent. Flush them from cache */
532 for_each_cbr_in_allocation_map(i
, &cbrmap
, scr
)
533 gru_flush_cache(cbe
+ i
* GRU_HANDLE_STRIDE
);
534 mb(); /* Let the CL flush complete */
536 gru_prefetch_context(gseg
, cb
, cbe
, cbrmap
, length
);
538 for_each_cbr_in_allocation_map(i
, &cbrmap
, scr
) {
539 save
+= gru_copy_handle(save
, cb
);
540 save
+= gru_copy_handle(save
, cbe
+ i
* GRU_HANDLE_STRIDE
);
541 cb
+= GRU_HANDLE_STRIDE
;
543 memcpy(save
, gseg
+ GRU_DS_BASE
, length
);
546 void gru_unload_context(struct gru_thread_state
*gts
, int savestate
)
548 struct gru_state
*gru
= gts
->ts_gru
;
549 struct gru_context_configuration_handle
*cch
;
550 int ctxnum
= gts
->ts_ctxnum
;
552 if (!is_kernel_context(gts
))
553 zap_vma_ptes(gts
->ts_vma
, UGRUADDR(gts
), GRU_GSEG_PAGESIZE
);
554 cch
= get_cch(gru
->gs_gru_base_vaddr
, ctxnum
);
556 gru_dbg(grudev
, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n",
557 gts
, gts
->ts_cbr_map
, gts
->ts_dsr_map
);
558 lock_cch_handle(cch
);
559 if (cch_interrupt_sync(cch
))
562 if (!is_kernel_context(gts
))
563 gru_unload_mm_tracker(gru
, gts
);
565 gru_unload_context_data(gts
->ts_gdata
, gru
->gs_gru_base_vaddr
,
566 ctxnum
, gts
->ts_cbr_map
,
568 gts
->ts_data_valid
= 1;
571 if (cch_deallocate(cch
))
573 unlock_cch_handle(cch
);
575 gru_free_gru_context(gts
);
579 * Load a GRU context by copying it from the thread data structure in memory
582 void gru_load_context(struct gru_thread_state
*gts
)
584 struct gru_state
*gru
= gts
->ts_gru
;
585 struct gru_context_configuration_handle
*cch
;
586 int i
, err
, asid
, ctxnum
= gts
->ts_ctxnum
;
588 cch
= get_cch(gru
->gs_gru_base_vaddr
, ctxnum
);
589 lock_cch_handle(cch
);
590 cch
->tfm_fault_bit_enable
=
591 (gts
->ts_user_options
== GRU_OPT_MISS_FMM_POLL
592 || gts
->ts_user_options
== GRU_OPT_MISS_FMM_INTR
);
593 cch
->tlb_int_enable
= (gts
->ts_user_options
== GRU_OPT_MISS_FMM_INTR
);
594 if (cch
->tlb_int_enable
) {
595 gts
->ts_tlb_int_select
= gru_cpu_fault_map_id();
596 cch
->tlb_int_select
= gts
->ts_tlb_int_select
;
598 if (gts
->ts_cch_req_slice
>= 0) {
599 cch
->req_slice_set_enable
= 1;
600 cch
->req_slice
= gts
->ts_cch_req_slice
;
602 cch
->req_slice_set_enable
=0;
604 cch
->tfm_done_bit_enable
= 0;
605 cch
->dsr_allocation_map
= gts
->ts_dsr_map
;
606 cch
->cbr_allocation_map
= gts
->ts_cbr_map
;
608 if (is_kernel_context(gts
)) {
609 cch
->unmap_enable
= 1;
610 cch
->tfm_done_bit_enable
= 1;
611 cch
->cb_int_enable
= 1;
612 cch
->tlb_int_select
= 0; /* For now, ints go to cpu 0 */
614 cch
->unmap_enable
= 0;
615 cch
->tfm_done_bit_enable
= 0;
616 cch
->cb_int_enable
= 0;
617 asid
= gru_load_mm_tracker(gru
, gts
);
618 for (i
= 0; i
< 8; i
++) {
619 cch
->asid
[i
] = asid
+ i
;
620 cch
->sizeavail
[i
] = gts
->ts_sizeavail
;
624 err
= cch_allocate(cch
);
627 "err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n",
628 err
, cch
, gts
, gts
->ts_cbr_map
, gts
->ts_dsr_map
);
632 gru_load_context_data(gts
->ts_gdata
, gru
->gs_gru_base_vaddr
, ctxnum
,
633 gts
->ts_cbr_map
, gts
->ts_dsr_map
, gts
->ts_data_valid
);
637 unlock_cch_handle(cch
);
639 gru_dbg(grudev
, "gid %d, gts %p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n",
640 gts
->ts_gru
->gs_gid
, gts
, gts
->ts_cbr_map
, gts
->ts_dsr_map
,
641 (gts
->ts_user_options
== GRU_OPT_MISS_FMM_INTR
), gts
->ts_tlb_int_select
);
645 * Update fields in an active CCH:
646 * - retarget interrupts on local blade
647 * - update sizeavail mask
649 int gru_update_cch(struct gru_thread_state
*gts
)
651 struct gru_context_configuration_handle
*cch
;
652 struct gru_state
*gru
= gts
->ts_gru
;
653 int i
, ctxnum
= gts
->ts_ctxnum
, ret
= 0;
655 cch
= get_cch(gru
->gs_gru_base_vaddr
, ctxnum
);
657 lock_cch_handle(cch
);
658 if (cch
->state
== CCHSTATE_ACTIVE
) {
659 if (gru
->gs_gts
[gts
->ts_ctxnum
] != gts
)
661 if (cch_interrupt(cch
))
663 for (i
= 0; i
< 8; i
++)
664 cch
->sizeavail
[i
] = gts
->ts_sizeavail
;
665 gts
->ts_tlb_int_select
= gru_cpu_fault_map_id();
666 cch
->tlb_int_select
= gru_cpu_fault_map_id();
667 cch
->tfm_fault_bit_enable
=
668 (gts
->ts_user_options
== GRU_OPT_MISS_FMM_POLL
669 || gts
->ts_user_options
== GRU_OPT_MISS_FMM_INTR
);
675 unlock_cch_handle(cch
);
680 * Update CCH tlb interrupt select. Required when all the following is true:
681 * - task's GRU context is loaded into a GRU
682 * - task is using interrupt notification for TLB faults
683 * - task has migrated to a different cpu on the same blade where
684 * it was previously running.
686 static int gru_retarget_intr(struct gru_thread_state
*gts
)
688 if (gts
->ts_tlb_int_select
< 0
689 || gts
->ts_tlb_int_select
== gru_cpu_fault_map_id())
692 gru_dbg(grudev
, "retarget from %d to %d\n", gts
->ts_tlb_int_select
,
693 gru_cpu_fault_map_id());
694 return gru_update_cch(gts
);
698 * Unload the gru context if it is not assigned to the correct blade or
699 * chiplet. Misassignment can occur if the process migrates to a different
700 * blade or if the user changes the selected blade/chiplet.
701 * Return 0 if context correct placed, otherwise 1
703 void gru_check_context_placement(struct gru_thread_state
*gts
)
705 struct gru_state
*gru
;
706 int blade_id
, chiplet_id
;
709 * If the current task is the context owner, verify that the
710 * context is correctly placed. This test is skipped for non-owner
711 * references. Pthread apps use non-owner references to the CBRs.
714 if (!gru
|| gts
->ts_tgid_owner
!= current
->tgid
)
717 blade_id
= gts
->ts_user_blade_id
;
719 blade_id
= uv_numa_blade_id();
721 chiplet_id
= gts
->ts_user_chiplet_id
;
722 if (gru
->gs_blade_id
!= blade_id
||
723 (chiplet_id
>= 0 && chiplet_id
!= gru
->gs_chiplet_id
)) {
724 STAT(check_context_unload
);
725 gru_unload_context(gts
, 1);
726 } else if (gru_retarget_intr(gts
)) {
727 STAT(check_context_retarget_intr
);
733 * Insufficient GRU resources available on the local blade. Steal a context from
734 * a process. This is a hack until a _real_ resource scheduler is written....
736 #define next_ctxnum(n) ((n) < GRU_NUM_CCH - 2 ? (n) + 1 : 0)
737 #define next_gru(b, g) (((g) < &(b)->bs_grus[GRU_CHIPLETS_PER_BLADE - 1]) ? \
738 ((g)+1) : &(b)->bs_grus[0])
740 static int is_gts_stealable(struct gru_thread_state
*gts
,
741 struct gru_blade_state
*bs
)
743 if (is_kernel_context(gts
))
744 return down_write_trylock(&bs
->bs_kgts_sema
);
746 return mutex_trylock(>s
->ts_ctxlock
);
749 static void gts_stolen(struct gru_thread_state
*gts
,
750 struct gru_blade_state
*bs
)
752 if (is_kernel_context(gts
)) {
753 up_write(&bs
->bs_kgts_sema
);
754 STAT(steal_kernel_context
);
756 mutex_unlock(>s
->ts_ctxlock
);
757 STAT(steal_user_context
);
761 void gru_steal_context(struct gru_thread_state
*gts
)
763 struct gru_blade_state
*blade
;
764 struct gru_state
*gru
, *gru0
;
765 struct gru_thread_state
*ngts
= NULL
;
766 int ctxnum
, ctxnum0
, flag
= 0, cbr
, dsr
;
767 int blade_id
= gts
->ts_user_blade_id
;
768 int chiplet_id
= gts
->ts_user_chiplet_id
;
771 blade_id
= uv_numa_blade_id();
772 cbr
= gts
->ts_cbr_au_count
;
773 dsr
= gts
->ts_dsr_au_count
;
775 blade
= gru_base
[blade_id
];
776 spin_lock(&blade
->bs_lock
);
778 ctxnum
= next_ctxnum(blade
->bs_lru_ctxnum
);
779 gru
= blade
->bs_lru_gru
;
781 gru
= next_gru(blade
, gru
);
782 blade
->bs_lru_gru
= gru
;
783 blade
->bs_lru_ctxnum
= ctxnum
;
787 if (chiplet_id
< 0 || chiplet_id
== gru
->gs_chiplet_id
) {
788 if (check_gru_resources(gru
, cbr
, dsr
, GRU_NUM_CCH
))
790 spin_lock(&gru
->gs_lock
);
791 for (; ctxnum
< GRU_NUM_CCH
; ctxnum
++) {
792 if (flag
&& gru
== gru0
&& ctxnum
== ctxnum0
)
794 ngts
= gru
->gs_gts
[ctxnum
];
796 * We are grabbing locks out of order, so trylock is
797 * needed. GTSs are usually not locked, so the odds of
798 * success are high. If trylock fails, try to steal a
801 if (ngts
&& is_gts_stealable(ngts
, blade
))
805 spin_unlock(&gru
->gs_lock
);
806 if (ngts
|| (flag
&& gru
== gru0
&& ctxnum
== ctxnum0
))
809 if (flag
&& gru
== gru0
)
813 gru
= next_gru(blade
, gru
);
815 spin_unlock(&blade
->bs_lock
);
818 gts
->ustats
.context_stolen
++;
819 ngts
->ts_steal_jiffies
= jiffies
;
820 gru_unload_context(ngts
, is_kernel_context(ngts
) ? 0 : 1);
821 gts_stolen(ngts
, blade
);
823 STAT(steal_context_failed
);
826 "stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;"
827 " avail cb %ld, ds %ld\n",
828 gru
->gs_gid
, ctxnum
, ngts
, cbr
, dsr
, hweight64(gru
->gs_cbr_map
),
829 hweight64(gru
->gs_dsr_map
));
833 * Assign a gru context.
835 static int gru_assign_context_number(struct gru_state
*gru
)
839 ctxnum
= find_first_zero_bit(&gru
->gs_context_map
, GRU_NUM_CCH
);
840 __set_bit(ctxnum
, &gru
->gs_context_map
);
845 * Scan the GRUs on the local blade & assign a GRU context.
847 struct gru_state
*gru_assign_gru_context(struct gru_thread_state
*gts
)
849 struct gru_state
*gru
, *grux
;
850 int i
, max_active_contexts
;
851 int blade_id
= gts
->ts_user_blade_id
;
852 int chiplet_id
= gts
->ts_user_chiplet_id
;
855 blade_id
= uv_numa_blade_id();
858 max_active_contexts
= GRU_NUM_CCH
;
859 for_each_gru_on_blade(grux
, blade_id
, i
) {
860 if (chiplet_id
>= 0 && chiplet_id
!= grux
->gs_chiplet_id
)
862 if (check_gru_resources(grux
, gts
->ts_cbr_au_count
,
863 gts
->ts_dsr_au_count
,
864 max_active_contexts
)) {
866 max_active_contexts
= grux
->gs_active_contexts
;
867 if (max_active_contexts
== 0)
873 spin_lock(&gru
->gs_lock
);
874 if (!check_gru_resources(gru
, gts
->ts_cbr_au_count
,
875 gts
->ts_dsr_au_count
, GRU_NUM_CCH
)) {
876 spin_unlock(&gru
->gs_lock
);
879 reserve_gru_resources(gru
, gts
);
881 gts
->ts_blade
= gru
->gs_blade_id
;
882 gts
->ts_ctxnum
= gru_assign_context_number(gru
);
883 atomic_inc(>s
->ts_refcnt
);
884 gru
->gs_gts
[gts
->ts_ctxnum
] = gts
;
885 spin_unlock(&gru
->gs_lock
);
887 STAT(assign_context
);
889 "gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n",
890 gseg_virtual_address(gts
->ts_gru
, gts
->ts_ctxnum
), gts
,
891 gts
->ts_gru
->gs_gid
, gts
->ts_ctxnum
,
892 gts
->ts_cbr_au_count
, gts
->ts_dsr_au_count
);
894 gru_dbg(grudev
, "failed to allocate a GTS %s\n", "");
895 STAT(assign_context_failed
);
904 * Map the user's GRU segment
906 * Note: gru segments alway mmaped on GRU_GSEG_PAGESIZE boundaries.
908 int gru_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
910 struct gru_thread_state
*gts
;
911 unsigned long paddr
, vaddr
;
913 vaddr
= (unsigned long)vmf
->virtual_address
;
914 gru_dbg(grudev
, "vma %p, vaddr 0x%lx (0x%lx)\n",
915 vma
, vaddr
, GSEG_BASE(vaddr
));
918 /* The following check ensures vaddr is a valid address in the VMA */
919 gts
= gru_find_thread_state(vma
, TSID(vaddr
, vma
));
921 return VM_FAULT_SIGBUS
;
924 mutex_lock(>s
->ts_ctxlock
);
927 gru_check_context_placement(gts
);
930 STAT(load_user_context
);
931 if (!gru_assign_gru_context(gts
)) {
933 mutex_unlock(>s
->ts_ctxlock
);
934 set_current_state(TASK_INTERRUPTIBLE
);
935 schedule_timeout(GRU_ASSIGN_DELAY
); /* true hack ZZZ */
936 if (gts
->ts_steal_jiffies
+ GRU_STEAL_DELAY
< jiffies
)
937 gru_steal_context(gts
);
940 gru_load_context(gts
);
941 paddr
= gseg_physical_address(gts
->ts_gru
, gts
->ts_ctxnum
);
942 remap_pfn_range(vma
, vaddr
& ~(GRU_GSEG_PAGESIZE
- 1),
943 paddr
>> PAGE_SHIFT
, GRU_GSEG_PAGESIZE
,
948 mutex_unlock(>s
->ts_ctxlock
);
950 return VM_FAULT_NOPAGE
;