?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ifq: Switch to drop-head for default enqueue method.
[dragonfly.git] / sys / kern / usched_bsd4.c
bloba2db3574e0ef979653d52ab4b960209cac3020cc
1 /*
2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
3 * Copyright (c) 1999 Peter Wemm <peter@FreeBSD.org>. All rights reserved.
4 *
5 * This code is derived from software contributed to The DragonFly Project
6 * by Matthew Dillon <dillon@backplane.com>,
7 * by Mihai Carabas <mihai.carabas@gmail.com>
8 * and many others.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/lock.h>
36 #include <sys/queue.h>
37 #include <sys/proc.h>
38 #include <sys/rtprio.h>
39 #include <sys/uio.h>
40 #include <sys/sysctl.h>
41 #include <sys/resourcevar.h>
42 #include <sys/spinlock.h>
43 #include <sys/cpu_topology.h>
44 #include <sys/thread2.h>
45 #include <sys/spinlock2.h>
46 #include <sys/mplock2.h>
47
48 #include <sys/ktr.h>
49
50 #include <machine/cpu.h>
51 #include <machine/smp.h>
52
53 /*
54 * Priorities. Note that with 32 run queues per scheduler each queue
55 * represents four priority levels.
56 */
57
58 #define MAXPRI 128
59 #define PRIMASK (MAXPRI - 1)
60 #define PRIBASE_REALTIME 0
61 #define PRIBASE_NORMAL MAXPRI
62 #define PRIBASE_IDLE (MAXPRI * 2)
63 #define PRIBASE_THREAD (MAXPRI * 3)
64 #define PRIBASE_NULL (MAXPRI * 4)
65
66 #define NQS 32 /* 32 run queues. */
67 #define PPQ (MAXPRI / NQS) /* priorities per queue */
68 #define PPQMASK (PPQ - 1)
69
70 /*
71 * NICEPPQ - number of nice units per priority queue
72 *
73 * ESTCPUPPQ - number of estcpu units per priority queue
74 * ESTCPUMAX - number of estcpu units
75 */
76 #define NICEPPQ 2
77 #define ESTCPUPPQ 512
78 #define ESTCPUMAX (ESTCPUPPQ * NQS)
79 #define BATCHMAX (ESTCPUFREQ * 30)
80 #define PRIO_RANGE (PRIO_MAX - PRIO_MIN + 1)
81
82 #define ESTCPULIM(v) min((v), ESTCPUMAX)
83
84 TAILQ_HEAD(rq, lwp);
85
86 #define lwp_priority lwp_usdata.bsd4.priority
87 #define lwp_rqindex lwp_usdata.bsd4.rqindex
88 #define lwp_estcpu lwp_usdata.bsd4.estcpu
89 #define lwp_batch lwp_usdata.bsd4.batch
90 #define lwp_rqtype lwp_usdata.bsd4.rqtype
91
92 static void bsd4_acquire_curproc(struct lwp *lp);
93 static void bsd4_release_curproc(struct lwp *lp);
94 static void bsd4_select_curproc(globaldata_t gd);
95 static void bsd4_setrunqueue(struct lwp *lp);
96 static void bsd4_schedulerclock(struct lwp *lp, sysclock_t period,
97 sysclock_t cpstamp);
98 static void bsd4_recalculate_estcpu(struct lwp *lp);
99 static void bsd4_resetpriority(struct lwp *lp);
100 static void bsd4_forking(struct lwp *plp, struct lwp *lp);
101 static void bsd4_exiting(struct lwp *lp, struct proc *);
102 static void bsd4_uload_update(struct lwp *lp);
103 static void bsd4_yield(struct lwp *lp);
104 static void bsd4_need_user_resched_remote(void *dummy);
105 static int bsd4_batchy_looser_pri_test(struct lwp* lp);
106 static struct lwp *bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp);
107 static void bsd4_kick_helper(struct lwp *lp);
108 static struct lwp *bsd4_chooseproc_locked(struct lwp *chklp);
109 static void bsd4_remrunqueue_locked(struct lwp *lp);
110 static void bsd4_setrunqueue_locked(struct lwp *lp);
111 static void bsd4_changedcpu(struct lwp *lp);
112
113 struct usched usched_bsd4 = {
114 { NULL },
115 "bsd4", "Original DragonFly Scheduler",
116 NULL, /* default registration */
117 NULL, /* default deregistration */
118 bsd4_acquire_curproc,
119 bsd4_release_curproc,
120 bsd4_setrunqueue,
121 bsd4_schedulerclock,
122 bsd4_recalculate_estcpu,
123 bsd4_resetpriority,
124 bsd4_forking,
125 bsd4_exiting,
126 bsd4_uload_update,
127 NULL, /* setcpumask not supported */
128 bsd4_yield,
129 bsd4_changedcpu
130 };
131
132 struct usched_bsd4_pcpu {
133 struct thread *helper_thread;
134 short rrcount;
135 short upri;
136 struct lwp *uschedcp;
137 struct lwp *old_uschedcp;
138 cpu_node_t *cpunode;
139 };
140
141 typedef struct usched_bsd4_pcpu *bsd4_pcpu_t;
142
143 /*
144 * We have NQS (32) run queues per scheduling class. For the normal
145 * class, there are 128 priorities scaled onto these 32 queues. New
146 * processes are added to the last entry in each queue, and processes
147 * are selected for running by taking them from the head and maintaining
148 * a simple FIFO arrangement. Realtime and Idle priority processes have
149 * and explicit 0-31 priority which maps directly onto their class queue
150 * index. When a queue has something in it, the corresponding bit is
151 * set in the queuebits variable, allowing a single read to determine
152 * the state of all 32 queues and then a ffs() to find the first busy
153 * queue.
154 */
155 static struct rq bsd4_queues[NQS];
156 static struct rq bsd4_rtqueues[NQS];
157 static struct rq bsd4_idqueues[NQS];
158 static u_int32_t bsd4_queuebits;
159 static u_int32_t bsd4_rtqueuebits;
160 static u_int32_t bsd4_idqueuebits;
161 /* currently running a user process */
162 static cpumask_t bsd4_curprocmask = CPUMASK_INITIALIZER_ALLONES;
163 /* ready to accept a user process */
164 static cpumask_t bsd4_rdyprocmask;
165 static int bsd4_runqcount;
166 static volatile int bsd4_scancpu;
167 static struct spinlock bsd4_spin;
168 static struct usched_bsd4_pcpu bsd4_pcpu[MAXCPU];
169 static struct sysctl_ctx_list usched_bsd4_sysctl_ctx;
170 static struct sysctl_oid *usched_bsd4_sysctl_tree;
171
172 /* Debug info exposed through debug.* sysctl */
173
174 SYSCTL_INT(_debug, OID_AUTO, bsd4_runqcount, CTLFLAG_RD,
175 &bsd4_runqcount, 0,
176 "Number of run queues");
177
178 static int usched_bsd4_debug = -1;
179 SYSCTL_INT(_debug, OID_AUTO, bsd4_scdebug, CTLFLAG_RW,
180 &usched_bsd4_debug, 0,
181 "Print debug information for this pid");
182
183 static int usched_bsd4_pid_debug = -1;
184 SYSCTL_INT(_debug, OID_AUTO, bsd4_pid_debug, CTLFLAG_RW,
185 &usched_bsd4_pid_debug, 0,
186 "Print KTR debug information for this pid");
187
188 /* Tunning usched_bsd4 - configurable through kern.usched_bsd4.* */
189 static int usched_bsd4_smt = 0;
190 static int usched_bsd4_cache_coherent = 0;
191 static int usched_bsd4_upri_affinity = 16; /* 32 queues - half-way */
192 static int usched_bsd4_queue_checks = 5;
193 static int usched_bsd4_stick_to_level = 0;
194 static long usched_bsd4_kicks;
195 static int usched_bsd4_rrinterval = (ESTCPUFREQ + 9) / 10;
196 static int usched_bsd4_decay = 8;
197 static int usched_bsd4_batch_time = 10;
198
199 /* KTR debug printings */
200
201 KTR_INFO_MASTER_EXTERN(usched);
202
203 #if !defined(KTR_USCHED_BSD4)
204 #define KTR_USCHED_BSD4 KTR_ALL
205 #endif
206
207 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_acquire_curproc_urw, 0,
208 "USCHED_BSD4(bsd4_acquire_curproc in user_reseched_wanted "
209 "after release: pid %d, cpuid %d, curr_cpuid %d)",
210 pid_t pid, int cpuid, int curr);
211 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_acquire_curproc_before_loop, 0,
212 "USCHED_BSD4(bsd4_acquire_curproc before loop: pid %d, cpuid %d, "
213 "curr_cpuid %d)",
214 pid_t pid, int cpuid, int curr);
215 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_acquire_curproc_not, 0,
216 "USCHED_BSD4(bsd4_acquire_curproc couldn't acquire after "
217 "bsd4_setrunqueue: pid %d, cpuid %d, curr_lp pid %d, curr_cpuid %d)",
218 pid_t pid, int cpuid, pid_t curr_pid, int curr_cpuid);
219 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_acquire_curproc_switch, 0,
220 "USCHED_BSD4(bsd4_acquire_curproc after lwkt_switch: pid %d, "
221 "cpuid %d, curr_cpuid %d)",
222 pid_t pid, int cpuid, int curr);
223
224 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_release_curproc, 0,
225 "USCHED_BSD4(bsd4_release_curproc before select: pid %d, "
226 "cpuid %d, curr_cpuid %d)",
227 pid_t pid, int cpuid, int curr);
228
229 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_select_curproc, 0,
230 "USCHED_BSD4(bsd4_release_curproc before select: pid %d, "
231 "cpuid %d, old_pid %d, old_cpuid %d, curr_cpuid %d)",
232 pid_t pid, int cpuid, pid_t old_pid, int old_cpuid, int curr);
233
234 KTR_INFO(KTR_USCHED_BSD4, usched, batchy_test_false, 0,
235 "USCHED_BSD4(batchy_looser_pri_test false: pid %d, "
236 "cpuid %d, verify_mask %lu)",
237 pid_t pid, int cpuid, unsigned long mask);
238 KTR_INFO(KTR_USCHED_BSD4, usched, batchy_test_true, 0,
239 "USCHED_BSD4(batchy_looser_pri_test true: pid %d, "
240 "cpuid %d, verify_mask %lu)",
241 pid_t pid, int cpuid, unsigned long mask);
242
243 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_setrunqueue_fc_smt, 0,
244 "USCHED_BSD4(bsd4_setrunqueue free cpus smt: pid %d, cpuid %d, "
245 "mask %lu, curr_cpuid %d)",
246 pid_t pid, int cpuid, unsigned long mask, int curr);
247 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_setrunqueue_fc_non_smt, 0,
248 "USCHED_BSD4(bsd4_setrunqueue free cpus check non_smt: pid %d, "
249 "cpuid %d, mask %lu, curr_cpuid %d)",
250 pid_t pid, int cpuid, unsigned long mask, int curr);
251 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_setrunqueue_rc, 0,
252 "USCHED_BSD4(bsd4_setrunqueue running cpus check: pid %d, "
253 "cpuid %d, mask %lu, curr_cpuid %d)",
254 pid_t pid, int cpuid, unsigned long mask, int curr);
255 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_setrunqueue_found, 0,
256 "USCHED_BSD4(bsd4_setrunqueue found cpu: pid %d, cpuid %d, "
257 "mask %lu, found_cpuid %d, curr_cpuid %d)",
258 pid_t pid, int cpuid, unsigned long mask, int found_cpuid, int curr);
259 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_setrunqueue_not_found, 0,
260 "USCHED_BSD4(bsd4_setrunqueue not found cpu: pid %d, cpuid %d, "
261 "try_cpuid %d, curr_cpuid %d)",
262 pid_t pid, int cpuid, int try_cpuid, int curr);
263 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_setrunqueue_found_best_cpuid, 0,
264 "USCHED_BSD4(bsd4_setrunqueue found cpu: pid %d, cpuid %d, "
265 "mask %lu, found_cpuid %d, curr_cpuid %d)",
266 pid_t pid, int cpuid, unsigned long mask, int found_cpuid, int curr);
267
268 KTR_INFO(KTR_USCHED_BSD4, usched, bsd4_chooseproc, 0,
269 "USCHED_BSD4(chooseproc: pid %d, old_cpuid %d, curr_cpuid %d)",
270 pid_t pid, int old_cpuid, int curr);
271 KTR_INFO(KTR_USCHED_BSD4, usched, chooseproc_cc, 0,
272 "USCHED_BSD4(chooseproc_cc: pid %d, old_cpuid %d, curr_cpuid %d)",
273 pid_t pid, int old_cpuid, int curr);
274 KTR_INFO(KTR_USCHED_BSD4, usched, chooseproc_cc_not_good, 0,
275 "USCHED_BSD4(chooseproc_cc not good: pid %d, old_cpumask %lu, "
276 "sibling_mask %lu, curr_cpumask %lu)",
277 pid_t pid, unsigned long old_cpumask, unsigned long sibling_mask, unsigned long curr);
278 KTR_INFO(KTR_USCHED_BSD4, usched, chooseproc_cc_elected, 0,
279 "USCHED_BSD4(chooseproc_cc elected: pid %d, old_cpumask %lu, "
280 "sibling_mask %lu, curr_cpumask: %lu)",
281 pid_t pid, unsigned long old_cpumask, unsigned long sibling_mask, unsigned long curr);
282
283 KTR_INFO(KTR_USCHED_BSD4, usched, sched_thread_no_process, 0,
284 "USCHED_BSD4(sched_thread %d no process scheduled: pid %d, old_cpuid %d)",
285 int id, pid_t pid, int cpuid);
286 KTR_INFO(KTR_USCHED_BSD4, usched, sched_thread_process, 0,
287 "USCHED_BSD4(sched_thread %d process scheduled: pid %d, old_cpuid %d)",
288 int id, pid_t pid, int cpuid);
289 KTR_INFO(KTR_USCHED_BSD4, usched, sched_thread_no_process_found, 0,
290 "USCHED_BSD4(sched_thread %d no process found; tmpmask %lu)",
291 int id, unsigned long tmpmask);
292
293 /*
294 * Initialize the run queues at boot time.
295 */
296 static void
297 bsd4_rqinit(void *dummy)
298 {
299 int i;
300
301 spin_init(&bsd4_spin, "bsd4rq");
302 for (i = 0; i < NQS; i++) {
303 TAILQ_INIT(&bsd4_queues[i]);
304 TAILQ_INIT(&bsd4_rtqueues[i]);
305 TAILQ_INIT(&bsd4_idqueues[i]);
306 }
307 ATOMIC_CPUMASK_NANDBIT(bsd4_curprocmask, 0);
308 }
309 SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, bsd4_rqinit, NULL);
310
311 /*
312 * BSD4_ACQUIRE_CURPROC
313 *
314 * This function is called when the kernel intends to return to userland.
315 * It is responsible for making the thread the current designated userland
316 * thread for this cpu, blocking if necessary.
317 *
318 * The kernel will not depress our LWKT priority until after we return,
319 * in case we have to shove over to another cpu.
320 *
321 * We must determine our thread's disposition before we switch away. This
322 * is very sensitive code.
323 *
324 * WARNING! THIS FUNCTION IS ALLOWED TO CAUSE THE CURRENT THREAD TO MIGRATE
325 * TO ANOTHER CPU! Because most of the kernel assumes that no migration will
326 * occur, this function is called only under very controlled circumstances.
327 *
328 * MPSAFE
329 */
330 static void
331 bsd4_acquire_curproc(struct lwp *lp)
332 {
333 globaldata_t gd;
334 bsd4_pcpu_t dd;
335 thread_t td;
336 #if 0
337 struct lwp *olp;
338 #endif
339
340 /*
341 * Make sure we aren't sitting on a tsleep queue.
342 */
343 td = lp->lwp_thread;
344 crit_enter_quick(td);
345 if (td->td_flags & TDF_TSLEEPQ)
346 tsleep_remove(td);
347 bsd4_recalculate_estcpu(lp);
348
349 /*
350 * If a reschedule was requested give another thread the
351 * driver's seat.
352 */
353 if (user_resched_wanted()) {
354 clear_user_resched();
355 bsd4_release_curproc(lp);
356
357 KTR_COND_LOG(usched_bsd4_acquire_curproc_urw,
358 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
359 lp->lwp_proc->p_pid,
360 lp->lwp_thread->td_gd->gd_cpuid,
361 mycpu->gd_cpuid);
362 }
363
364 /*
365 * Loop until we are the current user thread
366 */
367 gd = mycpu;
368 dd = &bsd4_pcpu[gd->gd_cpuid];
369
370 KTR_COND_LOG(usched_bsd4_acquire_curproc_before_loop,
371 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
372 lp->lwp_proc->p_pid,
373 lp->lwp_thread->td_gd->gd_cpuid,
374 gd->gd_cpuid);
375
376 do {
377 /*
378 * Process any pending events and higher priority threads.
379 */
380 lwkt_yield();
381
382 /*
383 * Become the currently scheduled user thread for this cpu
384 * if we can do so trivially.
385 *
386 * We can steal another thread's current thread designation
387 * on this cpu since if we are running that other thread
388 * must not be, so we can safely deschedule it.
389 */
390 if (dd->uschedcp == lp) {
391 /*
392 * We are already the current lwp (hot path).
393 */
394 dd->upri = lp->lwp_priority;
395 } else if (dd->uschedcp == NULL) {
396 /*
397 * We can trivially become the current lwp.
398 */
399 ATOMIC_CPUMASK_ORBIT(bsd4_curprocmask, gd->gd_cpuid);
400 dd->uschedcp = lp;
401 dd->upri = lp->lwp_priority;
402 } else if (dd->upri > lp->lwp_priority) {
403 /*
404 * We can steal the current cpu's lwp designation
405 * away simply by replacing it. The other thread
406 * will stall when it tries to return to userland.
407 */
408 dd->uschedcp = lp;
409 dd->upri = lp->lwp_priority;
410 /*
411 lwkt_deschedule(olp->lwp_thread);
412 bsd4_setrunqueue(olp);
413 */
414 } else {
415 /*
416 * We cannot become the current lwp, place the lp
417 * on the bsd4 run-queue and deschedule ourselves.
418 *
419 * When we are reactivated we will have another
420 * chance.
421 */
422 lwkt_deschedule(lp->lwp_thread);
423
424 bsd4_setrunqueue(lp);
425
426 KTR_COND_LOG(usched_bsd4_acquire_curproc_not,
427 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
428 lp->lwp_proc->p_pid,
429 lp->lwp_thread->td_gd->gd_cpuid,
430 dd->uschedcp->lwp_proc->p_pid,
431 gd->gd_cpuid);
432
433
434 lwkt_switch();
435
436 /*
437 * Reload after a switch or setrunqueue/switch possibly
438 * moved us to another cpu.
439 */
440 gd = mycpu;
441 dd = &bsd4_pcpu[gd->gd_cpuid];
442
443 KTR_COND_LOG(usched_bsd4_acquire_curproc_switch,
444 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
445 lp->lwp_proc->p_pid,
446 lp->lwp_thread->td_gd->gd_cpuid,
447 gd->gd_cpuid);
448 }
449 } while (dd->uschedcp != lp);
450
451 crit_exit_quick(td);
452 KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
453 }
454
455 /*
456 * BSD4_RELEASE_CURPROC
457 *
458 * This routine detaches the current thread from the userland scheduler,
459 * usually because the thread needs to run or block in the kernel (at
460 * kernel priority) for a while.
461 *
462 * This routine is also responsible for selecting a new thread to
463 * make the current thread.
464 *
465 * NOTE: This implementation differs from the dummy example in that
466 * bsd4_select_curproc() is able to select the current process, whereas
467 * dummy_select_curproc() is not able to select the current process.
468 * This means we have to NULL out uschedcp.
469 *
470 * Additionally, note that we may already be on a run queue if releasing
471 * via the lwkt_switch() in bsd4_setrunqueue().
472 *
473 * MPSAFE
474 */
475
476 static void
477 bsd4_release_curproc(struct lwp *lp)
478 {
479 globaldata_t gd = mycpu;
480 bsd4_pcpu_t dd = &bsd4_pcpu[gd->gd_cpuid];
481
482 if (dd->uschedcp == lp) {
483 crit_enter();
484 KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
485
486 KTR_COND_LOG(usched_bsd4_release_curproc,
487 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
488 lp->lwp_proc->p_pid,
489 lp->lwp_thread->td_gd->gd_cpuid,
490 gd->gd_cpuid);
491
492 dd->uschedcp = NULL; /* don't let lp be selected */
493 dd->upri = PRIBASE_NULL;
494 ATOMIC_CPUMASK_NANDBIT(bsd4_curprocmask, gd->gd_cpuid);
495 dd->old_uschedcp = lp; /* used only for KTR debug prints */
496 bsd4_select_curproc(gd);
497 crit_exit();
498 }
499 }
500
501 /*
502 * BSD4_SELECT_CURPROC
503 *
504 * Select a new current process for this cpu and clear any pending user
505 * reschedule request. The cpu currently has no current process.
506 *
507 * This routine is also responsible for equal-priority round-robining,
508 * typically triggered from bsd4_schedulerclock(). In our dummy example
509 * all the 'user' threads are LWKT scheduled all at once and we just
510 * call lwkt_switch().
511 *
512 * The calling process is not on the queue and cannot be selected.
513 *
514 * MPSAFE
515 */
516 static
517 void
518 bsd4_select_curproc(globaldata_t gd)
519 {
520 bsd4_pcpu_t dd = &bsd4_pcpu[gd->gd_cpuid];
521 struct lwp *nlp;
522 int cpuid = gd->gd_cpuid;
523
524 crit_enter_gd(gd);
525
526 spin_lock(&bsd4_spin);
527 if(usched_bsd4_cache_coherent)
528 nlp = bsd4_chooseproc_locked_cache_coherent(dd->uschedcp);
529 else
530 nlp = bsd4_chooseproc_locked(dd->uschedcp);
531
532 if (nlp) {
533
534 KTR_COND_LOG(usched_bsd4_select_curproc,
535 nlp->lwp_proc->p_pid == usched_bsd4_pid_debug,
536 nlp->lwp_proc->p_pid,
537 nlp->lwp_thread->td_gd->gd_cpuid,
538 dd->old_uschedcp->lwp_proc->p_pid,
539 dd->old_uschedcp->lwp_thread->td_gd->gd_cpuid,
540 gd->gd_cpuid);
541
542 ATOMIC_CPUMASK_ORBIT(bsd4_curprocmask, cpuid);
543 dd->upri = nlp->lwp_priority;
544 dd->uschedcp = nlp;
545 dd->rrcount = 0; /* reset round robin */
546 spin_unlock(&bsd4_spin);
547 lwkt_acquire(nlp->lwp_thread);
548 lwkt_schedule(nlp->lwp_thread);
549 } else {
550 spin_unlock(&bsd4_spin);
551 }
552
553 #if 0
554 } else if (bsd4_runqcount && CPUMASK_TESTBIT(bsd4_rdyprocmask, cpuid)) {
555 ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, cpuid);
556 spin_unlock(&bsd4_spin);
557 lwkt_schedule(dd->helper_thread);
558 } else {
559 spin_unlock(&bsd4_spin);
560 }
561 #endif
562 crit_exit_gd(gd);
563 }
564
565 /*
566 * batchy_looser_pri_test() - determine if a process is batchy or not
567 * relative to the other processes running in the system
568 */
569 static int
570 bsd4_batchy_looser_pri_test(struct lwp* lp)
571 {
572 cpumask_t mask;
573 bsd4_pcpu_t other_dd;
574 int cpu;
575
576 /* Current running processes */
577 mask = bsd4_curprocmask;
578 CPUMASK_ANDMASK(mask, smp_active_mask);
579 CPUMASK_ANDMASK(mask, usched_global_cpumask);
580
581 while (CPUMASK_TESTNZERO(mask)) {
582 cpu = BSFCPUMASK(mask);
583 other_dd = &bsd4_pcpu[cpu];
584 if (other_dd->upri - lp->lwp_priority > usched_bsd4_upri_affinity * PPQ) {
585
586 KTR_COND_LOG(usched_batchy_test_false,
587 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
588 lp->lwp_proc->p_pid,
589 lp->lwp_thread->td_gd->gd_cpuid,
590 (unsigned long)CPUMASK_LOWMASK(mask));
591
592 return 0;
593 }
594 CPUMASK_NANDBIT(mask, cpu);
595 }
596
597 KTR_COND_LOG(usched_batchy_test_true,
598 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
599 lp->lwp_proc->p_pid,
600 lp->lwp_thread->td_gd->gd_cpuid,
601 (unsigned long)CPUMASK_LOWMASK(mask));
602
603 return 1;
604 }
605
606 /*
607 *
608 * BSD4_SETRUNQUEUE
609 *
610 * Place the specified lwp on the user scheduler's run queue. This routine
611 * must be called with the thread descheduled. The lwp must be runnable.
612 *
613 * The thread may be the current thread as a special case.
614 *
615 * MPSAFE
616 */
617 static void
618 bsd4_setrunqueue(struct lwp *lp)
619 {
620 globaldata_t gd;
621 bsd4_pcpu_t dd;
622 int cpuid;
623 cpumask_t mask;
624 cpumask_t tmpmask;
625
626 /*
627 * First validate the process state relative to the current cpu.
628 * We don't need the spinlock for this, just a critical section.
629 * We are in control of the process.
630 */
631 crit_enter();
632 KASSERT(lp->lwp_stat == LSRUN, ("setrunqueue: lwp not LSRUN"));
633 KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0,
634 ("lwp %d/%d already on runq! flag %08x/%08x", lp->lwp_proc->p_pid,
635 lp->lwp_tid, lp->lwp_proc->p_flags, lp->lwp_flags));
636 KKASSERT((lp->lwp_thread->td_flags & TDF_RUNQ) == 0);
637
638 /*
639 * Note: gd and dd are relative to the target thread's last cpu,
640 * NOT our current cpu.
641 */
642 gd = lp->lwp_thread->td_gd;
643 dd = &bsd4_pcpu[gd->gd_cpuid];
644
645 /*
646 * This process is not supposed to be scheduled anywhere or assigned
647 * as the current process anywhere. Assert the condition.
648 */
649 KKASSERT(dd->uschedcp != lp);
650
651 /*
652 * XXX fixme. Could be part of a remrunqueue/setrunqueue
653 * operation when the priority is recalculated, so TDF_MIGRATING
654 * may already be set.
655 */
656 if ((lp->lwp_thread->td_flags & TDF_MIGRATING) == 0)
657 lwkt_giveaway(lp->lwp_thread);
658
659 /*
660 * We lose control of lp the moment we release the spinlock after
661 * having placed lp on the queue. i.e. another cpu could pick it
662 * up and it could exit, or its priority could be further adjusted,
663 * or something like that.
664 */
665 spin_lock(&bsd4_spin);
666 bsd4_setrunqueue_locked(lp);
667 lp->lwp_rebal_ticks = sched_ticks;
668
669 /*
670 * Kick the scheduler helper on one of the other cpu's
671 * and request a reschedule if appropriate.
672 *
673 * NOTE: We check all cpus whos rdyprocmask is set. First we
674 * look for cpus without designated lps, then we look for
675 * cpus with designated lps with a worse priority than our
676 * process.
677 */
678 ++bsd4_scancpu;
679
680 if (usched_bsd4_smt) {
681
682 /*
683 * SMT heuristic - Try to schedule on a free physical core.
684 * If no physical core found than choose the one that has
685 * an interactive thread.
686 */
687
688 int best_cpuid = -1;
689 int min_prio = MAXPRI * MAXPRI;
690 int sibling;
691
692 cpuid = (bsd4_scancpu & 0xFFFF) % ncpus;
693 mask = bsd4_rdyprocmask;
694 CPUMASK_NANDMASK(mask, bsd4_curprocmask);
695 CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
696 CPUMASK_ANDMASK(mask, smp_active_mask);
697 CPUMASK_ANDMASK(mask, usched_global_cpumask);
698
699 KTR_COND_LOG(usched_bsd4_setrunqueue_fc_smt,
700 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
701 lp->lwp_proc->p_pid,
702 lp->lwp_thread->td_gd->gd_cpuid,
703 (unsigned long)CPUMASK_LOWMASK(mask),
704 mycpu->gd_cpuid);
705
706 while (CPUMASK_TESTNZERO(mask)) {
707 CPUMASK_ASSNBMASK(tmpmask, cpuid);
708 if (CPUMASK_TESTMASK(tmpmask, mask)) {
709 CPUMASK_ANDMASK(tmpmask, mask);
710 cpuid = BSFCPUMASK(tmpmask);
711 } else {
712 cpuid = BSFCPUMASK(mask);
713 }
714 gd = globaldata_find(cpuid);
715 dd = &bsd4_pcpu[cpuid];
716
717 if ((dd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK)) {
718 tmpmask = dd->cpunode->parent_node->members;
719 CPUMASK_NANDMASK(tmpmask, dd->cpunode->members);
720 CPUMASK_ANDMASK(tmpmask, mask);
721 if (CPUMASK_TESTNZERO(tmpmask)) {
722 KTR_COND_LOG(usched_bsd4_setrunqueue_found,
723 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
724 lp->lwp_proc->p_pid,
725 lp->lwp_thread->td_gd->gd_cpuid,
726 (unsigned long)CPUMASK_LOWMASK(mask),
727 cpuid,
728 mycpu->gd_cpuid);
729
730 goto found;
731 } else {
732 tmpmask =
733 dd->cpunode->parent_node->members;
734 CPUMASK_NANDMASK(tmpmask,
735 dd->cpunode->members);
736 sibling = BSFCPUMASK(tmpmask);
737 if (min_prio >
738 bsd4_pcpu[sibling].upri) {
739 min_prio =
740 bsd4_pcpu[sibling].upri;
741 best_cpuid = cpuid;
742 }
743 }
744 }
745 CPUMASK_NANDBIT(mask, cpuid);
746 }
747
748 if (best_cpuid != -1) {
749 cpuid = best_cpuid;
750 gd = globaldata_find(cpuid);
751 dd = &bsd4_pcpu[cpuid];
752
753 KTR_COND_LOG(usched_bsd4_setrunqueue_found_best_cpuid,
754 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
755 lp->lwp_proc->p_pid,
756 lp->lwp_thread->td_gd->gd_cpuid,
757 (unsigned long)CPUMASK_LOWMASK(mask),
758 cpuid,
759 mycpu->gd_cpuid);
760
761 goto found;
762 }
763 } else {
764 /* Fallback to the original heuristic */
765 cpuid = (bsd4_scancpu & 0xFFFF) % ncpus;
766 mask = bsd4_rdyprocmask;
767 CPUMASK_NANDMASK(mask, bsd4_curprocmask);
768 CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
769 CPUMASK_ANDMASK(mask, smp_active_mask);
770 CPUMASK_ANDMASK(mask, usched_global_cpumask);
771
772 KTR_COND_LOG(usched_bsd4_setrunqueue_fc_non_smt,
773 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
774 lp->lwp_proc->p_pid,
775 lp->lwp_thread->td_gd->gd_cpuid,
776 (unsigned long)CPUMASK_LOWMASK(mask),
777 mycpu->gd_cpuid);
778
779 while (CPUMASK_TESTNZERO(mask)) {
780 CPUMASK_ASSNBMASK(tmpmask, cpuid);
781 if (CPUMASK_TESTMASK(tmpmask, mask)) {
782 CPUMASK_ANDMASK(tmpmask, mask);
783 cpuid = BSFCPUMASK(tmpmask);
784 } else {
785 cpuid = BSFCPUMASK(mask);
786 }
787 gd = globaldata_find(cpuid);
788 dd = &bsd4_pcpu[cpuid];
789
790 if ((dd->upri & ~PPQMASK) >=
791 (lp->lwp_priority & ~PPQMASK)) {
792 KTR_COND_LOG(usched_bsd4_setrunqueue_found,
793 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
794 lp->lwp_proc->p_pid,
795 lp->lwp_thread->td_gd->gd_cpuid,
796 (unsigned long)CPUMASK_LOWMASK(mask),
797 cpuid,
798 mycpu->gd_cpuid);
799
800 goto found;
801 }
802 CPUMASK_NANDBIT(mask, cpuid);
803 }
804 }
805
806 /*
807 * Then cpus which might have a currently running lp
808 */
809 mask = bsd4_curprocmask;
810 CPUMASK_ANDMASK(mask, bsd4_rdyprocmask);
811 CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
812 CPUMASK_ANDMASK(mask, smp_active_mask);
813 CPUMASK_ANDMASK(mask, usched_global_cpumask);
814
815 KTR_COND_LOG(usched_bsd4_setrunqueue_rc,
816 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
817 lp->lwp_proc->p_pid,
818 lp->lwp_thread->td_gd->gd_cpuid,
819 (unsigned long)CPUMASK_LOWMASK(mask),
820 mycpu->gd_cpuid);
821
822 while (CPUMASK_TESTNZERO(mask)) {
823 CPUMASK_ASSNBMASK(tmpmask, cpuid);
824 if (CPUMASK_TESTMASK(tmpmask, mask)) {
825 CPUMASK_ANDMASK(tmpmask, mask);
826 cpuid = BSFCPUMASK(tmpmask);
827 } else {
828 cpuid = BSFCPUMASK(mask);
829 }
830 gd = globaldata_find(cpuid);
831 dd = &bsd4_pcpu[cpuid];
832
833 if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
834 KTR_COND_LOG(usched_bsd4_setrunqueue_found,
835 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
836 lp->lwp_proc->p_pid,
837 lp->lwp_thread->td_gd->gd_cpuid,
838 (unsigned long)CPUMASK_LOWMASK(mask),
839 cpuid,
840 mycpu->gd_cpuid);
841
842 goto found;
843 }
844 CPUMASK_NANDBIT(mask, cpuid);
845 }
846
847 /*
848 * If we cannot find a suitable cpu we reload from bsd4_scancpu
849 * and round-robin. Other cpus will pickup as they release their
850 * current lwps or become ready.
851 *
852 * Avoid a degenerate system lockup case if usched_global_cpumask
853 * is set to 0 or otherwise does not cover lwp_cpumask.
854 *
855 * We only kick the target helper thread in this case, we do not
856 * set the user resched flag because
857 */
858 cpuid = (bsd4_scancpu & 0xFFFF) % ncpus;
859 if (CPUMASK_TESTBIT(usched_global_cpumask, cpuid) == 0)
860 cpuid = 0;
861 gd = globaldata_find(cpuid);
862 dd = &bsd4_pcpu[cpuid];
863
864 KTR_COND_LOG(usched_bsd4_setrunqueue_not_found,
865 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
866 lp->lwp_proc->p_pid,
867 lp->lwp_thread->td_gd->gd_cpuid,
868 cpuid,
869 mycpu->gd_cpuid);
870
871 found:
872 if (gd == mycpu) {
873 spin_unlock(&bsd4_spin);
874 if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
875 if (dd->uschedcp == NULL) {
876 wakeup_mycpu(dd->helper_thread);
877 } else {
878 need_user_resched();
879 }
880 }
881 } else {
882 ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, cpuid);
883 spin_unlock(&bsd4_spin);
884 if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK))
885 lwkt_send_ipiq(gd, bsd4_need_user_resched_remote, NULL);
886 else
887 wakeup(dd->helper_thread);
888 }
889 crit_exit();
890 }
891
892 /*
893 * This routine is called from a systimer IPI. It MUST be MP-safe and
894 * the BGL IS NOT HELD ON ENTRY. This routine is called at ESTCPUFREQ on
895 * each cpu.
896 *
897 * This routine is called on every sched tick. If the currently running
898 * thread belongs to this scheduler it will be called with a non-NULL lp,
899 * otherwise it will be called with a NULL lp.
900 *
901 * MPSAFE
902 */
903 static
904 void
905 bsd4_schedulerclock(struct lwp *lp, sysclock_t period, sysclock_t cpstamp)
906 {
907 globaldata_t gd = mycpu;
908 bsd4_pcpu_t dd = &bsd4_pcpu[gd->gd_cpuid];
909
910 /*
911 * No impl if no lp running.
912 */
913 if (lp == NULL)
914 return;
915
916 /*
917 * Do we need to round-robin? We round-robin 10 times a second.
918 * This should only occur for cpu-bound batch processes.
919 */
920 if (++dd->rrcount >= usched_bsd4_rrinterval) {
921 dd->rrcount = 0;
922 need_user_resched();
923 }
924
925 /*
926 * Adjust estcpu upward using a real time equivalent calculation.
927 */
928 lp->lwp_estcpu = ESTCPULIM(lp->lwp_estcpu + ESTCPUMAX / ESTCPUFREQ + 1);
929
930 /*
931 * Spinlocks also hold a critical section so there should not be
932 * any active.
933 */
934 KKASSERT(gd->gd_spinlocks == 0);
935
936 bsd4_resetpriority(lp);
937 }
938
939 /*
940 * Called from acquire and from kern_synch's one-second timer (one of the
941 * callout helper threads) with a critical section held.
942 *
943 * Decay p_estcpu based on the number of ticks we haven't been running
944 * and our p_nice. As the load increases each process observes a larger
945 * number of idle ticks (because other processes are running in them).
946 * This observation leads to a larger correction which tends to make the
947 * system more 'batchy'.
948 *
949 * Note that no recalculation occurs for a process which sleeps and wakes
950 * up in the same tick. That is, a system doing thousands of context
951 * switches per second will still only do serious estcpu calculations
952 * ESTCPUFREQ times per second.
953 *
954 * MPSAFE
955 */
956 static
957 void
958 bsd4_recalculate_estcpu(struct lwp *lp)
959 {
960 globaldata_t gd = mycpu;
961 sysclock_t cpbase;
962 sysclock_t ttlticks;
963 int estcpu;
964 int decay_factor;
965
966 /*
967 * We have to subtract periodic to get the last schedclock
968 * timeout time, otherwise we would get the upcoming timeout.
969 * Keep in mind that a process can migrate between cpus and
970 * while the scheduler clock should be very close, boundary
971 * conditions could lead to a small negative delta.
972 */
973 cpbase = gd->gd_schedclock.time - gd->gd_schedclock.periodic;
974
975 if (lp->lwp_slptime > 1) {
976 /*
977 * Too much time has passed, do a coarse correction.
978 */
979 lp->lwp_estcpu = lp->lwp_estcpu >> 1;
980 bsd4_resetpriority(lp);
981 lp->lwp_cpbase = cpbase;
982 lp->lwp_cpticks = 0;
983 lp->lwp_batch -= ESTCPUFREQ;
984 if (lp->lwp_batch < 0)
985 lp->lwp_batch = 0;
986 } else if (lp->lwp_cpbase != cpbase) {
987 /*
988 * Adjust estcpu if we are in a different tick. Don't waste
989 * time if we are in the same tick.
990 *
991 * First calculate the number of ticks in the measurement
992 * interval. The ttlticks calculation can wind up 0 due to
993 * a bug in the handling of lwp_slptime (as yet not found),
994 * so make sure we do not get a divide by 0 panic.
995 */
996 ttlticks = (cpbase - lp->lwp_cpbase) /
997 gd->gd_schedclock.periodic;
998 if ((ssysclock_t)ttlticks < 0) {
999 ttlticks = 0;
1000 lp->lwp_cpbase = cpbase;
1001 }
1002 if (ttlticks == 0)
1003 return;
1004 updatepcpu(lp, lp->lwp_cpticks, ttlticks);
1005
1006 /*
1007 * Calculate the percentage of one cpu used factoring in ncpus
1008 * and the load and adjust estcpu. Handle degenerate cases
1009 * by adding 1 to bsd4_runqcount.
1010 *
1011 * estcpu is scaled by ESTCPUMAX.
1012 *
1013 * bsd4_runqcount is the excess number of user processes
1014 * that cannot be immediately scheduled to cpus. We want
1015 * to count these as running to avoid range compression
1016 * in the base calculation (which is the actual percentage
1017 * of one cpu used).
1018 */
1019 estcpu = (lp->lwp_cpticks * ESTCPUMAX) *
1020 (bsd4_runqcount + ncpus) / (ncpus * ttlticks);
1021
1022 /*
1023 * If estcpu is > 50% we become more batch-like
1024 * If estcpu is <= 50% we become less batch-like
1025 *
1026 * It takes 30 cpu seconds to traverse the entire range.
1027 */
1028 if (estcpu > ESTCPUMAX / 2) {
1029 lp->lwp_batch += ttlticks;
1030 if (lp->lwp_batch > BATCHMAX)
1031 lp->lwp_batch = BATCHMAX;
1032 } else {
1033 lp->lwp_batch -= ttlticks;
1034 if (lp->lwp_batch < 0)
1035 lp->lwp_batch = 0;
1036 }
1037
1038 if (usched_bsd4_debug == lp->lwp_proc->p_pid) {
1039 kprintf("pid %d lwp %p estcpu %3d %3d bat %d cp %d/%d",
1040 lp->lwp_proc->p_pid, lp,
1041 estcpu, lp->lwp_estcpu,
1042 lp->lwp_batch,
1043 lp->lwp_cpticks, ttlticks);
1044 }
1045
1046 /*
1047 * Adjust lp->lwp_esetcpu. The decay factor determines how
1048 * quickly lwp_estcpu collapses to its realtime calculation.
1049 * A slower collapse gives us a more accurate number but
1050 * can cause a cpu hog to eat too much cpu before the
1051 * scheduler decides to downgrade it.
1052 *
1053 * NOTE: p_nice is accounted for in bsd4_resetpriority(),
1054 * and not here, but we must still ensure that a
1055 * cpu-bound nice -20 process does not completely
1056 * override a cpu-bound nice +20 process.
1057 *
1058 * NOTE: We must use ESTCPULIM() here to deal with any
1059 * overshoot.
1060 */
1061 decay_factor = usched_bsd4_decay;
1062 if (decay_factor < 1)
1063 decay_factor = 1;
1064 if (decay_factor > 1024)
1065 decay_factor = 1024;
1066
1067 lp->lwp_estcpu = ESTCPULIM(
1068 (lp->lwp_estcpu * decay_factor + estcpu) /
1069 (decay_factor + 1));
1070
1071 if (usched_bsd4_debug == lp->lwp_proc->p_pid)
1072 kprintf(" finalestcpu %d\n", lp->lwp_estcpu);
1073 bsd4_resetpriority(lp);
1074 lp->lwp_cpbase += ttlticks * gd->gd_schedclock.periodic;
1075 lp->lwp_cpticks = 0;
1076 }
1077 }
1078
1079 /*
1080 * Compute the priority of a process when running in user mode.
1081 * Arrange to reschedule if the resulting priority is better
1082 * than that of the current process.
1083 *
1084 * This routine may be called with any process.
1085 *
1086 * This routine is called by fork1() for initial setup with the process
1087 * of the run queue, and also may be called normally with the process on or
1088 * off the run queue.
1089 *
1090 * MPSAFE
1091 */
1092 static void
1093 bsd4_resetpriority(struct lwp *lp)
1094 {
1095 bsd4_pcpu_t dd;
1096 int newpriority;
1097 u_short newrqtype;
1098 int reschedcpu;
1099 int checkpri;
1100 int estcpu;
1101
1102 /*
1103 * Calculate the new priority and queue type
1104 */
1105 crit_enter();
1106 spin_lock(&bsd4_spin);
1107
1108 newrqtype = lp->lwp_rtprio.type;
1109
1110 switch(newrqtype) {
1111 case RTP_PRIO_REALTIME:
1112 case RTP_PRIO_FIFO:
1113 newpriority = PRIBASE_REALTIME +
1114 (lp->lwp_rtprio.prio & PRIMASK);
1115 break;
1116 case RTP_PRIO_NORMAL:
1117 /*
1118 * Detune estcpu based on batchiness. lwp_batch ranges
1119 * from 0 to BATCHMAX. Limit estcpu for the sake of
1120 * the priority calculation to between 50% and 100%.
1121 */
1122 estcpu = lp->lwp_estcpu * (lp->lwp_batch + BATCHMAX) /
1123 (BATCHMAX * 2);
1124
1125 /*
1126 * p_nice piece Adds (0-40) * 2 0-80
1127 * estcpu Adds 16384 * 4 / 512 0-128
1128 */
1129 newpriority = (lp->lwp_proc->p_nice - PRIO_MIN) * PPQ / NICEPPQ;
1130 newpriority += estcpu * PPQ / ESTCPUPPQ;
1131 newpriority = newpriority * MAXPRI / (PRIO_RANGE * PPQ /
1132 NICEPPQ + ESTCPUMAX * PPQ / ESTCPUPPQ);
1133 newpriority = PRIBASE_NORMAL + (newpriority & PRIMASK);
1134 break;
1135 case RTP_PRIO_IDLE:
1136 newpriority = PRIBASE_IDLE + (lp->lwp_rtprio.prio & PRIMASK);
1137 break;
1138 case RTP_PRIO_THREAD:
1139 newpriority = PRIBASE_THREAD + (lp->lwp_rtprio.prio & PRIMASK);
1140 break;
1141 default:
1142 panic("Bad RTP_PRIO %d", newrqtype);
1143 /* NOT REACHED */
1144 }
1145
1146 /*
1147 * The newpriority incorporates the queue type so do a simple masked
1148 * check to determine if the process has moved to another queue. If
1149 * it has, and it is currently on a run queue, then move it.
1150 *
1151 * td_upri has normal sense (higher values are more desireable), so
1152 * negate it.
1153 */
1154 lp->lwp_thread->td_upri = -(newpriority & ~PPQMASK);
1155 if ((lp->lwp_priority ^ newpriority) & ~PPQMASK) {
1156 lp->lwp_priority = newpriority;
1157 if (lp->lwp_mpflags & LWP_MP_ONRUNQ) {
1158 bsd4_remrunqueue_locked(lp);
1159 lp->lwp_rqtype = newrqtype;
1160 lp->lwp_rqindex = (newpriority & PRIMASK) / PPQ;
1161 bsd4_setrunqueue_locked(lp);
1162 checkpri = 1;
1163 } else {
1164 lp->lwp_rqtype = newrqtype;
1165 lp->lwp_rqindex = (newpriority & PRIMASK) / PPQ;
1166 checkpri = 0;
1167 }
1168 reschedcpu = lp->lwp_thread->td_gd->gd_cpuid;
1169 } else {
1170 lp->lwp_priority = newpriority;
1171 reschedcpu = -1;
1172 checkpri = 1;
1173 }
1174
1175 /*
1176 * Determine if we need to reschedule the target cpu. This only
1177 * occurs if the LWP is already on a scheduler queue, which means
1178 * that idle cpu notification has already occured. At most we
1179 * need only issue a need_user_resched() on the appropriate cpu.
1180 *
1181 * The LWP may be owned by a CPU different from the current one,
1182 * in which case dd->uschedcp may be modified without an MP lock
1183 * or a spinlock held. The worst that happens is that the code
1184 * below causes a spurious need_user_resched() on the target CPU
1185 * and dd->pri to be wrong for a short period of time, both of
1186 * which are harmless.
1187 *
1188 * If checkpri is 0 we are adjusting the priority of the current
1189 * process, possibly higher (less desireable), so ignore the upri
1190 * check which will fail in that case.
1191 */
1192 if (reschedcpu >= 0) {
1193 dd = &bsd4_pcpu[reschedcpu];
1194 if (CPUMASK_TESTBIT(bsd4_rdyprocmask, reschedcpu) &&
1195 (checkpri == 0 ||
1196 (dd->upri & ~PRIMASK) > (lp->lwp_priority & ~PRIMASK))) {
1197 if (reschedcpu == mycpu->gd_cpuid) {
1198 spin_unlock(&bsd4_spin);
1199 need_user_resched();
1200 } else {
1201 spin_unlock(&bsd4_spin);
1202 ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask,
1203 reschedcpu);
1204 lwkt_send_ipiq(lp->lwp_thread->td_gd,
1205 bsd4_need_user_resched_remote,
1206 NULL);
1207 }
1208 } else {
1209 spin_unlock(&bsd4_spin);
1210 }
1211 } else {
1212 spin_unlock(&bsd4_spin);
1213 }
1214 crit_exit();
1215 }
1216
1217 /*
1218 * MPSAFE
1219 */
1220 static
1221 void
1222 bsd4_yield(struct lwp *lp)
1223 {
1224 #if 0
1225 /* FUTURE (or something similar) */
1226 switch(lp->lwp_rqtype) {
1227 case RTP_PRIO_NORMAL:
1228 lp->lwp_estcpu = ESTCPULIM(lp->lwp_estcpu + ESTCPUINCR);
1229 break;
1230 default:
1231 break;
1232 }
1233 #endif
1234 need_user_resched();
1235 }
1236
1237 static
1238 void
1239 bsd4_changedcpu(struct lwp *lp __unused)
1240 {
1241 }
1242
1243 /*
1244 * Called from fork1() when a new child process is being created.
1245 *
1246 * Give the child process an initial estcpu that is more batch then
1247 * its parent and dock the parent for the fork (but do not
1248 * reschedule the parent). This comprises the main part of our batch
1249 * detection heuristic for both parallel forking and sequential execs.
1250 *
1251 * XXX lwp should be "spawning" instead of "forking"
1252 *
1253 * MPSAFE
1254 */
1255 static void
1256 bsd4_forking(struct lwp *plp, struct lwp *lp)
1257 {
1258 /*
1259 * Put the child 4 queue slots (out of 32) higher than the parent
1260 * (less desireable than the parent).
1261 */
1262 lp->lwp_estcpu = ESTCPULIM(plp->lwp_estcpu + ESTCPUPPQ * 4);
1263
1264 /*
1265 * The batch status of children always starts out centerline
1266 * and will inch-up or inch-down as appropriate. It takes roughly
1267 * ~15 seconds of >50% cpu to hit the limit.
1268 */
1269 lp->lwp_batch = BATCHMAX / 2;
1270
1271 /*
1272 * Dock the parent a cost for the fork, protecting us from fork
1273 * bombs. If the parent is forking quickly make the child more
1274 * batchy.
1275 */
1276 plp->lwp_estcpu = ESTCPULIM(plp->lwp_estcpu + ESTCPUPPQ / 16);
1277 }
1278
1279 /*
1280 * Called when a lwp is being removed from this scheduler, typically
1281 * during lwp_exit().
1282 */
1283 static void
1284 bsd4_exiting(struct lwp *lp, struct proc *child_proc)
1285 {
1286 }
1287
1288 static void
1289 bsd4_uload_update(struct lwp *lp)
1290 {
1291 }
1292
1293 /*
1294 * chooseproc() is called when a cpu needs a user process to LWKT schedule,
1295 * it selects a user process and returns it. If chklp is non-NULL and chklp
1296 * has a better or equal priority then the process that would otherwise be
1297 * chosen, NULL is returned.
1298 *
1299 * Until we fix the RUNQ code the chklp test has to be strict or we may
1300 * bounce between processes trying to acquire the current process designation.
1301 *
1302 * MPSAFE - must be called with bsd4_spin exclusive held. The spinlock is
1303 * left intact through the entire routine.
1304 */
1305 static
1306 struct lwp *
1307 bsd4_chooseproc_locked(struct lwp *chklp)
1308 {
1309 struct lwp *lp;
1310 struct rq *q;
1311 u_int32_t *which, *which2;
1312 u_int32_t pri;
1313 u_int32_t rtqbits;
1314 u_int32_t tsqbits;
1315 u_int32_t idqbits;
1316 cpumask_t cpumask;
1317
1318 rtqbits = bsd4_rtqueuebits;
1319 tsqbits = bsd4_queuebits;
1320 idqbits = bsd4_idqueuebits;
1321 cpumask = mycpu->gd_cpumask;
1322
1323
1324 again:
1325 if (rtqbits) {
1326 pri = bsfl(rtqbits);
1327 q = &bsd4_rtqueues[pri];
1328 which = &bsd4_rtqueuebits;
1329 which2 = &rtqbits;
1330 } else if (tsqbits) {
1331 pri = bsfl(tsqbits);
1332 q = &bsd4_queues[pri];
1333 which = &bsd4_queuebits;
1334 which2 = &tsqbits;
1335 } else if (idqbits) {
1336 pri = bsfl(idqbits);
1337 q = &bsd4_idqueues[pri];
1338 which = &bsd4_idqueuebits;
1339 which2 = &idqbits;
1340 } else {
1341 return NULL;
1342 }
1343 lp = TAILQ_FIRST(q);
1344 KASSERT(lp, ("chooseproc: no lwp on busy queue"));
1345
1346 while (CPUMASK_TESTMASK(lp->lwp_cpumask, cpumask) == 0) {
1347 lp = TAILQ_NEXT(lp, lwp_procq);
1348 if (lp == NULL) {
1349 *which2 &= ~(1 << pri);
1350 goto again;
1351 }
1352 }
1353
1354 /*
1355 * If the passed lwp <chklp> is reasonably close to the selected
1356 * lwp <lp>, return NULL (indicating that <chklp> should be kept).
1357 *
1358 * Note that we must error on the side of <chklp> to avoid bouncing
1359 * between threads in the acquire code.
1360 */
1361 if (chklp) {
1362 if (chklp->lwp_priority < lp->lwp_priority + PPQ)
1363 return(NULL);
1364 }
1365
1366 /*
1367 * If the chosen lwp does not reside on this cpu spend a few
1368 * cycles looking for a better candidate at the same priority level.
1369 * This is a fallback check, setrunqueue() tries to wakeup the
1370 * correct cpu and is our front-line affinity.
1371 */
1372 if (lp->lwp_thread->td_gd != mycpu &&
1373 (chklp = TAILQ_NEXT(lp, lwp_procq)) != NULL
1374 ) {
1375 if (chklp->lwp_thread->td_gd == mycpu) {
1376 lp = chklp;
1377 }
1378 }
1379
1380 KTR_COND_LOG(usched_bsd4_chooseproc,
1381 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
1382 lp->lwp_proc->p_pid,
1383 lp->lwp_thread->td_gd->gd_cpuid,
1384 mycpu->gd_cpuid);
1385
1386 TAILQ_REMOVE(q, lp, lwp_procq);
1387 --bsd4_runqcount;
1388 if (TAILQ_EMPTY(q))
1389 *which &= ~(1 << pri);
1390 KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) != 0, ("not on runq6!"));
1391 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1392
1393 return lp;
1394 }
1395
1396 /*
1397 * chooseproc() - with a cache coherence heuristic. Try to pull a process that
1398 * has its home on the current CPU> If the process doesn't have its home here
1399 * and is a batchy one (see batcy_looser_pri_test), we can wait for a
1400 * sched_tick, may be its home will become free and pull it in. Anyway,
1401 * we can't wait more than one tick. If that tick expired, we pull in that
1402 * process, no matter what.
1403 */
1404 static
1405 struct lwp *
1406 bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp)
1407 {
1408 struct lwp *lp;
1409 struct rq *q;
1410 u_int32_t *which, *which2;
1411 u_int32_t pri;
1412 u_int32_t checks;
1413 u_int32_t rtqbits;
1414 u_int32_t tsqbits;
1415 u_int32_t idqbits;
1416 cpumask_t cpumask;
1417
1418 struct lwp * min_level_lwp = NULL;
1419 struct rq *min_q = NULL;
1420 cpumask_t siblings;
1421 cpu_node_t* cpunode = NULL;
1422 u_int32_t min_level = MAXCPU; /* number of levels < MAXCPU */
1423 u_int32_t *min_which = NULL;
1424 u_int32_t min_pri = 0;
1425 u_int32_t level = 0;
1426
1427 rtqbits = bsd4_rtqueuebits;
1428 tsqbits = bsd4_queuebits;
1429 idqbits = bsd4_idqueuebits;
1430 cpumask = mycpu->gd_cpumask;
1431
1432 /* Get the mask coresponding to the sysctl configured level */
1433 cpunode = bsd4_pcpu[mycpu->gd_cpuid].cpunode;
1434 level = usched_bsd4_stick_to_level;
1435 while (level) {
1436 cpunode = cpunode->parent_node;
1437 level--;
1438 }
1439 /* The cpus which can ellect a process */
1440 siblings = cpunode->members;
1441 checks = 0;
1442
1443 again:
1444 if (rtqbits) {
1445 pri = bsfl(rtqbits);
1446 q = &bsd4_rtqueues[pri];
1447 which = &bsd4_rtqueuebits;
1448 which2 = &rtqbits;
1449 } else if (tsqbits) {
1450 pri = bsfl(tsqbits);
1451 q = &bsd4_queues[pri];
1452 which = &bsd4_queuebits;
1453 which2 = &tsqbits;
1454 } else if (idqbits) {
1455 pri = bsfl(idqbits);
1456 q = &bsd4_idqueues[pri];
1457 which = &bsd4_idqueuebits;
1458 which2 = &idqbits;
1459 } else {
1460 /*
1461 * No more left and we didn't reach the checks limit.
1462 */
1463 bsd4_kick_helper(min_level_lwp);
1464 return NULL;
1465 }
1466 lp = TAILQ_FIRST(q);
1467 KASSERT(lp, ("chooseproc: no lwp on busy queue"));
1468
1469 /*
1470 * Limit the number of checks/queue to a configurable value to
1471 * minimize the contention (we are in a locked region
1472 */
1473 while (checks < usched_bsd4_queue_checks) {
1474 if (CPUMASK_TESTMASK(lp->lwp_cpumask, cpumask) == 0 ||
1475 (CPUMASK_TESTMASK(siblings,
1476 lp->lwp_thread->td_gd->gd_cpumask) == 0 &&
1477 (lp->lwp_rebal_ticks == sched_ticks ||
1478 lp->lwp_rebal_ticks == (int)(sched_ticks - 1)) &&
1479 bsd4_batchy_looser_pri_test(lp))) {
1480
1481 KTR_COND_LOG(usched_chooseproc_cc_not_good,
1482 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
1483 lp->lwp_proc->p_pid,
1484 (unsigned long)CPUMASK_LOWMASK(
1485 lp->lwp_thread->td_gd->gd_cpumask),
1486 (unsigned long)CPUMASK_LOWMASK(siblings),
1487 (unsigned long)CPUMASK_LOWMASK(cpumask));
1488
1489 cpunode = bsd4_pcpu[lp->lwp_thread->td_gd->gd_cpuid].cpunode;
1490 level = 0;
1491 while (cpunode) {
1492 if (CPUMASK_TESTMASK(cpunode->members,
1493 cpumask)) {
1494 break;
1495 }
1496 cpunode = cpunode->parent_node;
1497 level++;
1498 }
1499 if (level < min_level ||
1500 (level == min_level && min_level_lwp &&
1501 lp->lwp_priority < min_level_lwp->lwp_priority)) {
1502 bsd4_kick_helper(min_level_lwp);
1503 min_level_lwp = lp;
1504 min_level = level;
1505 min_q = q;
1506 min_which = which;
1507 min_pri = pri;
1508 } else {
1509 bsd4_kick_helper(lp);
1510 }
1511 lp = TAILQ_NEXT(lp, lwp_procq);
1512 if (lp == NULL) {
1513 *which2 &= ~(1 << pri);
1514 goto again;
1515 }
1516 } else {
1517 KTR_COND_LOG(usched_chooseproc_cc_elected,
1518 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
1519 lp->lwp_proc->p_pid,
1520 (unsigned long)CPUMASK_LOWMASK(
1521 lp->lwp_thread->td_gd->gd_cpumask),
1522 (unsigned long)CPUMASK_LOWMASK(siblings),
1523 (unsigned long)CPUMASK_LOWMASK(cpumask));
1524
1525 goto found;
1526 }
1527 ++checks;
1528 }
1529
1530 /*
1531 * Checks exhausted, we tried to defer too many threads, so schedule
1532 * the best of the worst.
1533 */
1534 lp = min_level_lwp;
1535 q = min_q;
1536 which = min_which;
1537 pri = min_pri;
1538 KASSERT(lp, ("chooseproc: at least the first lp was good"));
1539
1540 found:
1541
1542 /*
1543 * If the passed lwp <chklp> is reasonably close to the selected
1544 * lwp <lp>, return NULL (indicating that <chklp> should be kept).
1545 *
1546 * Note that we must error on the side of <chklp> to avoid bouncing
1547 * between threads in the acquire code.
1548 */
1549 if (chklp) {
1550 if (chklp->lwp_priority < lp->lwp_priority + PPQ) {
1551 bsd4_kick_helper(lp);
1552 return(NULL);
1553 }
1554 }
1555
1556 KTR_COND_LOG(usched_chooseproc_cc,
1557 lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
1558 lp->lwp_proc->p_pid,
1559 lp->lwp_thread->td_gd->gd_cpuid,
1560 mycpu->gd_cpuid);
1561
1562 TAILQ_REMOVE(q, lp, lwp_procq);
1563 --bsd4_runqcount;
1564 if (TAILQ_EMPTY(q))
1565 *which &= ~(1 << pri);
1566 KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) != 0, ("not on runq6!"));
1567 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1568
1569 return lp;
1570 }
1571
1572 /*
1573 * If we aren't willing to schedule a ready process on our cpu, give it's
1574 * target cpu a kick rather than wait for the next tick.
1575 *
1576 * Called with bsd4_spin held.
1577 */
1578 static
1579 void
1580 bsd4_kick_helper(struct lwp *lp)
1581 {
1582 globaldata_t gd;
1583 bsd4_pcpu_t dd;
1584 cpumask_t tmpmask;
1585
1586 if (lp == NULL)
1587 return;
1588 gd = lp->lwp_thread->td_gd;
1589 dd = &bsd4_pcpu[gd->gd_cpuid];
1590
1591 tmpmask = smp_active_mask;
1592 CPUMASK_ANDMASK(tmpmask, usched_global_cpumask);
1593 CPUMASK_ANDMASK(tmpmask, bsd4_rdyprocmask);
1594 CPUMASK_ANDMASK(tmpmask, gd->gd_cpumask);
1595 if (CPUMASK_TESTZERO(tmpmask))
1596 return;
1597
1598 ++usched_bsd4_kicks;
1599 ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, gd->gd_cpuid);
1600 if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
1601 lwkt_send_ipiq(gd, bsd4_need_user_resched_remote, NULL);
1602 } else {
1603 wakeup(dd->helper_thread);
1604 }
1605 }
1606
1607 static
1608 void
1609 bsd4_need_user_resched_remote(void *dummy)
1610 {
1611 globaldata_t gd = mycpu;
1612 bsd4_pcpu_t dd = &bsd4_pcpu[gd->gd_cpuid];
1613
1614 need_user_resched();
1615
1616 /* Call wakeup_mycpu to avoid sending IPIs to other CPUs */
1617 wakeup_mycpu(dd->helper_thread);
1618 }
1619
1620 /*
1621 * bsd4_remrunqueue_locked() removes a given process from the run queue
1622 * that it is on, clearing the queue busy bit if it becomes empty.
1623 *
1624 * Note that user process scheduler is different from the LWKT schedule.
1625 * The user process scheduler only manages user processes but it uses LWKT
1626 * underneath, and a user process operating in the kernel will often be
1627 * 'released' from our management.
1628 *
1629 * MPSAFE - bsd4_spin must be held exclusively on call
1630 */
1631 static void
1632 bsd4_remrunqueue_locked(struct lwp *lp)
1633 {
1634 struct rq *q;
1635 u_int32_t *which;
1636 u_int8_t pri;
1637
1638 KKASSERT(lp->lwp_mpflags & LWP_MP_ONRUNQ);
1639 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1640 --bsd4_runqcount;
1641 KKASSERT(bsd4_runqcount >= 0);
1642
1643 pri = lp->lwp_rqindex;
1644 switch(lp->lwp_rqtype) {
1645 case RTP_PRIO_NORMAL:
1646 q = &bsd4_queues[pri];
1647 which = &bsd4_queuebits;
1648 break;
1649 case RTP_PRIO_REALTIME:
1650 case RTP_PRIO_FIFO:
1651 q = &bsd4_rtqueues[pri];
1652 which = &bsd4_rtqueuebits;
1653 break;
1654 case RTP_PRIO_IDLE:
1655 q = &bsd4_idqueues[pri];
1656 which = &bsd4_idqueuebits;
1657 break;
1658 default:
1659 panic("remrunqueue: invalid rtprio type");
1660 /* NOT REACHED */
1661 }
1662 TAILQ_REMOVE(q, lp, lwp_procq);
1663 if (TAILQ_EMPTY(q)) {
1664 KASSERT((*which & (1 << pri)) != 0,
1665 ("remrunqueue: remove from empty queue"));
1666 *which &= ~(1 << pri);
1667 }
1668 }
1669
1670 /*
1671 * bsd4_setrunqueue_locked()
1672 *
1673 * Add a process whos rqtype and rqindex had previously been calculated
1674 * onto the appropriate run queue. Determine if the addition requires
1675 * a reschedule on a cpu and return the cpuid or -1.
1676 *
1677 * NOTE: Lower priorities are better priorities.
1678 *
1679 * MPSAFE - bsd4_spin must be held exclusively on call
1680 */
1681 static void
1682 bsd4_setrunqueue_locked(struct lwp *lp)
1683 {
1684 struct rq *q;
1685 u_int32_t *which;
1686 int pri;
1687
1688 KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
1689 atomic_set_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1690 ++bsd4_runqcount;
1691
1692 pri = lp->lwp_rqindex;
1693
1694 switch(lp->lwp_rqtype) {
1695 case RTP_PRIO_NORMAL:
1696 q = &bsd4_queues[pri];
1697 which = &bsd4_queuebits;
1698 break;
1699 case RTP_PRIO_REALTIME:
1700 case RTP_PRIO_FIFO:
1701 q = &bsd4_rtqueues[pri];
1702 which = &bsd4_rtqueuebits;
1703 break;
1704 case RTP_PRIO_IDLE:
1705 q = &bsd4_idqueues[pri];
1706 which = &bsd4_idqueuebits;
1707 break;
1708 default:
1709 panic("remrunqueue: invalid rtprio type");
1710 /* NOT REACHED */
1711 }
1712
1713 /*
1714 * Add to the correct queue and set the appropriate bit. If no
1715 * lower priority (i.e. better) processes are in the queue then
1716 * we want a reschedule, calculate the best cpu for the job.
1717 *
1718 * Always run reschedules on the LWPs original cpu.
1719 */
1720 TAILQ_INSERT_TAIL(q, lp, lwp_procq);
1721 *which |= 1 << pri;
1722 }
1723
1724 /*
1725 * For SMP systems a user scheduler helper thread is created for each
1726 * cpu and is used to allow one cpu to wakeup another for the purposes of
1727 * scheduling userland threads from setrunqueue().
1728 *
1729 * UP systems do not need the helper since there is only one cpu.
1730 *
1731 * We can't use the idle thread for this because we might block.
1732 * Additionally, doing things this way allows us to HLT idle cpus
1733 * on MP systems.
1734 *
1735 * MPSAFE
1736 */
1737 static void
1738 sched_thread(void *dummy)
1739 {
1740 globaldata_t gd;
1741 bsd4_pcpu_t dd;
1742 bsd4_pcpu_t tmpdd;
1743 struct lwp *nlp;
1744 cpumask_t mask;
1745 int cpuid;
1746 cpumask_t tmpmask;
1747 int tmpid;
1748
1749 gd = mycpu;
1750 cpuid = gd->gd_cpuid; /* doesn't change */
1751 mask = gd->gd_cpumask; /* doesn't change */
1752 dd = &bsd4_pcpu[cpuid];
1753
1754 /*
1755 * Since we are woken up only when no user processes are scheduled
1756 * on a cpu, we can run at an ultra low priority.
1757 */
1758 lwkt_setpri_self(TDPRI_USER_SCHEDULER);
1759
1760 tsleep(dd->helper_thread, 0, "sched_thread_sleep", 0);
1761
1762 for (;;) {
1763 /*
1764 * We use the LWKT deschedule-interlock trick to avoid racing
1765 * bsd4_rdyprocmask. This means we cannot block through to the
1766 * manual lwkt_switch() call we make below.
1767 */
1768 crit_enter_gd(gd);
1769 tsleep_interlock(dd->helper_thread, 0);
1770 spin_lock(&bsd4_spin);
1771 ATOMIC_CPUMASK_ORMASK(bsd4_rdyprocmask, mask);
1772
1773 clear_user_resched(); /* This satisfied the reschedule request */
1774 dd->rrcount = 0; /* Reset the round-robin counter */
1775
1776 if (CPUMASK_TESTMASK(bsd4_curprocmask, mask) == 0) {
1777 /*
1778 * No thread is currently scheduled.
1779 */
1780 KKASSERT(dd->uschedcp == NULL);
1781 if ((nlp = bsd4_chooseproc_locked(NULL)) != NULL) {
1782 KTR_COND_LOG(usched_sched_thread_no_process,
1783 nlp->lwp_proc->p_pid == usched_bsd4_pid_debug,
1784 gd->gd_cpuid,
1785 nlp->lwp_proc->p_pid,
1786 nlp->lwp_thread->td_gd->gd_cpuid);
1787
1788 ATOMIC_CPUMASK_ORMASK(bsd4_curprocmask, mask);
1789 dd->upri = nlp->lwp_priority;
1790 dd->uschedcp = nlp;
1791 dd->rrcount = 0; /* reset round robin */
1792 spin_unlock(&bsd4_spin);
1793 lwkt_acquire(nlp->lwp_thread);
1794 lwkt_schedule(nlp->lwp_thread);
1795 } else {
1796 spin_unlock(&bsd4_spin);
1797 }
1798 } else if (bsd4_runqcount) {
1799 if ((nlp = bsd4_chooseproc_locked(dd->uschedcp)) != NULL) {
1800 KTR_COND_LOG(usched_sched_thread_process,
1801 nlp->lwp_proc->p_pid == usched_bsd4_pid_debug,
1802 gd->gd_cpuid,
1803 nlp->lwp_proc->p_pid,
1804 nlp->lwp_thread->td_gd->gd_cpuid);
1805
1806 dd->upri = nlp->lwp_priority;
1807 dd->uschedcp = nlp;
1808 dd->rrcount = 0; /* reset round robin */
1809 spin_unlock(&bsd4_spin);
1810 lwkt_acquire(nlp->lwp_thread);
1811 lwkt_schedule(nlp->lwp_thread);
1812 } else {
1813 /*
1814 * CHAINING CONDITION TRAIN
1815 *
1816 * We could not deal with the scheduler wakeup
1817 * request on this cpu, locate a ready scheduler
1818 * with no current lp assignment and chain to it.
1819 *
1820 * This ensures that a wakeup race which fails due
1821 * to priority test does not leave other unscheduled
1822 * cpus idle when the runqueue is not empty.
1823 */
1824 tmpmask = bsd4_rdyprocmask;
1825 CPUMASK_NANDMASK(tmpmask, bsd4_curprocmask);
1826 CPUMASK_ANDMASK(tmpmask, smp_active_mask);
1827 if (CPUMASK_TESTNZERO(tmpmask)) {
1828 tmpid = BSFCPUMASK(tmpmask);
1829 tmpdd = &bsd4_pcpu[tmpid];
1830 ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, tmpid);
1831 spin_unlock(&bsd4_spin);
1832 wakeup(tmpdd->helper_thread);
1833 } else {
1834 spin_unlock(&bsd4_spin);
1835 }
1836
1837 KTR_LOG(usched_sched_thread_no_process_found,
1838 gd->gd_cpuid, (unsigned long)CPUMASK_LOWMASK(tmpmask));
1839 }
1840 } else {
1841 /*
1842 * The runq is empty.
1843 */
1844 spin_unlock(&bsd4_spin);
1845 }
1846
1847 /*
1848 * We're descheduled unless someone scheduled us. Switch away.
1849 * Exiting the critical section will cause splz() to be called
1850 * for us if interrupts and such are pending.
1851 */
1852 crit_exit_gd(gd);
1853 tsleep(dd->helper_thread, PINTERLOCKED, "schslp", 0);
1854 }
1855 }
1856
1857 /* sysctl stick_to_level parameter */
1858 static int
1859 sysctl_usched_bsd4_stick_to_level(SYSCTL_HANDLER_ARGS)
1860 {
1861 int error, new_val;
1862
1863 new_val = usched_bsd4_stick_to_level;
1864
1865 error = sysctl_handle_int(oidp, &new_val, 0, req);
1866 if (error != 0 || req->newptr == NULL)
1867 return (error);
1868 if (new_val > cpu_topology_levels_number - 1 || new_val < 0)
1869 return (EINVAL);
1870 usched_bsd4_stick_to_level = new_val;
1871 return (0);
1872 }
1873
1874 /*
1875 * Setup our scheduler helpers. Note that curprocmask bit 0 has already
1876 * been cleared by rqinit() and we should not mess with it further.
1877 */
1878 static void
1879 sched_thread_cpu_init(void)
1880 {
1881 int i;
1882 int smt_not_supported = 0;
1883 int cache_coherent_not_supported = 0;
1884
1885 if (bootverbose)
1886 kprintf("Start usched_bsd4 helpers on cpus:\n");
1887
1888 sysctl_ctx_init(&usched_bsd4_sysctl_ctx);
1889 usched_bsd4_sysctl_tree =
1890 SYSCTL_ADD_NODE(&usched_bsd4_sysctl_ctx,
1891 SYSCTL_STATIC_CHILDREN(_kern), OID_AUTO,
1892 "usched_bsd4", CTLFLAG_RD, 0, "");
1893
1894 for (i = 0; i < ncpus; ++i) {
1895 bsd4_pcpu_t dd = &bsd4_pcpu[i];
1896 cpumask_t mask;
1897
1898 CPUMASK_ASSBIT(mask, i);
1899
1900 if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0)
1901 continue;
1902
1903 dd->cpunode = get_cpu_node_by_cpuid(i);
1904
1905 if (dd->cpunode == NULL) {
1906 smt_not_supported = 1;
1907 cache_coherent_not_supported = 1;
1908 if (bootverbose)
1909 kprintf (" cpu%d - WARNING: No CPU NODE "
1910 "found for cpu\n", i);
1911 } else {
1912 switch (dd->cpunode->type) {
1913 case THREAD_LEVEL:
1914 if (bootverbose)
1915 kprintf (" cpu%d - HyperThreading "
1916 "available. Core siblings: ",
1917 i);
1918 break;
1919 case CORE_LEVEL:
1920 smt_not_supported = 1;
1921
1922 if (bootverbose)
1923 kprintf (" cpu%d - No HT available, "
1924 "multi-core/physical "
1925 "cpu. Physical siblings: ",
1926 i);
1927 break;
1928 case CHIP_LEVEL:
1929 smt_not_supported = 1;
1930
1931 if (bootverbose)
1932 kprintf (" cpu%d - No HT available, "
1933 "single-core/physical cpu. "
1934 "Package Siblings: ",
1935 i);
1936 break;
1937 default:
1938 /* Let's go for safe defaults here */
1939 smt_not_supported = 1;
1940 cache_coherent_not_supported = 1;
1941 if (bootverbose)
1942 kprintf (" cpu%d - Unknown cpunode->"
1943 "type=%u. Siblings: ",
1944 i,
1945 (u_int)dd->cpunode->type);
1946 break;
1947 }
1948
1949 if (bootverbose) {
1950 if (dd->cpunode->parent_node != NULL) {
1951 kprint_cpuset(&dd->cpunode->
1952 parent_node->members);
1953 kprintf("\n");
1954 } else {
1955 kprintf(" no siblings\n");
1956 }
1957 }
1958 }
1959
1960 lwkt_create(sched_thread, NULL, &dd->helper_thread, NULL,
1961 0, i, "usched %d", i);
1962
1963 /*
1964 * Allow user scheduling on the target cpu. cpu #0 has already
1965 * been enabled in rqinit().
1966 */
1967 if (i)
1968 ATOMIC_CPUMASK_NANDMASK(bsd4_curprocmask, mask);
1969 ATOMIC_CPUMASK_ORMASK(bsd4_rdyprocmask, mask);
1970 dd->upri = PRIBASE_NULL;
1971
1972 }
1973
1974 /* usched_bsd4 sysctl configurable parameters */
1975
1976 SYSCTL_ADD_INT(&usched_bsd4_sysctl_ctx,
1977 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
1978 OID_AUTO, "rrinterval", CTLFLAG_RW,
1979 &usched_bsd4_rrinterval, 0, "");
1980 SYSCTL_ADD_INT(&usched_bsd4_sysctl_ctx,
1981 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
1982 OID_AUTO, "decay", CTLFLAG_RW,
1983 &usched_bsd4_decay, 0, "Extra decay when not running");
1984 SYSCTL_ADD_INT(&usched_bsd4_sysctl_ctx,
1985 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
1986 OID_AUTO, "batch_time", CTLFLAG_RW,
1987 &usched_bsd4_batch_time, 0, "Min batch counter value");
1988 SYSCTL_ADD_LONG(&usched_bsd4_sysctl_ctx,
1989 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
1990 OID_AUTO, "kicks", CTLFLAG_RW,
1991 &usched_bsd4_kicks, "Number of kickstarts");
1992
1993 /* Add enable/disable option for SMT scheduling if supported */
1994 if (smt_not_supported) {
1995 usched_bsd4_smt = 0;
1996 SYSCTL_ADD_STRING(&usched_bsd4_sysctl_ctx,
1997 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
1998 OID_AUTO, "smt", CTLFLAG_RD,
1999 "NOT SUPPORTED", 0, "SMT NOT SUPPORTED");
2000 } else {
2001 usched_bsd4_smt = 1;
2002 SYSCTL_ADD_INT(&usched_bsd4_sysctl_ctx,
2003 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
2004 OID_AUTO, "smt", CTLFLAG_RW,
2005 &usched_bsd4_smt, 0, "Enable SMT scheduling");
2006 }
2007
2008 /*
2009 * Add enable/disable option for cache coherent scheduling
2010 * if supported
2011 */
2012 if (cache_coherent_not_supported) {
2013 usched_bsd4_cache_coherent = 0;
2014 SYSCTL_ADD_STRING(&usched_bsd4_sysctl_ctx,
2015 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
2016 OID_AUTO, "cache_coherent", CTLFLAG_RD,
2017 "NOT SUPPORTED", 0,
2018 "Cache coherence NOT SUPPORTED");
2019 } else {
2020 usched_bsd4_cache_coherent = 1;
2021 SYSCTL_ADD_INT(&usched_bsd4_sysctl_ctx,
2022 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
2023 OID_AUTO, "cache_coherent", CTLFLAG_RW,
2024 &usched_bsd4_cache_coherent, 0,
2025 "Enable/Disable cache coherent scheduling");
2026
2027 SYSCTL_ADD_INT(&usched_bsd4_sysctl_ctx,
2028 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
2029 OID_AUTO, "upri_affinity", CTLFLAG_RW,
2030 &usched_bsd4_upri_affinity, 1,
2031 "Number of PPQs in user priority check");
2032
2033 SYSCTL_ADD_INT(&usched_bsd4_sysctl_ctx,
2034 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
2035 OID_AUTO, "queue_checks", CTLFLAG_RW,
2036 &usched_bsd4_queue_checks, 5,
2037 "LWPs to check from a queue before giving up");
2038
2039 SYSCTL_ADD_PROC(&usched_bsd4_sysctl_ctx,
2040 SYSCTL_CHILDREN(usched_bsd4_sysctl_tree),
2041 OID_AUTO, "stick_to_level",
2042 CTLTYPE_INT | CTLFLAG_RW,
2043 NULL, sizeof usched_bsd4_stick_to_level,
2044 sysctl_usched_bsd4_stick_to_level, "I",
2045 "Stick a process to this level. See sysctl"
2046 "paremter hw.cpu_topology.level_description");
2047 }
2048 }
2049 SYSINIT(uschedtd, SI_BOOT2_USCHED, SI_ORDER_SECOND,
2050 sched_thread_cpu_init, NULL);
DragonFly BSD