2 * Copyright (c) 2003-2011 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * Each cpu in a system has its own self-contained light weight kernel
37 * thread scheduler, which means that generally speaking we only need
38 * to use a critical section to avoid problems. Foreign thread
39 * scheduling is queued via (async) IPIs.
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
46 #include <sys/rtprio.h>
47 #include <sys/kinfo.h>
48 #include <sys/queue.h>
49 #include <sys/sysctl.h>
50 #include <sys/kthread.h>
51 #include <machine/cpu.h>
53 #include <sys/spinlock.h>
56 #include <sys/thread2.h>
57 #include <sys/spinlock2.h>
59 #include <sys/dsched.h>
62 #include <vm/vm_param.h>
63 #include <vm/vm_kern.h>
64 #include <vm/vm_object.h>
65 #include <vm/vm_page.h>
66 #include <vm/vm_map.h>
67 #include <vm/vm_pager.h>
68 #include <vm/vm_extern.h>
70 #include <machine/stdarg.h>
71 #include <machine/smp.h>
72 #include <machine/clock.h>
74 #ifdef _KERNEL_VIRTUAL
80 #if !defined(KTR_CTXSW)
81 #define KTR_CTXSW KTR_ALL
83 KTR_INFO_MASTER(ctxsw
);
84 KTR_INFO(KTR_CTXSW
, ctxsw
, sw
, 0, "#cpu[%d].td = %p", int cpu
, struct thread
*td
);
85 KTR_INFO(KTR_CTXSW
, ctxsw
, pre
, 1, "#cpu[%d].td = %p", int cpu
, struct thread
*td
);
86 KTR_INFO(KTR_CTXSW
, ctxsw
, newtd
, 2, "#threads[%p].name = %s", struct thread
*td
, char *comm
);
87 KTR_INFO(KTR_CTXSW
, ctxsw
, deadtd
, 3, "#threads[%p].name = <dead>", struct thread
*td
);
89 static MALLOC_DEFINE(M_THREAD
, "thread", "lwkt threads");
92 static int panic_on_cscount
= 0;
94 static int64_t switch_count
= 0;
95 static int64_t preempt_hit
= 0;
96 static int64_t preempt_miss
= 0;
97 static int64_t preempt_weird
= 0;
98 static int lwkt_use_spin_port
;
99 static struct objcache
*thread_cache
;
100 int cpu_mwait_spin
= 0;
102 static void lwkt_schedule_remote(void *arg
, int arg2
, struct intrframe
*frame
);
103 static void lwkt_setcpu_remote(void *arg
);
106 * We can make all thread ports use the spin backend instead of the thread
107 * backend. This should only be set to debug the spin backend.
109 TUNABLE_INT("lwkt.use_spin_port", &lwkt_use_spin_port
);
112 SYSCTL_INT(_lwkt
, OID_AUTO
, panic_on_cscount
, CTLFLAG_RW
, &panic_on_cscount
, 0,
113 "Panic if attempting to switch lwkt's while mastering cpusync");
115 SYSCTL_QUAD(_lwkt
, OID_AUTO
, switch_count
, CTLFLAG_RW
, &switch_count
, 0,
116 "Number of switched threads");
117 SYSCTL_QUAD(_lwkt
, OID_AUTO
, preempt_hit
, CTLFLAG_RW
, &preempt_hit
, 0,
118 "Successful preemption events");
119 SYSCTL_QUAD(_lwkt
, OID_AUTO
, preempt_miss
, CTLFLAG_RW
, &preempt_miss
, 0,
120 "Failed preemption events");
121 SYSCTL_QUAD(_lwkt
, OID_AUTO
, preempt_weird
, CTLFLAG_RW
, &preempt_weird
, 0,
122 "Number of preempted threads.");
123 static int fairq_enable
= 0;
124 SYSCTL_INT(_lwkt
, OID_AUTO
, fairq_enable
, CTLFLAG_RW
,
125 &fairq_enable
, 0, "Turn on fairq priority accumulators");
126 static int fairq_bypass
= -1;
127 SYSCTL_INT(_lwkt
, OID_AUTO
, fairq_bypass
, CTLFLAG_RW
,
128 &fairq_bypass
, 0, "Allow fairq to bypass td on token failure");
129 extern int lwkt_sched_debug
;
130 int lwkt_sched_debug
= 0;
131 SYSCTL_INT(_lwkt
, OID_AUTO
, sched_debug
, CTLFLAG_RW
,
132 &lwkt_sched_debug
, 0, "Scheduler debug");
133 static u_int lwkt_spin_loops
= 10;
134 SYSCTL_UINT(_lwkt
, OID_AUTO
, spin_loops
, CTLFLAG_RW
,
135 &lwkt_spin_loops
, 0, "Scheduler spin loops until sorted decon");
136 static int preempt_enable
= 1;
137 SYSCTL_INT(_lwkt
, OID_AUTO
, preempt_enable
, CTLFLAG_RW
,
138 &preempt_enable
, 0, "Enable preemption");
139 static int lwkt_cache_threads
= 0;
140 SYSCTL_INT(_lwkt
, OID_AUTO
, cache_threads
, CTLFLAG_RD
,
141 &lwkt_cache_threads
, 0, "thread+kstack cache");
144 * These helper procedures handle the runq, they can only be called from
145 * within a critical section.
147 * WARNING! Prior to SMP being brought up it is possible to enqueue and
148 * dequeue threads belonging to other cpus, so be sure to use td->td_gd
149 * instead of 'mycpu' when referencing the globaldata structure. Once
150 * SMP live enqueuing and dequeueing only occurs on the current cpu.
154 _lwkt_dequeue(thread_t td
)
156 if (td
->td_flags
& TDF_RUNQ
) {
157 struct globaldata
*gd
= td
->td_gd
;
159 td
->td_flags
&= ~TDF_RUNQ
;
160 TAILQ_REMOVE(&gd
->gd_tdrunq
, td
, td_threadq
);
161 --gd
->gd_tdrunqcount
;
162 if (TAILQ_FIRST(&gd
->gd_tdrunq
) == NULL
)
163 atomic_clear_int(&gd
->gd_reqflags
, RQF_RUNNING
);
170 * There are a limited number of lwkt threads runnable since user
171 * processes only schedule one at a time per cpu. However, there can
172 * be many user processes in kernel mode exiting from a tsleep() which
175 * NOTE: lwkt_schedulerclock() will force a round-robin based on td_pri and
176 * will ignore user priority. This is to ensure that user threads in
177 * kernel mode get cpu at some point regardless of what the user
182 _lwkt_enqueue(thread_t td
)
186 if ((td
->td_flags
& (TDF_RUNQ
|TDF_MIGRATING
|TDF_BLOCKQ
)) == 0) {
187 struct globaldata
*gd
= td
->td_gd
;
189 td
->td_flags
|= TDF_RUNQ
;
190 xtd
= TAILQ_FIRST(&gd
->gd_tdrunq
);
192 TAILQ_INSERT_TAIL(&gd
->gd_tdrunq
, td
, td_threadq
);
193 atomic_set_int(&gd
->gd_reqflags
, RQF_RUNNING
);
196 * NOTE: td_upri - higher numbers more desireable, same sense
197 * as td_pri (typically reversed from lwp_upri).
199 * In the equal priority case we want the best selection
200 * at the beginning so the less desireable selections know
201 * that they have to setrunqueue/go-to-another-cpu, even
202 * though it means switching back to the 'best' selection.
203 * This also avoids degenerate situations when many threads
204 * are runnable or waking up at the same time.
206 * If upri matches exactly place at end/round-robin.
209 (xtd
->td_pri
>= td
->td_pri
||
210 (xtd
->td_pri
== td
->td_pri
&&
211 xtd
->td_upri
>= td
->td_upri
))) {
212 xtd
= TAILQ_NEXT(xtd
, td_threadq
);
215 TAILQ_INSERT_BEFORE(xtd
, td
, td_threadq
);
217 TAILQ_INSERT_TAIL(&gd
->gd_tdrunq
, td
, td_threadq
);
219 ++gd
->gd_tdrunqcount
;
222 * Request a LWKT reschedule if we are now at the head of the queue.
224 if (TAILQ_FIRST(&gd
->gd_tdrunq
) == td
)
230 _lwkt_thread_ctor(void *obj
, void *privdata
, int ocflags
)
232 struct thread
*td
= (struct thread
*)obj
;
234 td
->td_kstack
= NULL
;
235 td
->td_kstack_size
= 0;
236 td
->td_flags
= TDF_ALLOCATED_THREAD
;
242 _lwkt_thread_dtor(void *obj
, void *privdata
)
244 struct thread
*td
= (struct thread
*)obj
;
246 KASSERT(td
->td_flags
& TDF_ALLOCATED_THREAD
,
247 ("_lwkt_thread_dtor: not allocated from objcache"));
248 KASSERT((td
->td_flags
& TDF_ALLOCATED_STACK
) && td
->td_kstack
&&
249 td
->td_kstack_size
> 0,
250 ("_lwkt_thread_dtor: corrupted stack"));
251 kmem_free(&kernel_map
, (vm_offset_t
)td
->td_kstack
, td
->td_kstack_size
);
252 td
->td_kstack
= NULL
;
257 * Initialize the lwkt s/system.
259 * Nominally cache up to 32 thread + kstack structures. Cache more on
260 * systems with a lot of cpu cores.
265 TUNABLE_INT("lwkt.cache_threads", &lwkt_cache_threads
);
266 if (lwkt_cache_threads
== 0) {
267 lwkt_cache_threads
= ncpus
* 4;
268 if (lwkt_cache_threads
< 32)
269 lwkt_cache_threads
= 32;
271 thread_cache
= objcache_create_mbacked(
272 M_THREAD
, sizeof(struct thread
),
273 0, lwkt_cache_threads
,
274 _lwkt_thread_ctor
, _lwkt_thread_dtor
, NULL
);
276 SYSINIT(lwkt_init
, SI_BOOT2_LWKT_INIT
, SI_ORDER_FIRST
, lwkt_init
, NULL
);
279 * Schedule a thread to run. As the current thread we can always safely
280 * schedule ourselves, and a shortcut procedure is provided for that
283 * (non-blocking, self contained on a per cpu basis)
286 lwkt_schedule_self(thread_t td
)
288 KKASSERT((td
->td_flags
& TDF_MIGRATING
) == 0);
289 crit_enter_quick(td
);
290 KASSERT(td
!= &td
->td_gd
->gd_idlethread
,
291 ("lwkt_schedule_self(): scheduling gd_idlethread is illegal!"));
292 KKASSERT(td
->td_lwp
== NULL
||
293 (td
->td_lwp
->lwp_mpflags
& LWP_MP_ONRUNQ
) == 0);
299 * Deschedule a thread.
301 * (non-blocking, self contained on a per cpu basis)
304 lwkt_deschedule_self(thread_t td
)
306 crit_enter_quick(td
);
312 * LWKTs operate on a per-cpu basis
314 * WARNING! Called from early boot, 'mycpu' may not work yet.
317 lwkt_gdinit(struct globaldata
*gd
)
319 TAILQ_INIT(&gd
->gd_tdrunq
);
320 TAILQ_INIT(&gd
->gd_tdallq
);
324 * Create a new thread. The thread must be associated with a process context
325 * or LWKT start address before it can be scheduled. If the target cpu is
326 * -1 the thread will be created on the current cpu.
328 * If you intend to create a thread without a process context this function
329 * does everything except load the startup and switcher function.
332 lwkt_alloc_thread(struct thread
*td
, int stksize
, int cpu
, int flags
)
334 static int cpu_rotator
;
335 globaldata_t gd
= mycpu
;
339 * If static thread storage is not supplied allocate a thread. Reuse
340 * a cached free thread if possible. gd_freetd is used to keep an exiting
341 * thread intact through the exit.
345 if ((td
= gd
->gd_freetd
) != NULL
) {
346 KKASSERT((td
->td_flags
& (TDF_RUNNING
|TDF_PREEMPT_LOCK
|
348 gd
->gd_freetd
= NULL
;
350 td
= objcache_get(thread_cache
, M_WAITOK
);
351 KKASSERT((td
->td_flags
& (TDF_RUNNING
|TDF_PREEMPT_LOCK
|
355 KASSERT((td
->td_flags
&
356 (TDF_ALLOCATED_THREAD
|TDF_RUNNING
|TDF_PREEMPT_LOCK
)) ==
357 TDF_ALLOCATED_THREAD
,
358 ("lwkt_alloc_thread: corrupted td flags 0x%X", td
->td_flags
));
359 flags
|= td
->td_flags
& (TDF_ALLOCATED_THREAD
|TDF_ALLOCATED_STACK
);
363 * Try to reuse cached stack.
365 if ((stack
= td
->td_kstack
) != NULL
&& td
->td_kstack_size
!= stksize
) {
366 if (flags
& TDF_ALLOCATED_STACK
) {
367 kmem_free(&kernel_map
, (vm_offset_t
)stack
, td
->td_kstack_size
);
373 stack
= (void *)kmem_alloc_stack(&kernel_map
, stksize
, 0);
375 stack
= (void *)kmem_alloc_stack(&kernel_map
, stksize
,
377 flags
|= TDF_ALLOCATED_STACK
;
384 lwkt_init_thread(td
, stack
, stksize
, flags
, globaldata_find(cpu
));
389 * Initialize a preexisting thread structure. This function is used by
390 * lwkt_alloc_thread() and also used to initialize the per-cpu idlethread.
392 * All threads start out in a critical section at a priority of
393 * TDPRI_KERN_DAEMON. Higher level code will modify the priority as
394 * appropriate. This function may send an IPI message when the
395 * requested cpu is not the current cpu and consequently gd_tdallq may
396 * not be initialized synchronously from the point of view of the originating
399 * NOTE! we have to be careful in regards to creating threads for other cpus
400 * if SMP has not yet been activated.
403 lwkt_init_thread_remote(void *arg
)
408 * Protected by critical section held by IPI dispatch
410 TAILQ_INSERT_TAIL(&td
->td_gd
->gd_tdallq
, td
, td_allq
);
414 * lwkt core thread structural initialization.
416 * NOTE: All threads are initialized as mpsafe threads.
419 lwkt_init_thread(thread_t td
, void *stack
, int stksize
, int flags
,
420 struct globaldata
*gd
)
422 globaldata_t mygd
= mycpu
;
424 bzero(td
, sizeof(struct thread
));
425 td
->td_kstack
= stack
;
426 td
->td_kstack_size
= stksize
;
427 td
->td_flags
= flags
;
429 td
->td_type
= TD_TYPE_GENERIC
;
431 td
->td_pri
= TDPRI_KERN_DAEMON
;
432 td
->td_critcount
= 1;
433 td
->td_toks_have
= NULL
;
434 td
->td_toks_stop
= &td
->td_toks_base
;
435 if (lwkt_use_spin_port
|| (flags
& TDF_FORCE_SPINPORT
)) {
436 lwkt_initport_spin(&td
->td_msgport
, td
,
437 (flags
& TDF_FIXEDCPU
) ? TRUE
: FALSE
);
439 lwkt_initport_thread(&td
->td_msgport
, td
);
441 pmap_init_thread(td
);
443 * Normally initializing a thread for a remote cpu requires sending an
444 * IPI. However, the idlethread is setup before the other cpus are
445 * activated so we have to treat it as a special case. XXX manipulation
446 * of gd_tdallq requires the BGL.
448 if (gd
== mygd
|| td
== &gd
->gd_idlethread
) {
450 TAILQ_INSERT_TAIL(&gd
->gd_tdallq
, td
, td_allq
);
453 lwkt_send_ipiq(gd
, lwkt_init_thread_remote
, td
);
455 dsched_enter_thread(td
);
459 lwkt_set_comm(thread_t td
, const char *ctl
, ...)
464 kvsnprintf(td
->td_comm
, sizeof(td
->td_comm
), ctl
, va
);
466 KTR_LOG(ctxsw_newtd
, td
, td
->td_comm
);
470 * Prevent the thread from getting destroyed. Note that unlike PHOLD/PRELE
471 * this does not prevent the thread from migrating to another cpu so the
472 * gd_tdallq state is not protected by this.
475 lwkt_hold(thread_t td
)
477 atomic_add_int(&td
->td_refs
, 1);
481 lwkt_rele(thread_t td
)
483 KKASSERT(td
->td_refs
> 0);
484 atomic_add_int(&td
->td_refs
, -1);
488 lwkt_free_thread(thread_t td
)
490 KKASSERT(td
->td_refs
== 0);
491 KKASSERT((td
->td_flags
& (TDF_RUNNING
| TDF_PREEMPT_LOCK
|
492 TDF_RUNQ
| TDF_TSLEEPQ
)) == 0);
493 if (td
->td_flags
& TDF_ALLOCATED_THREAD
) {
494 objcache_put(thread_cache
, td
);
495 } else if (td
->td_flags
& TDF_ALLOCATED_STACK
) {
496 /* client-allocated struct with internally allocated stack */
497 KASSERT(td
->td_kstack
&& td
->td_kstack_size
> 0,
498 ("lwkt_free_thread: corrupted stack"));
499 kmem_free(&kernel_map
, (vm_offset_t
)td
->td_kstack
, td
->td_kstack_size
);
500 td
->td_kstack
= NULL
;
501 td
->td_kstack_size
= 0;
504 KTR_LOG(ctxsw_deadtd
, td
);
509 * Switch to the next runnable lwkt. If no LWKTs are runnable then
510 * switch to the idlethread. Switching must occur within a critical
511 * section to avoid races with the scheduling queue.
513 * We always have full control over our cpu's run queue. Other cpus
514 * that wish to manipulate our queue must use the cpu_*msg() calls to
515 * talk to our cpu, so a critical section is all that is needed and
516 * the result is very, very fast thread switching.
518 * The LWKT scheduler uses a fixed priority model and round-robins at
519 * each priority level. User process scheduling is a totally
520 * different beast and LWKT priorities should not be confused with
521 * user process priorities.
523 * PREEMPTION NOTE: Preemption occurs via lwkt_preempt(). lwkt_switch()
524 * is not called by the current thread in the preemption case, only when
525 * the preempting thread blocks (in order to return to the original thread).
527 * SPECIAL NOTE ON SWITCH ATOMICY: Certain operations such as thread
528 * migration and tsleep deschedule the current lwkt thread and call
529 * lwkt_switch(). In particular, the target cpu of the migration fully
530 * expects the thread to become non-runnable and can deadlock against
531 * cpusync operations if we run any IPIs prior to switching the thread out.
533 * WE MUST BE VERY CAREFUL NOT TO RUN SPLZ DIRECTLY OR INDIRECTLY IF
534 * THE CURRENT THREAD HAS BEEN DESCHEDULED!
539 globaldata_t gd
= mycpu
;
540 thread_t td
= gd
->gd_curthread
;
544 uint64_t tsc_base
= rdtsc();
547 KKASSERT(gd
->gd_processing_ipiq
== 0);
548 KKASSERT(td
->td_flags
& TDF_RUNNING
);
551 * Switching from within a 'fast' (non thread switched) interrupt or IPI
552 * is illegal. However, we may have to do it anyway if we hit a fatal
553 * kernel trap or we have paniced.
555 * If this case occurs save and restore the interrupt nesting level.
557 if (gd
->gd_intr_nesting_level
) {
561 if (gd
->gd_trap_nesting_level
== 0 && panic_cpu_gd
!= mycpu
) {
562 panic("lwkt_switch: Attempt to switch from a "
563 "fast interrupt, ipi, or hard code section, "
567 savegdnest
= gd
->gd_intr_nesting_level
;
568 savegdtrap
= gd
->gd_trap_nesting_level
;
569 gd
->gd_intr_nesting_level
= 0;
570 gd
->gd_trap_nesting_level
= 0;
571 if ((td
->td_flags
& TDF_PANICWARN
) == 0) {
572 td
->td_flags
|= TDF_PANICWARN
;
573 kprintf("Warning: thread switch from interrupt, IPI, "
574 "or hard code section.\n"
575 "thread %p (%s)\n", td
, td
->td_comm
);
579 gd
->gd_intr_nesting_level
= savegdnest
;
580 gd
->gd_trap_nesting_level
= savegdtrap
;
586 * Release our current user process designation if we are blocking
587 * or if a user reschedule was requested.
589 * NOTE: This function is NOT called if we are switching into or
590 * returning from a preemption.
592 * NOTE: Releasing our current user process designation may cause
593 * it to be assigned to another thread, which in turn will
594 * cause us to block in the usched acquire code when we attempt
595 * to return to userland.
597 * NOTE: On SMP systems this can be very nasty when heavy token
598 * contention is present so we want to be careful not to
599 * release the designation gratuitously.
601 if (td
->td_release
&&
602 (user_resched_wanted() || (td
->td_flags
& TDF_RUNQ
) == 0)) {
607 * Release all tokens. Once we do this we must remain in the critical
608 * section and cannot run IPIs or other interrupts until we switch away
609 * because they may implode if they try to get a token using our thread
613 if (TD_TOKS_HELD(td
))
614 lwkt_relalltokens(td
);
617 * We had better not be holding any spin locks, but don't get into an
618 * endless panic loop.
620 KASSERT(gd
->gd_spinlocks
== 0 || panicstr
!= NULL
,
621 ("lwkt_switch: still holding %d exclusive spinlocks!",
625 if (td
->td_cscount
) {
626 kprintf("Diagnostic: attempt to switch while mastering cpusync: %p\n",
628 if (panic_on_cscount
)
629 panic("switching while mastering cpusync");
634 * If we had preempted another thread on this cpu, resume the preempted
635 * thread. This occurs transparently, whether the preempted thread
636 * was scheduled or not (it may have been preempted after descheduling
639 * We have to setup the MP lock for the original thread after backing
640 * out the adjustment that was made to curthread when the original
643 if ((ntd
= td
->td_preempted
) != NULL
) {
644 KKASSERT(ntd
->td_flags
& TDF_PREEMPT_LOCK
);
645 ntd
->td_flags
|= TDF_PREEMPT_DONE
;
646 ntd
->td_contended
= 0; /* reset contended */
649 * The interrupt may have woken a thread up, we need to properly
650 * set the reschedule flag if the originally interrupted thread is
651 * at a lower priority.
653 * The interrupt may not have descheduled.
655 if (TAILQ_FIRST(&gd
->gd_tdrunq
) != ntd
)
657 goto havethread_preempted
;
661 * Figure out switch target. If we cannot switch to our desired target
662 * look for a thread that we can switch to.
664 * NOTE! The limited spin loop and related parameters are extremely
665 * important for system performance, particularly for pipes and
666 * concurrent conflicting VM faults.
668 clear_lwkt_resched();
669 ntd
= TAILQ_FIRST(&gd
->gd_tdrunq
);
673 if (TD_TOKS_NOT_HELD(ntd
) ||
674 lwkt_getalltokens(ntd
, (ntd
->td_contended
> lwkt_spin_loops
)))
678 ++gd
->gd_cnt
.v_lock_colls
;
679 ++ntd
->td_contended
; /* overflow ok */
681 if (tsc_frequency
&& rdtsc() - tsc_base
> tsc_frequency
) {
682 kprintf("lwkt_switch: excessive contended %d "
683 "thread %p\n", ntd
->td_contended
, ntd
);
687 } while (ntd
->td_contended
< (lwkt_spin_loops
>> 1));
691 * Bleh, the thread we wanted to switch to has a contended token.
692 * See if we can switch to another thread.
694 * We generally don't want to do this because it represents a
695 * priority inversion. Do not allow the case if the thread
696 * is returning to userland (not a kernel thread) AND the thread
699 while ((ntd
= TAILQ_NEXT(ntd
, td_threadq
)) != NULL
) {
700 if (ntd
->td_pri
< TDPRI_KERN_LPSCHED
&& upri
> ntd
->td_upri
)
707 if (TD_TOKS_NOT_HELD(ntd
) ||
708 lwkt_getalltokens(ntd
, (ntd
->td_contended
> lwkt_spin_loops
))) {
711 ++ntd
->td_contended
; /* overflow ok */
712 ++gd
->gd_cnt
.v_lock_colls
;
716 * Fall through, switch to idle thread to get us out of the current
717 * context. Since we were contended, prevent HLT by flagging a
724 * We either contended on ntd or the runq is empty. We must switch
725 * through the idle thread to get out of the current context.
727 ntd
= &gd
->gd_idlethread
;
728 if (gd
->gd_trap_nesting_level
== 0 && panicstr
== NULL
)
729 ASSERT_NO_TOKENS_HELD(ntd
);
730 cpu_time
.cp_msg
[0] = 0;
735 * Clear gd_idle_repeat when doing a normal switch to a non-idle
738 ntd
->td_wmesg
= NULL
;
739 ntd
->td_contended
= 0; /* reset once scheduled */
740 ++gd
->gd_cnt
.v_swtch
;
741 gd
->gd_idle_repeat
= 0;
743 havethread_preempted
:
745 * If the new target does not need the MP lock and we are holding it,
746 * release the MP lock. If the new target requires the MP lock we have
747 * already acquired it for the target.
751 KASSERT(ntd
->td_critcount
,
752 ("priority problem in lwkt_switch %d %d",
753 td
->td_critcount
, ntd
->td_critcount
));
757 * Execute the actual thread switch operation. This function
758 * returns to the current thread and returns the previous thread
759 * (which may be different from the thread we switched to).
761 * We are responsible for marking ntd as TDF_RUNNING.
763 KKASSERT((ntd
->td_flags
& TDF_RUNNING
) == 0);
765 KTR_LOG(ctxsw_sw
, gd
->gd_cpuid
, ntd
);
766 ntd
->td_flags
|= TDF_RUNNING
;
767 lwkt_switch_return(td
->td_switch(ntd
));
768 /* ntd invalid, td_switch() can return a different thread_t */
772 * catch-all. XXX is this strictly needed?
776 /* NOTE: current cpu may have changed after switch */
781 * Called by assembly in the td_switch (thread restore path) for thread
782 * bootstrap cases which do not 'return' to lwkt_switch().
785 lwkt_switch_return(thread_t otd
)
789 uint64_t tsc_base
= rdtsc();
793 exiting
= otd
->td_flags
& TDF_EXITING
;
797 * Check if otd was migrating. Now that we are on ntd we can finish
798 * up the migration. This is a bit messy but it is the only place
799 * where td is known to be fully descheduled.
801 * We can only activate the migration if otd was migrating but not
802 * held on the cpu due to a preemption chain. We still have to
803 * clear TDF_RUNNING on the old thread either way.
805 * We are responsible for clearing the previously running thread's
808 if ((rgd
= otd
->td_migrate_gd
) != NULL
&&
809 (otd
->td_flags
& TDF_PREEMPT_LOCK
) == 0) {
810 KKASSERT((otd
->td_flags
& (TDF_MIGRATING
| TDF_RUNNING
)) ==
811 (TDF_MIGRATING
| TDF_RUNNING
));
812 otd
->td_migrate_gd
= NULL
;
813 otd
->td_flags
&= ~TDF_RUNNING
;
814 lwkt_send_ipiq(rgd
, lwkt_setcpu_remote
, otd
);
816 otd
->td_flags
&= ~TDF_RUNNING
;
820 * Final exit validations (see lwp_wait()). Note that otd becomes
821 * invalid the *instant* we set TDF_MP_EXITSIG.
823 * Use the EXITING status loaded from before we clear TDF_RUNNING,
824 * because if it is not set otd becomes invalid the instant we clear
825 * TDF_RUNNING on it (otherwise, if the system is fast enough, we
826 * might 'steal' TDF_EXITING from another switch-return!).
831 mpflags
= otd
->td_mpflags
;
834 if (mpflags
& TDF_MP_EXITWAIT
) {
835 if (atomic_cmpset_int(&otd
->td_mpflags
, mpflags
,
836 mpflags
| TDF_MP_EXITSIG
)) {
841 if (atomic_cmpset_int(&otd
->td_mpflags
, mpflags
,
842 mpflags
| TDF_MP_EXITSIG
)) {
849 if (tsc_frequency
&& rdtsc() - tsc_base
> tsc_frequency
) {
850 kprintf("lwkt_switch_return: excessive TDF_EXITING "
859 * Request that the target thread preempt the current thread. Preemption
860 * can only occur only:
862 * - If our critical section is the one that we were called with
863 * - The relative priority of the target thread is higher
864 * - The target is not excessively interrupt-nested via td_nest_count
865 * - The target thread holds no tokens.
866 * - The target thread is not already scheduled and belongs to the
868 * - The current thread is not holding any spin-locks.
870 * THE CALLER OF LWKT_PREEMPT() MUST BE IN A CRITICAL SECTION. Typically
871 * this is called via lwkt_schedule() through the td_preemptable callback.
872 * critcount is the managed critical priority that we should ignore in order
873 * to determine whether preemption is possible (aka usually just the crit
874 * priority of lwkt_schedule() itself).
876 * Preemption is typically limited to interrupt threads.
878 * Operation works in a fairly straight-forward manner. The normal
879 * scheduling code is bypassed and we switch directly to the target
880 * thread. When the target thread attempts to block or switch away
881 * code at the base of lwkt_switch() will switch directly back to our
882 * thread. Our thread is able to retain whatever tokens it holds and
883 * if the target needs one of them the target will switch back to us
884 * and reschedule itself normally.
887 lwkt_preempt(thread_t ntd
, int critcount
)
889 struct globaldata
*gd
= mycpu
;
892 int save_gd_intr_nesting_level
;
895 * The caller has put us in a critical section. We can only preempt
896 * if the caller of the caller was not in a critical section (basically
897 * a local interrupt), as determined by the 'critcount' parameter. We
898 * also can't preempt if the caller is holding any spinlocks (even if
899 * he isn't in a critical section). This also handles the tokens test.
901 * YYY The target thread must be in a critical section (else it must
902 * inherit our critical section? I dunno yet).
904 KASSERT(ntd
->td_critcount
, ("BADCRIT0 %d", ntd
->td_pri
));
906 td
= gd
->gd_curthread
;
907 if (preempt_enable
== 0) {
911 if (ntd
->td_pri
<= td
->td_pri
) {
915 if (td
->td_critcount
> critcount
) {
919 if (td
->td_nest_count
>= 2) {
923 if (td
->td_cscount
) {
927 if (ntd
->td_gd
!= gd
) {
933 * We don't have to check spinlocks here as they will also bump
936 * Do not try to preempt if the target thread is holding any tokens.
937 * We could try to acquire the tokens but this case is so rare there
938 * is no need to support it.
940 KKASSERT(gd
->gd_spinlocks
== 0);
942 if (TD_TOKS_HELD(ntd
)) {
946 if (td
== ntd
|| ((td
->td_flags
| ntd
->td_flags
) & TDF_PREEMPT_LOCK
)) {
950 if (ntd
->td_preempted
) {
954 KKASSERT(gd
->gd_processing_ipiq
== 0);
957 * Since we are able to preempt the current thread, there is no need to
958 * call need_lwkt_resched().
960 * We must temporarily clear gd_intr_nesting_level around the switch
961 * since switchouts from the target thread are allowed (they will just
962 * return to our thread), and since the target thread has its own stack.
964 * A preemption must switch back to the original thread, assert the
968 ntd
->td_preempted
= td
;
969 td
->td_flags
|= TDF_PREEMPT_LOCK
;
970 KTR_LOG(ctxsw_pre
, gd
->gd_cpuid
, ntd
);
971 save_gd_intr_nesting_level
= gd
->gd_intr_nesting_level
;
972 gd
->gd_intr_nesting_level
= 0;
974 KKASSERT((ntd
->td_flags
& TDF_RUNNING
) == 0);
975 ntd
->td_flags
|= TDF_RUNNING
;
976 xtd
= td
->td_switch(ntd
);
977 KKASSERT(xtd
== ntd
);
978 lwkt_switch_return(xtd
);
979 gd
->gd_intr_nesting_level
= save_gd_intr_nesting_level
;
981 KKASSERT(ntd
->td_preempted
&& (td
->td_flags
& TDF_PREEMPT_DONE
));
982 ntd
->td_preempted
= NULL
;
983 td
->td_flags
&= ~(TDF_PREEMPT_LOCK
|TDF_PREEMPT_DONE
);
987 * Conditionally call splz() if gd_reqflags indicates work is pending.
988 * This will work inside a critical section but not inside a hard code
991 * (self contained on a per cpu basis)
996 globaldata_t gd
= mycpu
;
997 thread_t td
= gd
->gd_curthread
;
999 if ((gd
->gd_reqflags
& RQF_IDLECHECK_MASK
) &&
1000 gd
->gd_intr_nesting_level
== 0 &&
1001 td
->td_nest_count
< 2)
1008 * This version is integrated into crit_exit, reqflags has already
1009 * been tested but td_critcount has not.
1011 * We only want to execute the splz() on the 1->0 transition of
1012 * critcount and not in a hard code section or if too deeply nested.
1014 * NOTE: gd->gd_spinlocks is implied to be 0 when td_critcount is 0.
1017 lwkt_maybe_splz(thread_t td
)
1019 globaldata_t gd
= td
->td_gd
;
1021 if (td
->td_critcount
== 0 &&
1022 gd
->gd_intr_nesting_level
== 0 &&
1023 td
->td_nest_count
< 2)
1030 * Drivers which set up processing co-threads can call this function to
1031 * run the co-thread at a higher priority and to allow it to preempt
1035 lwkt_set_interrupt_support_thread(void)
1037 thread_t td
= curthread
;
1039 lwkt_setpri_self(TDPRI_INT_SUPPORT
);
1040 td
->td_flags
|= TDF_INTTHREAD
;
1041 td
->td_preemptable
= lwkt_preempt
;
1046 * This function is used to negotiate a passive release of the current
1047 * process/lwp designation with the user scheduler, allowing the user
1048 * scheduler to schedule another user thread. The related kernel thread
1049 * (curthread) continues running in the released state.
1052 lwkt_passive_release(struct thread
*td
)
1054 struct lwp
*lp
= td
->td_lwp
;
1056 td
->td_release
= NULL
;
1057 lwkt_setpri_self(TDPRI_KERN_USER
);
1059 lp
->lwp_proc
->p_usched
->release_curproc(lp
);
1064 * This implements a LWKT yield, allowing a kernel thread to yield to other
1065 * kernel threads at the same or higher priority. This function can be
1066 * called in a tight loop and will typically only yield once per tick.
1068 * Most kernel threads run at the same priority in order to allow equal
1071 * (self contained on a per cpu basis)
1076 globaldata_t gd
= mycpu
;
1077 thread_t td
= gd
->gd_curthread
;
1080 * Should never be called with spinlocks held but there is a path
1081 * via ACPI where it might happen.
1083 if (gd
->gd_spinlocks
)
1087 * Safe to call splz if we are not too-heavily nested.
1089 if ((gd
->gd_reqflags
& RQF_IDLECHECK_MASK
) && td
->td_nest_count
< 2)
1093 * Caller allows switching
1095 if (lwkt_resched_wanted()) {
1096 lwkt_schedule_self(curthread
);
1102 * The quick version processes pending interrupts and higher-priority
1103 * LWKT threads but will not round-robin same-priority LWKT threads.
1105 * When called while attempting to return to userland the only same-pri
1106 * threads are the ones which have already tried to become the current
1110 lwkt_yield_quick(void)
1112 globaldata_t gd
= mycpu
;
1113 thread_t td
= gd
->gd_curthread
;
1115 if ((gd
->gd_reqflags
& RQF_IDLECHECK_MASK
) && td
->td_nest_count
< 2)
1117 if (lwkt_resched_wanted()) {
1119 if (TAILQ_FIRST(&gd
->gd_tdrunq
) == td
) {
1120 clear_lwkt_resched();
1122 lwkt_schedule_self(curthread
);
1130 * This yield is designed for kernel threads with a user context.
1132 * The kernel acting on behalf of the user is potentially cpu-bound,
1133 * this function will efficiently allow other threads to run and also
1134 * switch to other processes by releasing.
1136 * The lwkt_user_yield() function is designed to have very low overhead
1137 * if no yield is determined to be needed.
1140 lwkt_user_yield(void)
1142 globaldata_t gd
= mycpu
;
1143 thread_t td
= gd
->gd_curthread
;
1146 * Should never be called with spinlocks held but there is a path
1147 * via ACPI where it might happen.
1149 if (gd
->gd_spinlocks
)
1153 * Always run any pending interrupts in case we are in a critical
1156 if ((gd
->gd_reqflags
& RQF_IDLECHECK_MASK
) && td
->td_nest_count
< 2)
1160 * Switch (which forces a release) if another kernel thread needs
1161 * the cpu, if userland wants us to resched, or if our kernel
1162 * quantum has run out.
1164 if (lwkt_resched_wanted() ||
1165 user_resched_wanted())
1172 * Reacquire the current process if we are released.
1174 * XXX not implemented atm. The kernel may be holding locks and such,
1175 * so we want the thread to continue to receive cpu.
1177 if (td
->td_release
== NULL
&& lp
) {
1178 lp
->lwp_proc
->p_usched
->acquire_curproc(lp
);
1179 td
->td_release
= lwkt_passive_release
;
1180 lwkt_setpri_self(TDPRI_USER_NORM
);
1186 * Generic schedule. Possibly schedule threads belonging to other cpus and
1187 * deal with threads that might be blocked on a wait queue.
1189 * We have a little helper inline function which does additional work after
1190 * the thread has been enqueued, including dealing with preemption and
1191 * setting need_lwkt_resched() (which prevents the kernel from returning
1192 * to userland until it has processed higher priority threads).
1194 * It is possible for this routine to be called after a failed _enqueue
1195 * (due to the target thread migrating, sleeping, or otherwise blocked).
1196 * We have to check that the thread is actually on the run queue!
1200 _lwkt_schedule_post(globaldata_t gd
, thread_t ntd
, int ccount
)
1202 if (ntd
->td_flags
& TDF_RUNQ
) {
1203 if (ntd
->td_preemptable
) {
1204 ntd
->td_preemptable(ntd
, ccount
); /* YYY +token */
1211 _lwkt_schedule(thread_t td
)
1213 globaldata_t mygd
= mycpu
;
1215 KASSERT(td
!= &td
->td_gd
->gd_idlethread
,
1216 ("lwkt_schedule(): scheduling gd_idlethread is illegal!"));
1217 KKASSERT((td
->td_flags
& TDF_MIGRATING
) == 0);
1218 crit_enter_gd(mygd
);
1219 KKASSERT(td
->td_lwp
== NULL
||
1220 (td
->td_lwp
->lwp_mpflags
& LWP_MP_ONRUNQ
) == 0);
1222 if (td
== mygd
->gd_curthread
) {
1226 * If we own the thread, there is no race (since we are in a
1227 * critical section). If we do not own the thread there might
1228 * be a race but the target cpu will deal with it.
1230 if (td
->td_gd
== mygd
) {
1232 _lwkt_schedule_post(mygd
, td
, 1);
1234 lwkt_send_ipiq3(td
->td_gd
, lwkt_schedule_remote
, td
, 0);
1241 lwkt_schedule(thread_t td
)
1247 lwkt_schedule_noresched(thread_t td
) /* XXX not impl */
1253 * When scheduled remotely if frame != NULL the IPIQ is being
1254 * run via doreti or an interrupt then preemption can be allowed.
1256 * To allow preemption we have to drop the critical section so only
1257 * one is present in _lwkt_schedule_post.
1260 lwkt_schedule_remote(void *arg
, int arg2
, struct intrframe
*frame
)
1262 thread_t td
= curthread
;
1265 if (frame
&& ntd
->td_preemptable
) {
1266 crit_exit_noyield(td
);
1267 _lwkt_schedule(ntd
);
1268 crit_enter_quick(td
);
1270 _lwkt_schedule(ntd
);
1275 * Thread migration using a 'Pull' method. The thread may or may not be
1276 * the current thread. It MUST be descheduled and in a stable state.
1277 * lwkt_giveaway() must be called on the cpu owning the thread.
1279 * At any point after lwkt_giveaway() is called, the target cpu may
1280 * 'pull' the thread by calling lwkt_acquire().
1282 * We have to make sure the thread is not sitting on a per-cpu tsleep
1283 * queue or it will blow up when it moves to another cpu.
1285 * MPSAFE - must be called under very specific conditions.
1288 lwkt_giveaway(thread_t td
)
1290 globaldata_t gd
= mycpu
;
1293 if (td
->td_flags
& TDF_TSLEEPQ
)
1295 KKASSERT(td
->td_gd
== gd
);
1296 TAILQ_REMOVE(&gd
->gd_tdallq
, td
, td_allq
);
1297 td
->td_flags
|= TDF_MIGRATING
;
1302 lwkt_acquire(thread_t td
)
1307 KKASSERT(td
->td_flags
& TDF_MIGRATING
);
1312 uint64_t tsc_base
= rdtsc();
1315 KKASSERT((td
->td_flags
& TDF_RUNQ
) == 0);
1316 crit_enter_gd(mygd
);
1317 DEBUG_PUSH_INFO("lwkt_acquire");
1318 while (td
->td_flags
& (TDF_RUNNING
|TDF_PREEMPT_LOCK
)) {
1319 lwkt_process_ipiq();
1321 #ifdef _KERNEL_VIRTUAL
1325 if (tsc_frequency
&& rdtsc() - tsc_base
> tsc_frequency
) {
1326 kprintf("lwkt_acquire: stuck td %p td->td_flags %08x\n",
1335 TAILQ_INSERT_TAIL(&mygd
->gd_tdallq
, td
, td_allq
);
1336 td
->td_flags
&= ~TDF_MIGRATING
;
1339 crit_enter_gd(mygd
);
1340 TAILQ_INSERT_TAIL(&mygd
->gd_tdallq
, td
, td_allq
);
1341 td
->td_flags
&= ~TDF_MIGRATING
;
1347 * Generic deschedule. Descheduling threads other then your own should be
1348 * done only in carefully controlled circumstances. Descheduling is
1351 * This function may block if the cpu has run out of messages.
1354 lwkt_deschedule(thread_t td
)
1357 if (td
== curthread
) {
1360 if (td
->td_gd
== mycpu
) {
1363 lwkt_send_ipiq(td
->td_gd
, (ipifunc1_t
)lwkt_deschedule
, td
);
1370 * Set the target thread's priority. This routine does not automatically
1371 * switch to a higher priority thread, LWKT threads are not designed for
1372 * continuous priority changes. Yield if you want to switch.
1375 lwkt_setpri(thread_t td
, int pri
)
1377 if (td
->td_pri
!= pri
) {
1380 if (td
->td_flags
& TDF_RUNQ
) {
1381 KKASSERT(td
->td_gd
== mycpu
);
1393 * Set the initial priority for a thread prior to it being scheduled for
1394 * the first time. The thread MUST NOT be scheduled before or during
1395 * this call. The thread may be assigned to a cpu other then the current
1398 * Typically used after a thread has been created with TDF_STOPPREQ,
1399 * and before the thread is initially scheduled.
1402 lwkt_setpri_initial(thread_t td
, int pri
)
1405 KKASSERT((td
->td_flags
& TDF_RUNQ
) == 0);
1410 lwkt_setpri_self(int pri
)
1412 thread_t td
= curthread
;
1414 KKASSERT(pri
>= 0 && pri
<= TDPRI_MAX
);
1416 if (td
->td_flags
& TDF_RUNQ
) {
1427 * hz tick scheduler clock for LWKT threads
1430 lwkt_schedulerclock(thread_t td
)
1432 globaldata_t gd
= td
->td_gd
;
1435 if (TAILQ_FIRST(&gd
->gd_tdrunq
) == td
) {
1437 * If the current thread is at the head of the runq shift it to the
1438 * end of any equal-priority threads and request a LWKT reschedule
1441 * Ignore upri in this situation. There will only be one user thread
1442 * in user mode, all others will be user threads running in kernel
1443 * mode and we have to make sure they get some cpu.
1445 xtd
= TAILQ_NEXT(td
, td_threadq
);
1446 if (xtd
&& xtd
->td_pri
== td
->td_pri
) {
1447 TAILQ_REMOVE(&gd
->gd_tdrunq
, td
, td_threadq
);
1448 while (xtd
&& xtd
->td_pri
== td
->td_pri
)
1449 xtd
= TAILQ_NEXT(xtd
, td_threadq
);
1451 TAILQ_INSERT_BEFORE(xtd
, td
, td_threadq
);
1453 TAILQ_INSERT_TAIL(&gd
->gd_tdrunq
, td
, td_threadq
);
1454 need_lwkt_resched();
1458 * If we scheduled a thread other than the one at the head of the
1459 * queue always request a reschedule every tick.
1461 need_lwkt_resched();
1466 * Migrate the current thread to the specified cpu.
1468 * This is accomplished by descheduling ourselves from the current cpu
1469 * and setting td_migrate_gd. The lwkt_switch() code will detect that the
1470 * 'old' thread wants to migrate after it has been completely switched out
1471 * and will complete the migration.
1473 * TDF_MIGRATING prevents scheduling races while the thread is being migrated.
1475 * We must be sure to release our current process designation (if a user
1476 * process) before clearing out any tsleepq we are on because the release
1477 * code may re-add us.
1479 * We must be sure to remove ourselves from the current cpu's tsleepq
1480 * before potentially moving to another queue. The thread can be on
1481 * a tsleepq due to a left-over tsleep_interlock().
1485 lwkt_setcpu_self(globaldata_t rgd
)
1487 thread_t td
= curthread
;
1489 if (td
->td_gd
!= rgd
) {
1490 crit_enter_quick(td
);
1494 if (td
->td_flags
& TDF_TSLEEPQ
)
1498 * Set TDF_MIGRATING to prevent a spurious reschedule while we are
1499 * trying to deschedule ourselves and switch away, then deschedule
1500 * ourself, remove us from tdallq, and set td_migrate_gd. Finally,
1501 * call lwkt_switch() to complete the operation.
1503 td
->td_flags
|= TDF_MIGRATING
;
1504 lwkt_deschedule_self(td
);
1505 TAILQ_REMOVE(&td
->td_gd
->gd_tdallq
, td
, td_allq
);
1506 td
->td_migrate_gd
= rgd
;
1510 * We are now on the target cpu
1512 KKASSERT(rgd
== mycpu
);
1513 TAILQ_INSERT_TAIL(&rgd
->gd_tdallq
, td
, td_allq
);
1514 crit_exit_quick(td
);
1519 lwkt_migratecpu(int cpuid
)
1523 rgd
= globaldata_find(cpuid
);
1524 lwkt_setcpu_self(rgd
);
1528 * Remote IPI for cpu migration (called while in a critical section so we
1529 * do not have to enter another one).
1531 * The thread (td) has already been completely descheduled from the
1532 * originating cpu and we can simply assert the case. The thread is
1533 * assigned to the new cpu and enqueued.
1535 * The thread will re-add itself to tdallq when it resumes execution.
1538 lwkt_setcpu_remote(void *arg
)
1541 globaldata_t gd
= mycpu
;
1543 KKASSERT((td
->td_flags
& (TDF_RUNNING
|TDF_PREEMPT_LOCK
)) == 0);
1546 td
->td_flags
&= ~TDF_MIGRATING
;
1547 KKASSERT(td
->td_migrate_gd
== NULL
);
1548 KKASSERT(td
->td_lwp
== NULL
||
1549 (td
->td_lwp
->lwp_mpflags
& LWP_MP_ONRUNQ
) == 0);
1554 lwkt_preempted_proc(void)
1556 thread_t td
= curthread
;
1557 while (td
->td_preempted
)
1558 td
= td
->td_preempted
;
1563 * Create a kernel process/thread/whatever. It shares it's address space
1564 * with proc0 - ie: kernel only.
1566 * If the cpu is not specified one will be selected. In the future
1567 * specifying a cpu of -1 will enable kernel thread migration between
1571 lwkt_create(void (*func
)(void *), void *arg
, struct thread
**tdp
,
1572 thread_t
template, int tdflags
, int cpu
, const char *fmt
, ...)
1577 td
= lwkt_alloc_thread(template, LWKT_THREAD_STACK
, cpu
,
1581 cpu_set_thread_handler(td
, lwkt_exit
, func
, arg
);
1584 * Set up arg0 for 'ps' etc
1586 __va_start(ap
, fmt
);
1587 kvsnprintf(td
->td_comm
, sizeof(td
->td_comm
), fmt
, ap
);
1591 * Schedule the thread to run
1593 if (td
->td_flags
& TDF_NOSTART
)
1594 td
->td_flags
&= ~TDF_NOSTART
;
1601 * Destroy an LWKT thread. Warning! This function is not called when
1602 * a process exits, cpu_proc_exit() directly calls cpu_thread_exit() and
1603 * uses a different reaping mechanism.
1608 thread_t td
= curthread
;
1613 * Do any cleanup that might block here
1615 if (td
->td_flags
& TDF_VERBOSE
)
1616 kprintf("kthread %p %s has exited\n", td
, td
->td_comm
);
1618 dsched_exit_thread(td
);
1621 * Get us into a critical section to interlock gd_freetd and loop
1622 * until we can get it freed.
1624 * We have to cache the current td in gd_freetd because objcache_put()ing
1625 * it would rip it out from under us while our thread is still active.
1627 * We are the current thread so of course our own TDF_RUNNING bit will
1628 * be set, so unlike the lwp reap code we don't wait for it to clear.
1631 crit_enter_quick(td
);
1634 tsleep(td
, 0, "tdreap", 1);
1637 if ((std
= gd
->gd_freetd
) != NULL
) {
1638 KKASSERT((std
->td_flags
& (TDF_RUNNING
|TDF_PREEMPT_LOCK
)) == 0);
1639 gd
->gd_freetd
= NULL
;
1640 objcache_put(thread_cache
, std
);
1647 * Remove thread resources from kernel lists and deschedule us for
1648 * the last time. We cannot block after this point or we may end
1649 * up with a stale td on the tsleepq.
1651 * None of this may block, the critical section is the only thing
1652 * protecting tdallq and the only thing preventing new lwkt_hold()
1655 if (td
->td_flags
& TDF_TSLEEPQ
)
1657 lwkt_deschedule_self(td
);
1658 lwkt_remove_tdallq(td
);
1659 KKASSERT(td
->td_refs
== 0);
1664 KKASSERT(gd
->gd_freetd
== NULL
);
1665 if (td
->td_flags
& TDF_ALLOCATED_THREAD
)
1671 lwkt_remove_tdallq(thread_t td
)
1673 KKASSERT(td
->td_gd
== mycpu
);
1674 TAILQ_REMOVE(&td
->td_gd
->gd_tdallq
, td
, td_allq
);
1678 * Code reduction and branch prediction improvements. Call/return
1679 * overhead on modern cpus often degenerates into 0 cycles due to
1680 * the cpu's branch prediction hardware and return pc cache. We
1681 * can take advantage of this by not inlining medium-complexity
1682 * functions and we can also reduce the branch prediction impact
1683 * by collapsing perfectly predictable branches into a single
1684 * procedure instead of duplicating it.
1686 * Is any of this noticeable? Probably not, so I'll take the
1687 * smaller code size.
1690 crit_exit_wrapper(__DEBUG_CRIT_ARG__
)
1692 _crit_exit(mycpu __DEBUG_CRIT_PASS_ARG__
);
1698 thread_t td
= curthread
;
1699 int lcrit
= td
->td_critcount
;
1701 td
->td_critcount
= 0;
1702 panic("td_critcount is/would-go negative! %p %d", td
, lcrit
);
1707 * Called from debugger/panic on cpus which have been stopped. We must still
1708 * process the IPIQ while stopped.
1710 * If we are dumping also try to process any pending interrupts. This may
1711 * or may not work depending on the state of the cpu at the point it was
1715 lwkt_smp_stopped(void)
1717 globaldata_t gd
= mycpu
;
1720 lwkt_process_ipiq();
1721 --gd
->gd_intr_nesting_level
;
1723 ++gd
->gd_intr_nesting_level
;
1725 lwkt_process_ipiq();