2 * Copyright (c) 2003,2004,2009 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * lwkt_token - Implement soft token locks.
38 * Tokens are locks which serialize a thread only while the thread is
39 * running. If the thread blocks all tokens are released, then reacquired
40 * when the thread resumes.
42 * This implementation requires no critical sections or spin locks, but
43 * does use atomic_cmpset_ptr().
45 * Tokens may be recursively acquired by the same thread. However the
46 * caller must be sure to release such tokens in reverse order.
48 #include <sys/param.h>
49 #include <sys/systm.h>
50 #include <sys/kernel.h>
52 #include <sys/rtprio.h>
53 #include <sys/queue.h>
54 #include <sys/sysctl.h>
56 #include <sys/kthread.h>
57 #include <machine/cpu.h>
59 #include <sys/spinlock.h>
61 #include <sys/thread2.h>
62 #include <sys/spinlock2.h>
63 #include <sys/mplock2.h>
66 #include <vm/vm_param.h>
67 #include <vm/vm_kern.h>
68 #include <vm/vm_object.h>
69 #include <vm/vm_page.h>
70 #include <vm/vm_map.h>
71 #include <vm/vm_pager.h>
72 #include <vm/vm_extern.h>
73 #include <vm/vm_zone.h>
75 #include <machine/stdarg.h>
76 #include <machine/smp.h>
83 extern int lwkt_sched_debug
;
85 #ifndef LWKT_NUM_POOL_TOKENS
86 #define LWKT_NUM_POOL_TOKENS 4001 /* prime number */
89 struct lwkt_pool_token
{
90 struct lwkt_token token
;
93 static struct lwkt_pool_token pool_tokens
[LWKT_NUM_POOL_TOKENS
];
94 struct spinlock tok_debug_spin
= SPINLOCK_INITIALIZER(&tok_debug_spin
, "tok_debug_spin");
96 #define TOKEN_STRING "REF=%p TOK=%p TD=%p"
97 #define TOKEN_ARGS lwkt_tokref_t ref, lwkt_token_t tok, struct thread *td
98 #define CONTENDED_STRING TOKEN_STRING " (contention started)"
99 #define UNCONTENDED_STRING TOKEN_STRING " (contention stopped)"
100 #if !defined(KTR_TOKENS)
101 #define KTR_TOKENS KTR_ALL
104 KTR_INFO_MASTER(tokens
);
105 KTR_INFO(KTR_TOKENS
, tokens
, fail
, 0, TOKEN_STRING
, TOKEN_ARGS
);
106 KTR_INFO(KTR_TOKENS
, tokens
, succ
, 1, TOKEN_STRING
, TOKEN_ARGS
);
108 KTR_INFO(KTR_TOKENS
, tokens
, release
, 2, TOKEN_STRING
, TOKEN_ARGS
);
109 KTR_INFO(KTR_TOKENS
, tokens
, remote
, 3, TOKEN_STRING
, TOKEN_ARGS
);
110 KTR_INFO(KTR_TOKENS
, tokens
, reqremote
, 4, TOKEN_STRING
, TOKEN_ARGS
);
111 KTR_INFO(KTR_TOKENS
, tokens
, reqfail
, 5, TOKEN_STRING
, TOKEN_ARGS
);
112 KTR_INFO(KTR_TOKENS
, tokens
, drain
, 6, TOKEN_STRING
, TOKEN_ARGS
);
113 KTR_INFO(KTR_TOKENS
, tokens
, contention_start
, 7, CONTENDED_STRING
, TOKEN_ARGS
);
114 KTR_INFO(KTR_TOKENS
, tokens
, contention_stop
, 7, UNCONTENDED_STRING
, TOKEN_ARGS
);
117 #define logtoken(name, ref) \
118 KTR_LOG(tokens_ ## name, ref, ref->tr_tok, curthread)
121 * Global tokens. These replace the MP lock for major subsystem locking.
122 * These tokens are initially used to lockup both global and individual
125 * Once individual structures get their own locks these tokens are used
126 * only to protect global lists & other variables and to interlock
127 * allocations and teardowns and such.
129 * The UP initializer causes token acquisition to also acquire the MP lock
130 * for maximum compatibility. The feature may be enabled and disabled at
131 * any time, the MP state is copied to the tokref when the token is acquired
132 * and will not race against sysctl changes.
134 struct lwkt_token mp_token
= LWKT_TOKEN_INITIALIZER(mp_token
);
135 struct lwkt_token pmap_token
= LWKT_TOKEN_INITIALIZER(pmap_token
);
136 struct lwkt_token dev_token
= LWKT_TOKEN_INITIALIZER(dev_token
);
137 struct lwkt_token vm_token
= LWKT_TOKEN_INITIALIZER(vm_token
);
138 struct lwkt_token vmspace_token
= LWKT_TOKEN_INITIALIZER(vmspace_token
);
139 struct lwkt_token kvm_token
= LWKT_TOKEN_INITIALIZER(kvm_token
);
140 struct lwkt_token sigio_token
= LWKT_TOKEN_INITIALIZER(sigio_token
);
141 struct lwkt_token tty_token
= LWKT_TOKEN_INITIALIZER(tty_token
);
142 struct lwkt_token vnode_token
= LWKT_TOKEN_INITIALIZER(vnode_token
);
144 static int lwkt_token_spin
= 5;
145 SYSCTL_INT(_lwkt
, OID_AUTO
, token_spin
, CTLFLAG_RW
,
146 &lwkt_token_spin
, 0, "Decontention spin loops");
147 static int lwkt_token_delay
= 0;
148 SYSCTL_INT(_lwkt
, OID_AUTO
, token_delay
, CTLFLAG_RW
,
149 &lwkt_token_delay
, 0, "Decontention spin delay in ns");
152 * The collision count is bumped every time the LWKT scheduler fails
153 * to acquire needed tokens in addition to a normal lwkt_gettoken()
156 SYSCTL_LONG(_lwkt
, OID_AUTO
, mp_collisions
, CTLFLAG_RW
,
157 &mp_token
.t_collisions
, 0, "Collision counter of mp_token");
158 SYSCTL_LONG(_lwkt
, OID_AUTO
, pmap_collisions
, CTLFLAG_RW
,
159 &pmap_token
.t_collisions
, 0, "Collision counter of pmap_token");
160 SYSCTL_LONG(_lwkt
, OID_AUTO
, dev_collisions
, CTLFLAG_RW
,
161 &dev_token
.t_collisions
, 0, "Collision counter of dev_token");
162 SYSCTL_LONG(_lwkt
, OID_AUTO
, vm_collisions
, CTLFLAG_RW
,
163 &vm_token
.t_collisions
, 0, "Collision counter of vm_token");
164 SYSCTL_LONG(_lwkt
, OID_AUTO
, vmspace_collisions
, CTLFLAG_RW
,
165 &vmspace_token
.t_collisions
, 0, "Collision counter of vmspace_token");
166 SYSCTL_LONG(_lwkt
, OID_AUTO
, kvm_collisions
, CTLFLAG_RW
,
167 &kvm_token
.t_collisions
, 0, "Collision counter of kvm_token");
168 SYSCTL_LONG(_lwkt
, OID_AUTO
, sigio_collisions
, CTLFLAG_RW
,
169 &sigio_token
.t_collisions
, 0, "Collision counter of sigio_token");
170 SYSCTL_LONG(_lwkt
, OID_AUTO
, tty_collisions
, CTLFLAG_RW
,
171 &tty_token
.t_collisions
, 0, "Collision counter of tty_token");
172 SYSCTL_LONG(_lwkt
, OID_AUTO
, vnode_collisions
, CTLFLAG_RW
,
173 &vnode_token
.t_collisions
, 0, "Collision counter of vnode_token");
175 int tokens_debug_output
;
176 SYSCTL_INT(_lwkt
, OID_AUTO
, tokens_debug_output
, CTLFLAG_RW
,
177 &tokens_debug_output
, 0, "Generate stack trace N times");
180 #ifdef DEBUG_LOCKS_LATENCY
182 static long tokens_add_latency
;
183 SYSCTL_LONG(_debug
, OID_AUTO
, tokens_add_latency
, CTLFLAG_RW
,
184 &tokens_add_latency
, 0,
185 "Add spinlock latency");
190 static int _lwkt_getalltokens_sorted(thread_t td
);
193 * Acquire the initial mplock
195 * (low level boot only)
198 cpu_get_initial_mplock(void)
200 KKASSERT(mp_token
.t_ref
== NULL
);
201 if (lwkt_trytoken(&mp_token
) == FALSE
)
202 panic("cpu_get_initial_mplock");
206 * Return a pool token given an address. Use a prime number to reduce
211 _lwkt_token_pool_lookup(void *ptr
)
215 i
= (u_int
)(uintptr_t)ptr
% LWKT_NUM_POOL_TOKENS
;
216 return (&pool_tokens
[i
].token
);
220 * Initialize a tokref_t prior to making it visible in the thread's
225 _lwkt_tokref_init(lwkt_tokref_t ref
, lwkt_token_t tok
, thread_t td
, long excl
)
228 ref
->tr_count
= excl
;
233 * Attempt to acquire a shared or exclusive token. Returns TRUE on success,
236 * If TOK_EXCLUSIVE is set in mode we are attempting to get an exclusive
237 * token, otherwise are attempting to get a shared token.
239 * If TOK_EXCLREQ is set in mode this is a blocking operation, otherwise
240 * it is a non-blocking operation (for both exclusive or shared acquisions).
244 _lwkt_trytokref(lwkt_tokref_t ref
, thread_t td
, long mode
)
251 KASSERT(((mode
& TOK_EXCLREQ
) == 0 || /* non blocking */
252 td
->td_gd
->gd_intr_nesting_level
== 0 ||
253 panic_cpu_gd
== mycpu
),
254 ("Attempt to acquire token %p not already "
255 "held in hard code section", tok
));
257 if (mode
& TOK_EXCLUSIVE
) {
259 * Attempt to get an exclusive token
262 count
= tok
->t_count
;
263 oref
= tok
->t_ref
; /* can be NULL */
265 if ((count
& ~TOK_EXCLREQ
) == 0) {
267 * It is possible to get the exclusive bit.
268 * We must clear TOK_EXCLREQ on successful
271 if (atomic_cmpset_long(&tok
->t_count
, count
,
272 (count
& ~TOK_EXCLREQ
) |
274 KKASSERT(tok
->t_ref
== NULL
);
279 } else if ((count
& TOK_EXCLUSIVE
) &&
280 oref
>= &td
->td_toks_base
&&
281 oref
< td
->td_toks_stop
) {
283 * Our thread already holds the exclusive
284 * bit, we treat this tokref as a shared
285 * token (sorta) to make the token release
288 * NOTE: oref cannot race above if it
289 * happens to be ours, so we're good.
290 * But we must still have a stable
291 * variable for both parts of the
294 * NOTE: Since we already have an exclusive
295 * lock and don't need to check EXCLREQ
296 * we can just use an atomic_add here
298 atomic_add_long(&tok
->t_count
, TOK_INCR
);
299 ref
->tr_count
&= ~TOK_EXCLUSIVE
;
301 } else if ((mode
& TOK_EXCLREQ
) &&
302 (count
& TOK_EXCLREQ
) == 0) {
304 * Unable to get the exclusive bit but being
305 * asked to set the exclusive-request bit.
306 * Since we are going to retry anyway just
307 * set the bit unconditionally.
309 atomic_set_long(&tok
->t_count
, TOK_EXCLREQ
);
313 * Unable to get the exclusive bit and not
314 * being asked to set the exclusive-request
315 * (aka lwkt_trytoken()), or EXCLREQ was
325 * Attempt to get a shared token. Note that TOK_EXCLREQ
326 * for shared tokens simply means the caller intends to
327 * block. We never actually set the bit in tok->t_count.
330 count
= tok
->t_count
;
331 oref
= tok
->t_ref
; /* can be NULL */
333 if ((count
& (TOK_EXCLUSIVE
/*|TOK_EXCLREQ*/)) == 0) {
334 /* XXX EXCLREQ should work */
336 * It is possible to get the token shared.
338 if (atomic_cmpset_long(&tok
->t_count
, count
,
343 } else if ((count
& TOK_EXCLUSIVE
) &&
344 oref
>= &td
->td_toks_base
&&
345 oref
< td
->td_toks_stop
) {
347 * We own the exclusive bit on the token so
348 * we can in fact also get it shared.
350 atomic_add_long(&tok
->t_count
, TOK_INCR
);
354 * We failed to get the token shared
365 _lwkt_trytokref_spin(lwkt_tokref_t ref
, thread_t td
, long mode
)
369 if (_lwkt_trytokref(ref
, td
, mode
)) {
370 #ifdef DEBUG_LOCKS_LATENCY
372 for (j
= tokens_add_latency
; j
> 0; --j
)
377 for (spin
= lwkt_token_spin
; spin
> 0; --spin
) {
378 if (lwkt_token_delay
)
379 tsc_delay(lwkt_token_delay
);
382 if (_lwkt_trytokref(ref
, td
, mode
)) {
383 #ifdef DEBUG_LOCKS_LATENCY
385 for (j
= tokens_add_latency
; j
> 0; --j
)
395 * Release a token that we hold.
399 _lwkt_reltokref(lwkt_tokref_t ref
, thread_t td
)
406 count
= tok
->t_count
;
408 if (tok
->t_ref
== ref
) {
410 * We are an exclusive holder. We must clear tr_ref
411 * before we clear the TOK_EXCLUSIVE bit. If we are
412 * unable to clear the bit we must restore
415 KKASSERT(count
& TOK_EXCLUSIVE
);
417 if (atomic_cmpset_long(&tok
->t_count
, count
,
418 count
& ~TOK_EXCLUSIVE
)) {
425 * We are a shared holder
427 KKASSERT(count
& TOK_COUNTMASK
);
428 if (atomic_cmpset_long(&tok
->t_count
, count
,
439 * Obtain all the tokens required by the specified thread on the current
440 * cpu, return 0 on failure and non-zero on success. If a failure occurs
441 * any partially acquired tokens will be released prior to return.
443 * lwkt_getalltokens is called by the LWKT scheduler to re-acquire all
444 * tokens that the thread had to release when it switched away.
446 * If spinning is non-zero this function acquires the tokens in a particular
447 * order to deal with potential deadlocks. We simply use address order for
450 * Called from a critical section.
453 lwkt_getalltokens(thread_t td
, int spinning
)
459 return(_lwkt_getalltokens_sorted(td
));
462 * Acquire tokens in forward order, assign or validate tok->t_ref.
464 for (scan
= &td
->td_toks_base
; scan
< td
->td_toks_stop
; ++scan
) {
468 * Only try really hard on the last token
470 if (scan
== td
->td_toks_stop
- 1) {
471 if (_lwkt_trytokref_spin(scan
, td
, scan
->tr_count
))
474 if (_lwkt_trytokref(scan
, td
, scan
->tr_count
))
479 * Otherwise we failed to acquire all the tokens.
480 * Release whatever we did get.
483 ("token %p is not initialized", tok
));
484 strncpy(td
->td_gd
->gd_cnt
.v_lock_name
,
486 sizeof(td
->td_gd
->gd_cnt
.v_lock_name
) - 1);
488 if (lwkt_sched_debug
> 0) {
490 kprintf("toka %p %s %s\n",
491 tok
, tok
->t_desc
, td
->td_comm
);
493 td
->td_wmesg
= tok
->t_desc
;
495 while (--scan
>= &td
->td_toks_base
)
496 _lwkt_reltokref(scan
, td
);
504 * Release all tokens owned by the specified thread on the current cpu.
506 * This code is really simple. Even in cases where we own all the tokens
507 * note that t_ref may not match the scan for recursively held tokens which
508 * are held deeper in the stack, or for the case where a lwkt_getalltokens()
511 * Tokens are released in reverse order to reduce chasing race failures.
513 * Called from a critical section.
516 lwkt_relalltokens(thread_t td
)
521 * Weird order is to try to avoid a panic loop
523 if (td
->td_toks_have
) {
524 scan
= td
->td_toks_have
;
525 td
->td_toks_have
= NULL
;
527 scan
= td
->td_toks_stop
;
529 while (--scan
>= &td
->td_toks_base
)
530 _lwkt_reltokref(scan
, td
);
534 * This is the decontention version of lwkt_getalltokens(). The tokens are
535 * acquired in address-sorted order to deal with any deadlocks. Ultimately
536 * token failures will spin into the scheduler and get here.
538 * Called from critical section
542 _lwkt_getalltokens_sorted(thread_t td
)
544 lwkt_tokref_t sort_array
[LWKT_MAXTOKENS
];
552 * Sort the token array. Yah yah, I know this isn't fun.
554 * NOTE: Recursively acquired tokens are ordered the same as in the
555 * td_toks_array so we can always get the earliest one first.
558 scan
= &td
->td_toks_base
;
559 while (scan
< td
->td_toks_stop
) {
560 for (j
= 0; j
< i
; ++j
) {
561 if (scan
->tr_tok
< sort_array
[j
]->tr_tok
)
565 bcopy(sort_array
+ j
, sort_array
+ j
+ 1,
566 (i
- j
) * sizeof(lwkt_tokref_t
));
568 sort_array
[j
] = scan
;
575 * Acquire tokens in forward order, assign or validate tok->t_ref.
577 for (i
= 0; i
< n
; ++i
) {
578 scan
= sort_array
[i
];
582 * Only try really hard on the last token
584 if (scan
== td
->td_toks_stop
- 1) {
585 if (_lwkt_trytokref_spin(scan
, td
, scan
->tr_count
))
588 if (_lwkt_trytokref(scan
, td
, scan
->tr_count
))
593 * Otherwise we failed to acquire all the tokens.
594 * Release whatever we did get.
596 if (lwkt_sched_debug
> 0) {
598 kprintf("tokb %p %s %s\n",
599 tok
, tok
->t_desc
, td
->td_comm
);
601 td
->td_wmesg
= tok
->t_desc
;
604 scan
= sort_array
[i
];
605 _lwkt_reltokref(scan
, td
);
612 * We were successful, there is no need for another core to signal
619 * Get a serializing token. This routine can block.
622 lwkt_gettoken(lwkt_token_t tok
)
624 thread_t td
= curthread
;
627 ref
= td
->td_toks_stop
;
628 KKASSERT(ref
< &td
->td_toks_end
);
631 _lwkt_tokref_init(ref
, tok
, td
, TOK_EXCLUSIVE
|TOK_EXCLREQ
);
635 * Taking an exclusive token after holding it shared will
636 * livelock. Scan for that case and assert.
640 for (tk
= &td
->td_toks_base
; tk
< ref
; tk
++) {
641 if (tk
->tr_tok
!= tok
)
645 if (tk
->tr_count
& TOK_EXCLUSIVE
)
648 /* We found only shared instances of this token if found >0 here */
649 KASSERT((found
== 0), ("Token %p s/x livelock", tok
));
653 if (_lwkt_trytokref_spin(ref
, td
, TOK_EXCLUSIVE
|TOK_EXCLREQ
))
657 * Give up running if we can't acquire the token right now.
659 * Since the tokref is already active the scheduler now
660 * takes care of acquisition, so we need only call
663 * Since we failed this was not a recursive token so upon
664 * return tr_tok->t_ref should be assigned to this specific
667 td
->td_wmesg
= tok
->t_desc
;
670 td
->td_toks_have
= td
->td_toks_stop
- 1;
672 if (tokens_debug_output
> 0) {
673 --tokens_debug_output
;
674 spin_lock(&tok_debug_spin
);
675 kprintf("Excl Token thread %p %s %s\n",
676 td
, tok
->t_desc
, td
->td_comm
);
679 spin_unlock(&tok_debug_spin
);
684 KKASSERT(tok
->t_ref
== ref
);
688 * Similar to gettoken but we acquire a shared token instead of an exclusive
692 lwkt_gettoken_shared(lwkt_token_t tok
)
694 thread_t td
= curthread
;
697 ref
= td
->td_toks_stop
;
698 KKASSERT(ref
< &td
->td_toks_end
);
701 _lwkt_tokref_init(ref
, tok
, td
, TOK_EXCLREQ
);
705 * Taking a pool token in shared mode is a bad idea; other
706 * addresses deeper in the call stack may hash to the same pool
707 * token and you may end up with an exclusive-shared livelock.
708 * Warn in this condition.
710 if ((tok
>= &pool_tokens
[0].token
) &&
711 (tok
< &pool_tokens
[LWKT_NUM_POOL_TOKENS
].token
))
712 kprintf("Warning! Taking pool token %p in shared mode\n", tok
);
716 if (_lwkt_trytokref_spin(ref
, td
, TOK_EXCLREQ
))
720 * Give up running if we can't acquire the token right now.
722 * Since the tokref is already active the scheduler now
723 * takes care of acquisition, so we need only call
726 * Since we failed this was not a recursive token so upon
727 * return tr_tok->t_ref should be assigned to this specific
730 td
->td_wmesg
= tok
->t_desc
;
733 td
->td_toks_have
= td
->td_toks_stop
- 1;
735 if (tokens_debug_output
> 0) {
736 --tokens_debug_output
;
737 spin_lock(&tok_debug_spin
);
738 kprintf("Shar Token thread %p %s %s\n",
739 td
, tok
->t_desc
, td
->td_comm
);
742 spin_unlock(&tok_debug_spin
);
750 * Attempt to acquire a token, return TRUE on success, FALSE on failure.
752 * We setup the tokref in case we actually get the token (if we switch later
753 * it becomes mandatory so we set TOK_EXCLREQ), but we call trytokref without
754 * TOK_EXCLREQ in case we fail.
757 lwkt_trytoken(lwkt_token_t tok
)
759 thread_t td
= curthread
;
762 ref
= td
->td_toks_stop
;
763 KKASSERT(ref
< &td
->td_toks_end
);
766 _lwkt_tokref_init(ref
, tok
, td
, TOK_EXCLUSIVE
|TOK_EXCLREQ
);
768 if (_lwkt_trytokref(ref
, td
, TOK_EXCLUSIVE
))
772 * Failed, unpend the request
782 lwkt_gettoken_hard(lwkt_token_t tok
)
789 lwkt_getpooltoken(void *ptr
)
793 tok
= _lwkt_token_pool_lookup(ptr
);
799 * Release a serializing token.
801 * WARNING! All tokens must be released in reverse order. This will be
805 lwkt_reltoken(lwkt_token_t tok
)
807 thread_t td
= curthread
;
811 * Remove ref from thread token list and assert that it matches
812 * the token passed in. Tokens must be released in reverse order.
814 ref
= td
->td_toks_stop
- 1;
815 KKASSERT(ref
>= &td
->td_toks_base
&& ref
->tr_tok
== tok
);
816 _lwkt_reltokref(ref
, td
);
818 td
->td_toks_stop
= ref
;
822 lwkt_reltoken_hard(lwkt_token_t tok
)
829 * It is faster for users of lwkt_getpooltoken() to use the returned
830 * token and just call lwkt_reltoken(), but for convenience we provide
831 * this function which looks the token up based on the ident.
834 lwkt_relpooltoken(void *ptr
)
836 lwkt_token_t tok
= _lwkt_token_pool_lookup(ptr
);
841 * Return a count of the number of token refs the thread has to the
842 * specified token, whether it currently owns the token or not.
845 lwkt_cnttoken(lwkt_token_t tok
, thread_t td
)
850 for (scan
= &td
->td_toks_base
; scan
< td
->td_toks_stop
; ++scan
) {
851 if (scan
->tr_tok
== tok
)
858 * Pool tokens are used to provide a type-stable serializing token
859 * pointer that does not race against disappearing data structures.
861 * This routine is called in early boot just after we setup the BSP's
862 * globaldata structure.
865 lwkt_token_pool_init(void)
869 for (i
= 0; i
< LWKT_NUM_POOL_TOKENS
; ++i
)
870 lwkt_token_init(&pool_tokens
[i
].token
, "pool");
874 lwkt_token_pool_lookup(void *ptr
)
876 return (_lwkt_token_pool_lookup(ptr
));
880 * Initialize a token.
883 lwkt_token_init(lwkt_token_t tok
, const char *desc
)
887 tok
->t_collisions
= 0;
892 lwkt_token_uninit(lwkt_token_t tok
)
898 * Exchange the two most recent tokens on the tokref stack. This allows
899 * you to release a token out of order.
901 * We have to be careful about the case where the top two tokens are
902 * the same token. In this case tok->t_ref will point to the deeper
903 * ref and must remain pointing to the deeper ref. If we were to swap
904 * it the first release would clear the token even though a second
905 * ref is still present.
907 * Only exclusively held tokens contain a reference to the tokref which
908 * has to be flipped along with the swap.
911 lwkt_token_swap(void)
913 lwkt_tokref_t ref1
, ref2
;
914 lwkt_token_t tok1
, tok2
;
916 thread_t td
= curthread
;
920 ref1
= td
->td_toks_stop
- 1;
921 ref2
= td
->td_toks_stop
- 2;
922 KKASSERT(ref1
>= &td
->td_toks_base
);
923 KKASSERT(ref2
>= &td
->td_toks_base
);
927 count1
= ref1
->tr_count
;
928 count2
= ref2
->tr_count
;
932 ref1
->tr_count
= count2
;
934 ref2
->tr_count
= count1
;
935 if (tok1
->t_ref
== ref1
)
937 if (tok2
->t_ref
== ref2
)
945 DB_SHOW_COMMAND(tokens
, db_tok_all
)
947 struct lwkt_token
*tok
, **ptr
;
948 struct lwkt_token
*toklist
[16] = {
962 for (tok
= *ptr
; tok
; tok
= *(++ptr
)) {
963 db_printf("tok=%p tr_owner=%p t_colissions=%ld t_desc=%s\n", tok
,
964 (tok
->t_ref
? tok
->t_ref
->tr_owner
: NULL
),
965 tok
->t_collisions
, tok
->t_desc
);