2 * Copyright (c) 2009 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * Implement fast persistent locks based on atomic_cmpset_int() with
36 * semantics similar to lockmgr locks but faster and taking up much less
37 * space. Taken from HAMMER's lock implementation.
39 * These are meant to complement our LWKT tokens. Tokens are only held
40 * while the thread is running. Mutexes can be held across blocking
43 * - Exclusive priority over shared to prevent SMP starvation.
44 * - locks can be aborted (async callback, if any, will be made w/ENOLCK).
45 * - locks can be asynchronous.
46 * - synchronous fast path if no blocking occurs (async callback is not
49 * Generally speaking any caller-supplied link state must be properly
50 * initialized before use.
52 * Most of the support is in sys/mutex[2].h. We mostly provide backoff
56 #include <sys/param.h>
57 #include <sys/systm.h>
58 #include <sys/kernel.h>
59 #include <sys/sysctl.h>
60 #include <sys/thread.h>
62 #include <machine/cpufunc.h>
64 #include <sys/thread2.h>
65 #include <sys/mutex2.h>
67 static int mtx_chain_link_ex(mtx_t
*mtx
, u_int olock
);
68 static int mtx_chain_link_sh(mtx_t
*mtx
, u_int olock
);
69 static void mtx_delete_link(mtx_t
*mtx
, mtx_link_t
*link
);
72 * Exclusive-lock a mutex, block until acquired unless link is async.
73 * Recursion is allowed.
75 * Returns 0 on success, the tsleep() return code on failure, EINPROGRESS
76 * if async. If immediately successful an async exclusive lock will return 0
77 * and not issue the async callback or link the link structure. The caller
78 * must handle this case (typically this is an optimal code path).
80 * A tsleep() error can only be returned if PCATCH is specified in the flags.
83 __mtx_lock_ex(mtx_t
*mtx
, mtx_link_t
*link
, int flags
, int to
)
96 nlock
= MTX_EXCLUSIVE
| 1;
97 if (atomic_cmpset_int(&mtx
->mtx_lock
, 0, nlock
)) {
98 mtx
->mtx_owner
= curthread
;
100 link
->state
= MTX_LINK_ACQUIRED
;
106 if ((lock
& MTX_EXCLUSIVE
) && mtx
->mtx_owner
== curthread
) {
107 KKASSERT((lock
& MTX_MASK
) != MTX_MASK
);
109 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
)) {
111 link
->state
= MTX_LINK_ACQUIRED
;
119 * We need MTX_LINKSPIN to manipulate exlink or
122 * We must set MTX_EXWANTED with MTX_LINKSPIN to indicate
123 * pending exclusive requests. It cannot be set as a separate
124 * operation prior to acquiring MTX_LINKSPIN.
126 * To avoid unnecessary cpu cache traffic we poll
127 * for collisions. It is also possible that EXWANTED
128 * state failing the above test was spurious, so all the
129 * tests must be repeated if we cannot obtain LINKSPIN
130 * with the prior state tests intact (i.e. don't reload
131 * the (lock) variable here, for heaven's sake!).
133 if (lock
& MTX_LINKSPIN
) {
138 nlock
= lock
| MTX_EXWANTED
| MTX_LINKSPIN
;
140 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
) == 0) {
146 * Check for early abort.
148 if (link
->state
== MTX_LINK_ABORTED
) {
149 if (mtx
->mtx_exlink
== NULL
) {
150 atomic_clear_int(&mtx
->mtx_lock
,
154 atomic_clear_int(&mtx
->mtx_lock
,
158 link
->state
= MTX_LINK_IDLE
;
164 * Add our link to the exlink list and release LINKSPIN.
167 link
->state
= MTX_LINK_LINKED_EX
;
168 if (mtx
->mtx_exlink
) {
169 link
->next
= mtx
->mtx_exlink
;
170 link
->prev
= link
->next
->prev
;
171 link
->next
->prev
= link
;
172 link
->prev
->next
= link
;
176 mtx
->mtx_exlink
= link
;
178 isasync
= (link
->callback
!= NULL
);
179 atomic_clear_int(&mtx
->mtx_lock
, MTX_LINKSPIN
);
183 * If asynchronous lock request return without
184 * blocking, leave link structure linked.
194 error
= mtx_wait_link(mtx
, link
, flags
, to
);
201 _mtx_lock_ex_link(mtx_t
*mtx
, mtx_link_t
*link
, int flags
, int to
)
203 return(__mtx_lock_ex(mtx
, link
, flags
, to
));
207 _mtx_lock_ex(mtx_t
*mtx
, int flags
, int to
)
211 mtx_link_init(&link
);
212 return(__mtx_lock_ex(mtx
, &link
, flags
, to
));
216 _mtx_lock_ex_quick(mtx_t
*mtx
)
220 mtx_link_init(&link
);
221 return(__mtx_lock_ex(mtx
, &link
, 0, 0));
225 * Share-lock a mutex, block until acquired. Recursion is allowed.
227 * Returns 0 on success, or the tsleep() return code on failure.
228 * An error can only be returned if PCATCH is specified in the flags.
230 * NOTE: Shared locks get a mass-wakeup so if the tsleep fails we
231 * do not have to chain the wakeup().
234 __mtx_lock_sh(mtx_t
*mtx
, mtx_link_t
*link
, int flags
, int to
)
243 lock
= mtx
->mtx_lock
;
248 if (atomic_cmpset_int(&mtx
->mtx_lock
, 0, nlock
)) {
251 link
->state
= MTX_LINK_ACQUIRED
;
256 if ((lock
& (MTX_EXCLUSIVE
| MTX_EXWANTED
)) == 0) {
257 KKASSERT((lock
& MTX_MASK
) != MTX_MASK
);
259 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
)) {
262 link
->state
= MTX_LINK_ACQUIRED
;
269 * We need MTX_LINKSPIN to manipulate exlink or
272 * We must set MTX_SHWANTED with MTX_LINKSPIN to indicate
273 * pending shared requests. It cannot be set as a separate
274 * operation prior to acquiring MTX_LINKSPIN.
276 * To avoid unnecessary cpu cache traffic we poll
277 * for collisions. It is also possible that EXWANTED
278 * state failing the above test was spurious, so all the
279 * tests must be repeated if we cannot obtain LINKSPIN
280 * with the prior state tests intact (i.e. don't reload
281 * the (lock) variable here, for heaven's sake!).
283 if (lock
& MTX_LINKSPIN
) {
288 nlock
= lock
| MTX_SHWANTED
| MTX_LINKSPIN
;
290 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
) == 0) {
296 * Check for early abort. Other shared lock requestors
297 * could have sneaked in before we set LINKSPIN so make
298 * sure we undo the state properly.
300 if (link
->state
== MTX_LINK_ABORTED
) {
301 if (mtx
->mtx_shlink
) {
302 atomic_clear_int(&mtx
->mtx_lock
,
305 atomic_clear_int(&mtx
->mtx_lock
,
310 link
->state
= MTX_LINK_IDLE
;
316 * Add our link to the shlink list and release LINKSPIN.
319 link
->state
= MTX_LINK_LINKED_SH
;
320 if (mtx
->mtx_shlink
) {
321 link
->next
= mtx
->mtx_shlink
;
322 link
->prev
= link
->next
->prev
;
323 link
->next
->prev
= link
;
324 link
->prev
->next
= link
;
328 mtx
->mtx_shlink
= link
;
330 isasync
= (link
->callback
!= NULL
);
331 atomic_clear_int(&mtx
->mtx_lock
, MTX_LINKSPIN
);
335 * If asynchronous lock request return without
336 * blocking, leave link structure linked.
346 error
= mtx_wait_link(mtx
, link
, flags
, to
);
353 _mtx_lock_sh_link(mtx_t
*mtx
, mtx_link_t
*link
, int flags
, int to
)
355 return(__mtx_lock_sh(mtx
, link
, flags
, to
));
359 _mtx_lock_sh(mtx_t
*mtx
, int flags
, int to
)
363 mtx_link_init(&link
);
364 return(__mtx_lock_sh(mtx
, &link
, flags
, to
));
368 _mtx_lock_sh_quick(mtx_t
*mtx
)
372 mtx_link_init(&link
);
373 return(__mtx_lock_sh(mtx
, &link
, 0, 0));
377 * Get an exclusive spinlock the hard way.
380 _mtx_spinlock(mtx_t
*mtx
)
388 lock
= mtx
->mtx_lock
;
390 nlock
= MTX_EXCLUSIVE
| 1;
391 if (atomic_cmpset_int(&mtx
->mtx_lock
, 0, nlock
)) {
392 mtx
->mtx_owner
= curthread
;
395 } else if ((lock
& MTX_EXCLUSIVE
) &&
396 mtx
->mtx_owner
== curthread
) {
397 KKASSERT((lock
& MTX_MASK
) != MTX_MASK
);
399 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
406 for (bo
= 0; bo
< bb
; ++bo
)
414 * Attempt to acquire a spinlock, if we fail we must undo the
415 * gd->gd_spinlocks/gd->gd_curthead->td_critcount predisposition.
417 * Returns 0 on success, EAGAIN on failure.
420 _mtx_spinlock_try(mtx_t
*mtx
)
422 globaldata_t gd
= mycpu
;
428 lock
= mtx
->mtx_lock
;
430 nlock
= MTX_EXCLUSIVE
| 1;
431 if (atomic_cmpset_int(&mtx
->mtx_lock
, 0, nlock
)) {
432 mtx
->mtx_owner
= gd
->gd_curthread
;
435 } else if ((lock
& MTX_EXCLUSIVE
) &&
436 mtx
->mtx_owner
== gd
->gd_curthread
) {
437 KKASSERT((lock
& MTX_MASK
) != MTX_MASK
);
439 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
444 --gd
->gd_curthread
->td_critcount
;
456 _mtx_spinlock_sh(mtx_t
*mtx
)
464 lock
= mtx
->mtx_lock
;
465 if ((lock
& MTX_EXCLUSIVE
) == 0) {
466 KKASSERT((lock
& MTX_MASK
) != MTX_MASK
);
468 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
475 for (bo
= 0; bo
< bb
; ++bo
)
485 _mtx_lock_ex_try(mtx_t
*mtx
)
492 lock
= mtx
->mtx_lock
;
494 nlock
= MTX_EXCLUSIVE
| 1;
495 if (atomic_cmpset_int(&mtx
->mtx_lock
, 0, nlock
)) {
496 mtx
->mtx_owner
= curthread
;
500 } else if ((lock
& MTX_EXCLUSIVE
) &&
501 mtx
->mtx_owner
== curthread
) {
502 KKASSERT((lock
& MTX_MASK
) != MTX_MASK
);
504 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
)) {
518 _mtx_lock_sh_try(mtx_t
*mtx
)
525 lock
= mtx
->mtx_lock
;
526 if ((lock
& MTX_EXCLUSIVE
) == 0) {
527 KKASSERT((lock
& MTX_MASK
) != MTX_MASK
);
529 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
541 * If the lock is held exclusively it must be owned by the caller. If the
542 * lock is already a shared lock this operation is a NOP. A panic will
543 * occur if the lock is not held either shared or exclusive.
545 * The exclusive count is converted to a shared count.
548 _mtx_downgrade(mtx_t
*mtx
)
554 lock
= mtx
->mtx_lock
;
558 * NOP if already shared.
560 if ((lock
& MTX_EXCLUSIVE
) == 0) {
561 KKASSERT((lock
& MTX_MASK
) > 0);
566 * Transfer count to shared. Any additional pending shared
567 * waiters must be woken up.
569 if (lock
& MTX_SHWANTED
) {
570 if (mtx_chain_link_sh(mtx
, lock
))
574 nlock
= lock
& ~MTX_EXCLUSIVE
;
575 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
584 * Upgrade a shared lock to an exclusive lock. The upgrade will fail if
585 * the shared lock has a count other then 1. Optimize the most likely case
586 * but note that a single cmpset can fail due to WANTED races.
588 * If the lock is held exclusively it must be owned by the caller and
589 * this function will simply return without doing anything. A panic will
590 * occur if the lock is held exclusively by someone other then the caller.
592 * Returns 0 on success, EDEADLK on failure.
595 _mtx_upgrade_try(mtx_t
*mtx
)
602 lock
= mtx
->mtx_lock
;
605 if ((lock
& ~MTX_EXWANTED
) == 1) {
606 nlock
= lock
| MTX_EXCLUSIVE
;
607 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
)) {
608 mtx
->mtx_owner
= curthread
;
611 } else if (lock
& MTX_EXCLUSIVE
) {
612 KKASSERT(mtx
->mtx_owner
== curthread
);
624 * Unlock a lock. The caller must hold the lock either shared or exclusive.
626 * On the last release we handle any pending chains.
629 _mtx_unlock(mtx_t
*mtx
)
635 lock
= mtx
->mtx_lock
;
639 case MTX_EXCLUSIVE
| 1:
641 * Last release, exclusive lock.
642 * No exclusive or shared requests pending.
644 mtx
->mtx_owner
= NULL
;
646 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
649 case MTX_EXCLUSIVE
| MTX_EXWANTED
| 1:
650 case MTX_EXCLUSIVE
| MTX_EXWANTED
| MTX_SHWANTED
| 1:
652 * Last release, exclusive lock.
653 * Exclusive requests pending.
654 * Exclusive requests have priority over shared reqs.
656 if (mtx_chain_link_ex(mtx
, lock
))
659 case MTX_EXCLUSIVE
| MTX_SHWANTED
| 1:
661 * Last release, exclusive lock.
663 * Shared requests are pending. Transfer our count (1)
664 * to the first shared request, wakeup all shared reqs.
666 if (mtx_chain_link_sh(mtx
, lock
))
671 * Last release, shared lock.
672 * No exclusive or shared requests pending.
675 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
678 case MTX_EXWANTED
| 1:
679 case MTX_EXWANTED
| MTX_SHWANTED
| 1:
681 * Last release, shared lock.
683 * Exclusive requests are pending. Upgrade this
684 * final shared lock to exclusive and transfer our
685 * count (1) to the next exclusive request.
687 * Exclusive requests have priority over shared reqs.
689 if (mtx_chain_link_ex(mtx
, lock
))
692 case MTX_SHWANTED
| 1:
694 * Last release, shared lock.
695 * Shared requests pending.
697 if (mtx_chain_link_sh(mtx
, lock
))
702 * We have to loop if this is the last release but
703 * someone is fiddling with LINKSPIN.
705 if ((lock
& MTX_MASK
) == 1) {
706 KKASSERT(lock
& MTX_LINKSPIN
);
711 * Not the last release (shared or exclusive)
714 KKASSERT((nlock
& MTX_MASK
) != MTX_MASK
);
715 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
727 * Chain pending links. Called on the last release of an exclusive or
728 * shared lock when the appropriate WANTED bit is set. mtx_lock old state
729 * is passed in with the count left at 1, which we can inherit, and other
730 * bits which we must adjust in a single atomic operation.
732 * Return non-zero on success, 0 if caller needs to retry.
734 * NOTE: It's ok if MTX_EXWANTED is in an indeterminant state while we are
735 * acquiring LINKSPIN as all other cases will also need to acquire
736 * LINKSPIN when handling the EXWANTED case.
739 mtx_chain_link_ex(mtx_t
*mtx
, u_int olock
)
741 thread_t td
= curthread
;
745 olock
&= ~MTX_LINKSPIN
;
746 nlock
= olock
| MTX_LINKSPIN
| MTX_EXCLUSIVE
; /* upgrade if necc */
748 if (atomic_cmpset_int(&mtx
->mtx_lock
, olock
, nlock
)) {
749 link
= mtx
->mtx_exlink
;
750 KKASSERT(link
!= NULL
);
751 if (link
->next
== link
) {
752 mtx
->mtx_exlink
= NULL
;
753 nlock
= MTX_LINKSPIN
| MTX_EXWANTED
; /* to clear */
755 mtx
->mtx_exlink
= link
->next
;
756 link
->next
->prev
= link
->prev
;
757 link
->prev
->next
= link
->next
;
758 nlock
= MTX_LINKSPIN
; /* to clear */
760 KKASSERT(link
->state
== MTX_LINK_LINKED_EX
);
761 mtx
->mtx_owner
= link
->owner
;
765 * WARNING! The callback can only be safely
766 * made with LINKSPIN still held
767 * and in a critical section.
769 * WARNING! The link can go away after the
770 * state is set, or after the
773 if (link
->callback
) {
774 link
->state
= MTX_LINK_CALLEDBACK
;
775 link
->callback(link
, link
->arg
, 0);
777 link
->state
= MTX_LINK_ACQUIRED
;
780 atomic_clear_int(&mtx
->mtx_lock
, nlock
);
790 * Flush waiting shared locks. The lock's prior state is passed in and must
791 * be adjusted atomically only if it matches and LINKSPIN is not set.
793 * IMPORTANT! The caller has left one active count on the lock for us to
794 * consume. We will apply this to the first link, but must add
795 * additional counts for any other links.
798 mtx_chain_link_sh(mtx_t
*mtx
, u_int olock
)
800 thread_t td
= curthread
;
805 olock
&= ~MTX_LINKSPIN
;
806 nlock
= olock
| MTX_LINKSPIN
;
807 nlock
&= ~MTX_EXCLUSIVE
;
809 if (atomic_cmpset_int(&mtx
->mtx_lock
, olock
, nlock
)) {
811 * It should not be possible for SHWANTED to be set without
814 KKASSERT(mtx
->mtx_shlink
!= NULL
);
817 * We have to process the count for all shared locks before
818 * we process any of the links. Count the additional shared
819 * locks beyond the first link (which is already accounted
820 * for) and associate the full count with the lock
824 for (link
= mtx
->mtx_shlink
->next
; link
!= mtx
->mtx_shlink
;
829 atomic_add_int(&mtx
->mtx_lock
, addcount
);
832 * We can wakeup all waiting shared locks.
834 while ((link
= mtx
->mtx_shlink
) != NULL
) {
835 KKASSERT(link
->state
== MTX_LINK_LINKED_SH
);
836 if (link
->next
== link
) {
837 mtx
->mtx_shlink
= NULL
;
839 mtx
->mtx_shlink
= link
->next
;
840 link
->next
->prev
= link
->prev
;
841 link
->prev
->next
= link
->next
;
846 if (link
->callback
) {
847 link
->state
= MTX_LINK_CALLEDBACK
;
848 link
->callback(link
, link
->arg
, 0);
851 link
->state
= MTX_LINK_ACQUIRED
;
855 atomic_clear_int(&mtx
->mtx_lock
, MTX_LINKSPIN
|
866 * Delete a link structure after tsleep has failed. This code is not
867 * in the critical path as most exclusive waits are chained.
871 mtx_delete_link(mtx_t
*mtx
, mtx_link_t
*link
)
873 thread_t td
= curthread
;
878 * Acquire MTX_LINKSPIN.
880 * Do not use cmpxchg to wait for LINKSPIN to clear as this might
881 * result in too much cpu cache traffic.
885 lock
= mtx
->mtx_lock
;
886 if (lock
& MTX_LINKSPIN
) {
890 nlock
= lock
| MTX_LINKSPIN
;
891 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
897 * Delete the link and release LINKSPIN.
899 nlock
= MTX_LINKSPIN
; /* to clear */
901 switch(link
->state
) {
902 case MTX_LINK_LINKED_EX
:
903 if (link
->next
== link
) {
904 mtx
->mtx_exlink
= NULL
;
905 nlock
|= MTX_EXWANTED
; /* to clear */
907 mtx
->mtx_exlink
= link
->next
;
908 link
->next
->prev
= link
->prev
;
909 link
->prev
->next
= link
->next
;
912 case MTX_LINK_LINKED_SH
:
913 if (link
->next
== link
) {
914 mtx
->mtx_shlink
= NULL
;
915 nlock
|= MTX_SHWANTED
; /* to clear */
917 mtx
->mtx_shlink
= link
->next
;
918 link
->next
->prev
= link
->prev
;
919 link
->prev
->next
= link
->next
;
926 atomic_clear_int(&mtx
->mtx_lock
, nlock
);
931 * Wait for async lock completion or abort. Returns ENOLCK if an abort
935 mtx_wait_link(mtx_t
*mtx
, mtx_link_t
*link
, int flags
, int to
)
940 * Sleep. Handle false wakeups, interruptions, etc.
941 * The link may also have been aborted. The LINKED
942 * bit was set by this cpu so we can test it without
946 while (link
->state
& MTX_LINK_LINKED
) {
947 tsleep_interlock(link
, 0);
949 if (link
->state
& MTX_LINK_LINKED
) {
950 if (link
->state
& MTX_LINK_LINKED_SH
)
951 mycpu
->gd_cnt
.v_lock_name
[0] = 'S';
953 mycpu
->gd_cnt
.v_lock_name
[0] = 'X';
954 strncpy(mycpu
->gd_cnt
.v_lock_name
+ 1,
956 sizeof(mycpu
->gd_cnt
.v_lock_name
) - 2);
957 ++mycpu
->gd_cnt
.v_lock_colls
;
959 error
= tsleep(link
, flags
| PINTERLOCKED
,
967 * We need at least a lfence (load fence) to ensure our cpu does not
968 * reorder loads (of data outside the lock structure) prior to the
969 * remote cpu's release, since the above test may have run without
970 * any atomic interactions.
972 * If we do not do this then state updated by the other cpu before
973 * releasing its lock may not be read cleanly by our cpu when this
974 * function returns. Even though the other cpu ordered its stores,
975 * our loads can still be out of order.
980 * We are done, make sure the link structure is unlinked.
981 * It may still be on the list due to e.g. EINTR or
984 * It is possible for the tsleep to race an ABORT and cause
987 * The tsleep() can be woken up for numerous reasons and error
988 * might be zero in situations where we intend to return an error.
990 * (This is the synchronous case so state cannot be CALLEDBACK)
992 switch(link
->state
) {
993 case MTX_LINK_ACQUIRED
:
994 case MTX_LINK_CALLEDBACK
:
997 case MTX_LINK_ABORTED
:
1000 case MTX_LINK_LINKED_EX
:
1001 case MTX_LINK_LINKED_SH
:
1002 mtx_delete_link(mtx
, link
);
1006 error
= EWOULDBLOCK
;
1011 * Clear state on status returned.
1013 link
->state
= MTX_LINK_IDLE
;
1019 * Abort a mutex locking operation, causing mtx_lock_ex_link() to
1020 * return ENOLCK. This may be called at any time after the mtx_link
1021 * is initialized or the status from a previous lock has been
1022 * returned. If called prior to the next (non-try) lock attempt, the
1023 * next lock attempt using this link structure will abort instantly.
1025 * Caller must still wait for the operation to complete, either from a
1026 * blocking call that is still in progress or by calling mtx_wait_link().
1028 * If an asynchronous lock request is possibly in-progress, the caller
1029 * should call mtx_wait_link() synchronously. Note that the asynchronous
1030 * lock callback will NOT be called if a successful abort occurred. XXX
1033 mtx_abort_link(mtx_t
*mtx
, mtx_link_t
*link
)
1035 thread_t td
= curthread
;
1040 * Acquire MTX_LINKSPIN
1044 lock
= mtx
->mtx_lock
;
1045 if (lock
& MTX_LINKSPIN
) {
1049 nlock
= lock
| MTX_LINKSPIN
;
1050 if (atomic_cmpset_int(&mtx
->mtx_lock
, lock
, nlock
))
1058 * WARNING! Link structure can disappear once link->state is set.
1060 nlock
= MTX_LINKSPIN
; /* to clear */
1062 switch(link
->state
) {
1065 * Link not started yet
1067 link
->state
= MTX_LINK_ABORTED
;
1069 case MTX_LINK_LINKED_EX
:
1071 * de-link, mark aborted, and potentially wakeup the thread
1072 * or issue the callback.
1074 if (link
->next
== link
) {
1075 if (mtx
->mtx_exlink
== link
) {
1076 mtx
->mtx_exlink
= NULL
;
1077 nlock
|= MTX_EXWANTED
; /* to clear */
1080 if (mtx
->mtx_exlink
== link
)
1081 mtx
->mtx_exlink
= link
->next
;
1082 link
->next
->prev
= link
->prev
;
1083 link
->prev
->next
= link
->next
;
1087 * When aborting the async callback is still made. We must
1088 * not set the link status to ABORTED in the callback case
1089 * since there is nothing else to clear its status if the
1092 if (link
->callback
) {
1093 link
->state
= MTX_LINK_CALLEDBACK
;
1094 link
->callback(link
, link
->arg
, ENOLCK
);
1096 link
->state
= MTX_LINK_ABORTED
;
1100 case MTX_LINK_LINKED_SH
:
1102 * de-link, mark aborted, and potentially wakeup the thread
1103 * or issue the callback.
1105 if (link
->next
== link
) {
1106 if (mtx
->mtx_shlink
== link
) {
1107 mtx
->mtx_shlink
= NULL
;
1108 nlock
|= MTX_SHWANTED
; /* to clear */
1111 if (mtx
->mtx_shlink
== link
)
1112 mtx
->mtx_shlink
= link
->next
;
1113 link
->next
->prev
= link
->prev
;
1114 link
->prev
->next
= link
->next
;
1118 * When aborting the async callback is still made. We must
1119 * not set the link status to ABORTED in the callback case
1120 * since there is nothing else to clear its status if the
1123 if (link
->callback
) {
1124 link
->state
= MTX_LINK_CALLEDBACK
;
1125 link
->callback(link
, link
->arg
, ENOLCK
);
1127 link
->state
= MTX_LINK_ABORTED
;
1131 case MTX_LINK_ACQUIRED
:
1132 case MTX_LINK_CALLEDBACK
:
1134 * Too late, the lock was acquired. Let it complete.
1139 * link already aborted, do nothing.
1143 atomic_clear_int(&mtx
->mtx_lock
, nlock
);