3 * The Regents of the University of California. All rights reserved.
6 * John S. Dyson. All rights reserved.
8 * This code contains ideas from software contributed to Berkeley by
9 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
10 * System project at Carnegie-Mellon University.
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the University of
23 * California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95
41 * $FreeBSD: src/sys/kern/kern_lock.c,v 1.31.2.3 2001/12/25 01:44:44 dillon Exp $
42 * $DragonFly: src/sys/kern/kern_lock.c,v 1.27 2008/01/09 10:59:12 corecode Exp $
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
52 #include <sys/sysctl.h>
53 #include <sys/spinlock.h>
54 #include <sys/thread2.h>
55 #include <sys/spinlock2.h>
58 * 0: no warnings, 1: warnings, 2: panic
60 static int lockmgr_from_int
= 1;
61 SYSCTL_INT(_debug
, OID_AUTO
, lockmgr_from_int
, CTLFLAG_RW
, &lockmgr_from_int
, 0, "");
64 * Locking primitives implementation.
65 * Locks provide shared/exclusive sychronization.
68 #ifdef SIMPLELOCK_DEBUG
69 #define COUNT(td, x) (td)->td_locks += (x)
74 #define LOCK_WAIT_TIME 100
75 #define LOCK_SAMPLE_WAIT 7
77 #if defined(DIAGNOSTIC)
80 #define LOCK_INLINE __inline
83 #define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \
84 LK_SHARE_NONZERO | LK_WAIT_NONZERO)
86 static int acquire(struct lock
*lkp
, int extflags
, int wanted
);
88 static LOCK_INLINE
void
89 sharelock(struct lock
*lkp
, int incr
) {
90 lkp
->lk_flags
|= LK_SHARE_NONZERO
;
91 lkp
->lk_sharecount
+= incr
;
94 static LOCK_INLINE
int
95 shareunlock(struct lock
*lkp
, int decr
)
99 KASSERT(lkp
->lk_sharecount
>= decr
, ("shareunlock: count < decr"));
101 if (lkp
->lk_sharecount
== decr
) {
102 lkp
->lk_flags
&= ~LK_SHARE_NONZERO
;
103 if (lkp
->lk_flags
& (LK_WANT_UPGRADE
| LK_WANT_EXCL
)) {
106 lkp
->lk_sharecount
= 0;
108 lkp
->lk_sharecount
-= decr
;
114 * lock acquisition helper routine. Called with the lock's spinlock held.
117 acquire(struct lock
*lkp
, int extflags
, int wanted
)
121 if ((extflags
& LK_NOWAIT
) && (lkp
->lk_flags
& wanted
)) {
125 while ((lkp
->lk_flags
& wanted
) != 0) {
126 lkp
->lk_flags
|= LK_WAIT_NONZERO
;
130 * Atomic spinlock release/sleep/reacquire.
132 error
= ssleep(lkp
, &lkp
->lk_spinlock
,
133 ((extflags
& LK_PCATCH
) ? PCATCH
: 0),
135 ((extflags
& LK_TIMELOCK
) ? lkp
->lk_timo
: 0));
136 if (lkp
->lk_waitcount
== 1) {
137 lkp
->lk_flags
&= ~LK_WAIT_NONZERO
;
138 lkp
->lk_waitcount
= 0;
144 if (extflags
& LK_SLEEPFAIL
)
151 * Set, change, or release a lock.
153 * Shared requests increment the shared count. Exclusive requests set the
154 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already
155 * accepted shared locks and shared-to-exclusive upgrades to go away.
157 * A spinlock is held for most of the procedure. We must not do anything
158 * fancy while holding the spinlock.
162 lockmgr(struct lock
*lkp
, u_int flags
)
164 debuglockmgr(struct lock
*lkp
, u_int flags
,
165 const char *name
, const char *file
, int line
)
177 if (lockmgr_from_int
&& mycpu
->gd_intr_nesting_level
&&
178 (flags
& LK_NOWAIT
) == 0 &&
179 (flags
& LK_TYPE_MASK
) != LK_RELEASE
&& didpanic
== 0) {
181 if (lockmgr_from_int
== 2) {
184 "lockmgr %s from %p: called from interrupt",
185 lkp
->lk_wmesg
, ((int **)&lkp
)[-1]);
189 "lockmgr %s from %p: called from interrupt\n",
190 lkp
->lk_wmesg
, ((int **)&lkp
)[-1]);
193 if (lockmgr_from_int
== 2) {
196 "lockmgr %s from %s:%d: called from interrupt",
197 lkp
->lk_wmesg
, file
, line
);
201 "lockmgr %s from %s:%d: called from interrupt\n",
202 lkp
->lk_wmesg
, file
, line
);
208 * So sue me, I'm too tired.
210 if (spin_trylock_wr(&lkp
->lk_spinlock
) == FALSE
) {
211 if (flags
& LK_NOSPINWAIT
)
213 spin_lock_wr(&lkp
->lk_spinlock
);
216 extflags
= (flags
| lkp
->lk_flags
) & LK_EXTFLG_MASK
;
219 switch (flags
& LK_TYPE_MASK
) {
222 * If we are not the exclusive lock holder, we have to block
223 * while there is an exclusive lock holder or while an
224 * exclusive lock request or upgrade request is in progress.
226 * However, if P_DEADLKTREAT is set, we override exclusive
227 * lock requests or upgrade requests ( but not the exclusive
230 if (lkp
->lk_lockholder
!= td
) {
231 if (td
->td_flags
& TDF_DEADLKTREAT
) {
241 LK_HAVE_EXCL
| LK_WANT_EXCL
|
252 * We hold an exclusive lock, so downgrade it to shared.
253 * An alternative would be to fail with EDEADLK.
257 /* fall into downgrade */
260 if (lkp
->lk_lockholder
!= td
|| lkp
->lk_exclusivecount
== 0) {
261 spin_unlock_wr(&lkp
->lk_spinlock
);
262 panic("lockmgr: not holding exclusive lock");
264 sharelock(lkp
, lkp
->lk_exclusivecount
);
265 lkp
->lk_exclusivecount
= 0;
266 lkp
->lk_flags
&= ~LK_HAVE_EXCL
;
267 lkp
->lk_lockholder
= LK_NOTHREAD
;
268 if (lkp
->lk_waitcount
)
274 * If another process is ahead of us to get an upgrade,
275 * then we want to fail rather than have an intervening
278 if (lkp
->lk_flags
& LK_WANT_UPGRADE
) {
279 dowakeup
= shareunlock(lkp
, 1);
284 /* fall into normal upgrade */
288 * Upgrade a shared lock to an exclusive one. If another
289 * shared lock has already requested an upgrade to an
290 * exclusive lock, our shared lock is released and an
291 * exclusive lock is requested (which will be granted
292 * after the upgrade). If we return an error, the file
293 * will always be unlocked.
295 if ((lkp
->lk_lockholder
== td
) || (lkp
->lk_sharecount
<= 0)) {
296 spin_unlock_wr(&lkp
->lk_spinlock
);
297 panic("lockmgr: upgrade exclusive lock");
299 dowakeup
+= shareunlock(lkp
, 1);
302 * If we are just polling, check to see if we will block.
304 if ((extflags
& LK_NOWAIT
) &&
305 ((lkp
->lk_flags
& LK_WANT_UPGRADE
) ||
306 lkp
->lk_sharecount
> 1)) {
310 if ((lkp
->lk_flags
& LK_WANT_UPGRADE
) == 0) {
312 * We are first shared lock to request an upgrade, so
313 * request upgrade and wait for the shared count to
314 * drop to zero, then take exclusive lock.
316 lkp
->lk_flags
|= LK_WANT_UPGRADE
;
317 error
= acquire(lkp
, extflags
, LK_SHARE_NONZERO
);
318 lkp
->lk_flags
&= ~LK_WANT_UPGRADE
;
322 lkp
->lk_flags
|= LK_HAVE_EXCL
;
323 lkp
->lk_lockholder
= td
;
324 if (lkp
->lk_exclusivecount
!= 0) {
325 spin_unlock_wr(&lkp
->lk_spinlock
);
326 panic("lockmgr: non-zero exclusive count");
328 lkp
->lk_exclusivecount
= 1;
329 #if defined(DEBUG_LOCKS)
330 lkp
->lk_filename
= file
;
331 lkp
->lk_lineno
= line
;
332 lkp
->lk_lockername
= name
;
338 * Someone else has requested upgrade. Release our shared
339 * lock, awaken upgrade requestor if we are the last shared
340 * lock, then request an exclusive lock.
342 if ( (lkp
->lk_flags
& (LK_SHARE_NONZERO
|LK_WAIT_NONZERO
)) ==
346 /* fall into exclusive request */
349 if (lkp
->lk_lockholder
== td
&& td
!= LK_KERNTHREAD
) {
353 if ((extflags
& (LK_NOWAIT
| LK_CANRECURSE
)) == 0) {
354 spin_unlock_wr(&lkp
->lk_spinlock
);
355 panic("lockmgr: locking against myself");
357 if ((extflags
& LK_CANRECURSE
) != 0) {
358 lkp
->lk_exclusivecount
++;
364 * If we are just polling, check to see if we will sleep.
366 if ((extflags
& LK_NOWAIT
) &&
367 (lkp
->lk_flags
& (LK_HAVE_EXCL
| LK_WANT_EXCL
| LK_WANT_UPGRADE
| LK_SHARE_NONZERO
))) {
372 * Try to acquire the want_exclusive flag.
374 error
= acquire(lkp
, extflags
, (LK_HAVE_EXCL
| LK_WANT_EXCL
));
377 lkp
->lk_flags
|= LK_WANT_EXCL
;
379 * Wait for shared locks and upgrades to finish.
381 error
= acquire(lkp
, extflags
, LK_WANT_UPGRADE
| LK_SHARE_NONZERO
);
382 lkp
->lk_flags
&= ~LK_WANT_EXCL
;
385 lkp
->lk_flags
|= LK_HAVE_EXCL
;
386 lkp
->lk_lockholder
= td
;
387 if (lkp
->lk_exclusivecount
!= 0) {
388 spin_unlock_wr(&lkp
->lk_spinlock
);
389 panic("lockmgr: non-zero exclusive count");
391 lkp
->lk_exclusivecount
= 1;
392 #if defined(DEBUG_LOCKS)
393 lkp
->lk_filename
= file
;
394 lkp
->lk_lineno
= line
;
395 lkp
->lk_lockername
= name
;
401 if (lkp
->lk_exclusivecount
!= 0) {
402 if (lkp
->lk_lockholder
!= td
&&
403 lkp
->lk_lockholder
!= LK_KERNTHREAD
) {
404 spin_unlock_wr(&lkp
->lk_spinlock
);
405 panic("lockmgr: pid %d, not %s thr %p/%p unlocking",
406 (td
->td_proc
? td
->td_proc
->p_pid
: -1),
407 "exclusive lock holder",
408 td
, lkp
->lk_lockholder
);
410 if (lkp
->lk_lockholder
!= LK_KERNTHREAD
) {
413 if (lkp
->lk_exclusivecount
== 1) {
414 lkp
->lk_flags
&= ~LK_HAVE_EXCL
;
415 lkp
->lk_lockholder
= LK_NOTHREAD
;
416 lkp
->lk_exclusivecount
= 0;
418 lkp
->lk_exclusivecount
--;
420 } else if (lkp
->lk_flags
& LK_SHARE_NONZERO
) {
421 dowakeup
+= shareunlock(lkp
, 1);
424 if (lkp
->lk_flags
& LK_WAIT_NONZERO
)
429 spin_unlock_wr(&lkp
->lk_spinlock
);
430 panic("lockmgr: unknown locktype request %d",
431 flags
& LK_TYPE_MASK
);
434 spin_unlock_wr(&lkp
->lk_spinlock
);
441 lockmgr_kernproc(struct lock
*lp
)
443 struct thread
*td __debugvar
= curthread
;
445 if (lp
->lk_lockholder
!= LK_KERNTHREAD
) {
446 KASSERT(lp
->lk_lockholder
== td
,
447 ("lockmgr_kernproc: lock not owned by curthread %p", td
));
449 lp
->lk_lockholder
= LK_KERNTHREAD
;
454 * Set the lock to be exclusively held. The caller is holding the lock's
455 * spinlock and the spinlock remains held on return. A panic will occur
456 * if the lock cannot be set to exclusive.
459 lockmgr_setexclusive_interlocked(struct lock
*lkp
)
461 thread_t td
= curthread
;
463 KKASSERT((lkp
->lk_flags
& (LK_HAVE_EXCL
|LK_SHARE_NONZERO
)) == 0);
464 KKASSERT(lkp
->lk_exclusivecount
== 0);
465 lkp
->lk_flags
|= LK_HAVE_EXCL
;
466 lkp
->lk_lockholder
= td
;
467 lkp
->lk_exclusivecount
= 1;
472 * Clear the caller's exclusive lock. The caller is holding the lock's
473 * spinlock. THIS FUNCTION WILL UNLOCK THE SPINLOCK.
475 * A panic will occur if the caller does not hold the lock.
478 lockmgr_clrexclusive_interlocked(struct lock
*lkp
)
480 thread_t td __debugvar
= curthread
;
483 KKASSERT((lkp
->lk_flags
& LK_HAVE_EXCL
) && lkp
->lk_exclusivecount
== 1
484 && lkp
->lk_lockholder
== td
);
485 lkp
->lk_lockholder
= LK_NOTHREAD
;
486 lkp
->lk_flags
&= ~LK_HAVE_EXCL
;
487 lkp
->lk_exclusivecount
= 0;
488 if (lkp
->lk_flags
& LK_WAIT_NONZERO
)
491 spin_unlock_wr(&lkp
->lk_spinlock
);
497 * Initialize a lock; required before use.
500 lockinit(struct lock
*lkp
, char *wmesg
, int timo
, int flags
)
502 spin_init(&lkp
->lk_spinlock
);
503 lkp
->lk_flags
= (flags
& LK_EXTFLG_MASK
);
504 lkp
->lk_sharecount
= 0;
505 lkp
->lk_waitcount
= 0;
506 lkp
->lk_exclusivecount
= 0;
507 lkp
->lk_wmesg
= wmesg
;
509 lkp
->lk_lockholder
= LK_NOTHREAD
;
513 * Reinitialize a lock that is being reused for a different purpose, but
514 * which may have pending (blocked) threads sitting on it. The caller
515 * must already hold the interlock.
518 lockreinit(struct lock
*lkp
, char *wmesg
, int timo
, int flags
)
520 spin_lock_wr(&lkp
->lk_spinlock
);
521 lkp
->lk_flags
= (lkp
->lk_flags
& ~LK_EXTFLG_MASK
) |
522 (flags
& LK_EXTFLG_MASK
);
523 lkp
->lk_wmesg
= wmesg
;
525 spin_unlock_wr(&lkp
->lk_spinlock
);
529 * Requires that the caller is the exclusive owner of this lock.
532 lockuninit(struct lock
*l
)
535 * At this point we should have removed all the references to this lock
536 * so there can't be anyone waiting on it.
538 KKASSERT(l
->lk_waitcount
== 0);
540 spin_uninit(&l
->lk_spinlock
);
544 * Determine the status of a lock.
547 lockstatus(struct lock
*lkp
, struct thread
*td
)
551 spin_lock_wr(&lkp
->lk_spinlock
);
552 if (lkp
->lk_exclusivecount
!= 0) {
553 if (td
== NULL
|| lkp
->lk_lockholder
== td
)
554 lock_type
= LK_EXCLUSIVE
;
556 lock_type
= LK_EXCLOTHER
;
557 } else if (lkp
->lk_sharecount
!= 0) {
558 lock_type
= LK_SHARED
;
560 spin_unlock_wr(&lkp
->lk_spinlock
);
565 * Determine the number of holders of a lock.
567 * The non-blocking version can usually be used for assertions.
570 lockcount(struct lock
*lkp
)
574 spin_lock_wr(&lkp
->lk_spinlock
);
575 count
= lkp
->lk_exclusivecount
+ lkp
->lk_sharecount
;
576 spin_unlock_wr(&lkp
->lk_spinlock
);
581 lockcountnb(struct lock
*lkp
)
583 return (lkp
->lk_exclusivecount
+ lkp
->lk_sharecount
);
587 * Print out information about state of a lock. Used by VOP_PRINT
588 * routines to display status about contained locks.
591 lockmgr_printinfo(struct lock
*lkp
)
593 struct thread
*td
= lkp
->lk_lockholder
;
596 if (td
&& td
!= LK_KERNTHREAD
&& td
!= LK_NOTHREAD
)
601 if (lkp
->lk_sharecount
)
602 kprintf(" lock type %s: SHARED (count %d)", lkp
->lk_wmesg
,
604 else if (lkp
->lk_flags
& LK_HAVE_EXCL
)
605 kprintf(" lock type %s: EXCL (count %d) by td %p pid %d",
606 lkp
->lk_wmesg
, lkp
->lk_exclusivecount
, td
,
608 if (lkp
->lk_waitcount
> 0)
609 kprintf(" with %d pending", lkp
->lk_waitcount
);