3 * The Regents of the University of California. All rights reserved.
6 * John S. Dyson. All rights reserved.
8 * This code contains ideas from software contributed to Berkeley by
9 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
10 * System project at Carnegie-Mellon University.
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the University of
23 * California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95
41 * $FreeBSD: src/sys/kern/kern_lock.c,v 1.31.2.3 2001/12/25 01:44:44 dillon Exp $
42 * $DragonFly: src/sys/kern/kern_lock.c,v 1.19 2006/05/05 20:15:01 dillon Exp $
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
52 #include <sys/sysctl.h>
53 #include <sys/spinlock.h>
54 #include <sys/thread2.h>
55 #include <sys/spinlock2.h>
58 * 0: no warnings, 1: warnings, 2: panic
60 static int lockmgr_from_int
= 1;
61 SYSCTL_INT(_debug
, OID_AUTO
, lockmgr_from_int
, CTLFLAG_RW
, &lockmgr_from_int
, 0, "");
64 * Locking primitives implementation.
65 * Locks provide shared/exclusive sychronization.
68 #ifdef SIMPLELOCK_DEBUG
69 #define COUNT(td, x) (td)->td_locks += (x)
74 #define LOCK_WAIT_TIME 100
75 #define LOCK_SAMPLE_WAIT 7
77 #if defined(DIAGNOSTIC)
80 #define LOCK_INLINE __inline
83 #define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \
84 LK_SHARE_NONZERO | LK_WAIT_NONZERO)
86 static int acquire(struct lock
*lkp
, int extflags
, int wanted
);
88 static LOCK_INLINE
void
89 sharelock(struct lock
*lkp
, int incr
) {
90 lkp
->lk_flags
|= LK_SHARE_NONZERO
;
91 lkp
->lk_sharecount
+= incr
;
94 static LOCK_INLINE
void
95 shareunlock(struct lock
*lkp
, int decr
) {
97 KASSERT(lkp
->lk_sharecount
>= decr
, ("shareunlock: count < decr"));
99 if (lkp
->lk_sharecount
== decr
) {
100 lkp
->lk_flags
&= ~LK_SHARE_NONZERO
;
101 if (lkp
->lk_flags
& (LK_WANT_UPGRADE
| LK_WANT_EXCL
)) {
104 lkp
->lk_sharecount
= 0;
106 lkp
->lk_sharecount
-= decr
;
111 * lock acquisition helper routine. Called with the lock's spinlock held.
114 acquire(struct lock
*lkp
, int extflags
, int wanted
)
118 if ((extflags
& LK_NOWAIT
) && (lkp
->lk_flags
& wanted
)) {
122 if (((lkp
->lk_flags
| extflags
) & LK_NOPAUSE
) == 0) {
123 if ((lkp
->lk_flags
& wanted
) == 0)
127 while ((lkp
->lk_flags
& wanted
) != 0) {
128 lkp
->lk_flags
|= LK_WAIT_NONZERO
;
132 * Use the _quick version so the critical section is left
133 * intact, protecting the tsleep interlock. See
134 * tsleep_interlock() for a description of what is
137 tsleep_interlock(lkp
);
138 spin_unlock_quick(&lkp
->lk_spinlock
);
140 ((extflags
& LK_PCATCH
) ? PCATCH
: 0),
142 ((extflags
& LK_TIMELOCK
) ? lkp
->lk_timo
: 0));
143 spin_lock_quick(&lkp
->lk_spinlock
);
144 if (lkp
->lk_waitcount
== 1) {
145 lkp
->lk_flags
&= ~LK_WAIT_NONZERO
;
146 lkp
->lk_waitcount
= 0;
152 if (extflags
& LK_SLEEPFAIL
)
159 * Set, change, or release a lock.
161 * Shared requests increment the shared count. Exclusive requests set the
162 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already
163 * accepted shared locks and shared-to-exclusive upgrades to go away.
165 * A spinlock is held for most of the procedure. We must not do anything
166 * fancy while holding the spinlock.
170 lockmgr(struct lock
*lkp
, u_int flags
)
172 debuglockmgr(struct lock
*lkp
, u_int flags
,
173 const char *name
, const char *file
, int line
)
183 if (lockmgr_from_int
&& mycpu
->gd_intr_nesting_level
&&
184 (flags
& LK_NOWAIT
) == 0 &&
185 (flags
& LK_TYPE_MASK
) != LK_RELEASE
&& didpanic
== 0) {
187 if (lockmgr_from_int
== 2) {
190 "lockmgr %s from %p: called from interrupt",
191 lkp
->lk_wmesg
, ((int **)&lkp
)[-1]);
195 "lockmgr %s from %p: called from interrupt\n",
196 lkp
->lk_wmesg
, ((int **)&lkp
)[-1]);
199 if (lockmgr_from_int
== 2) {
202 "lockmgr %s from %s:%d: called from interrupt",
203 lkp
->lk_wmesg
, file
, line
);
207 "lockmgr %s from %s:%d: called from interrupt\n",
208 lkp
->lk_wmesg
, file
, line
);
213 spin_lock(&lkp
->lk_spinlock
);
215 extflags
= (flags
| lkp
->lk_flags
) & LK_EXTFLG_MASK
;
218 switch (flags
& LK_TYPE_MASK
) {
221 * If we are not the exclusive lock holder, we have to block
222 * while there is an exclusive lock holder or while an
223 * exclusive lock request or upgrade request is in progress.
225 * However, if P_DEADLKTREAT is set, we override exclusive
226 * lock requests or upgrade requests ( but not the exclusive
229 if (lkp
->lk_lockholder
!= td
) {
230 if (td
->td_flags
& TDF_DEADLKTREAT
) {
240 LK_HAVE_EXCL
| LK_WANT_EXCL
|
251 * We hold an exclusive lock, so downgrade it to shared.
252 * An alternative would be to fail with EDEADLK.
256 /* fall into downgrade */
259 if (lkp
->lk_lockholder
!= td
|| lkp
->lk_exclusivecount
== 0) {
260 spin_unlock(&lkp
->lk_spinlock
);
261 panic("lockmgr: not holding exclusive lock");
263 sharelock(lkp
, lkp
->lk_exclusivecount
);
264 lkp
->lk_exclusivecount
= 0;
265 lkp
->lk_flags
&= ~LK_HAVE_EXCL
;
266 lkp
->lk_lockholder
= LK_NOTHREAD
;
267 if (lkp
->lk_waitcount
)
273 * If another process is ahead of us to get an upgrade,
274 * then we want to fail rather than have an intervening
277 if (lkp
->lk_flags
& LK_WANT_UPGRADE
) {
283 /* fall into normal upgrade */
287 * Upgrade a shared lock to an exclusive one. If another
288 * shared lock has already requested an upgrade to an
289 * exclusive lock, our shared lock is released and an
290 * exclusive lock is requested (which will be granted
291 * after the upgrade). If we return an error, the file
292 * will always be unlocked.
294 if ((lkp
->lk_lockholder
== td
) || (lkp
->lk_sharecount
<= 0)) {
295 spin_unlock(&lkp
->lk_spinlock
);
296 panic("lockmgr: upgrade exclusive lock");
301 * If we are just polling, check to see if we will block.
303 if ((extflags
& LK_NOWAIT
) &&
304 ((lkp
->lk_flags
& LK_WANT_UPGRADE
) ||
305 lkp
->lk_sharecount
> 1)) {
309 if ((lkp
->lk_flags
& LK_WANT_UPGRADE
) == 0) {
311 * We are first shared lock to request an upgrade, so
312 * request upgrade and wait for the shared count to
313 * drop to zero, then take exclusive lock.
315 lkp
->lk_flags
|= LK_WANT_UPGRADE
;
316 error
= acquire(lkp
, extflags
, LK_SHARE_NONZERO
);
317 lkp
->lk_flags
&= ~LK_WANT_UPGRADE
;
321 lkp
->lk_flags
|= LK_HAVE_EXCL
;
322 lkp
->lk_lockholder
= td
;
323 if (lkp
->lk_exclusivecount
!= 0) {
324 spin_unlock(&lkp
->lk_spinlock
);
325 panic("lockmgr: non-zero exclusive count");
327 lkp
->lk_exclusivecount
= 1;
328 #if defined(DEBUG_LOCKS)
329 lkp
->lk_filename
= file
;
330 lkp
->lk_lineno
= line
;
331 lkp
->lk_lockername
= name
;
337 * Someone else has requested upgrade. Release our shared
338 * lock, awaken upgrade requestor if we are the last shared
339 * lock, then request an exclusive lock.
341 if ( (lkp
->lk_flags
& (LK_SHARE_NONZERO
|LK_WAIT_NONZERO
)) ==
344 /* fall into exclusive request */
347 if (lkp
->lk_lockholder
== td
&& td
!= LK_KERNTHREAD
) {
351 if ((extflags
& (LK_NOWAIT
| LK_CANRECURSE
)) == 0) {
352 spin_unlock(&lkp
->lk_spinlock
);
353 panic("lockmgr: locking against myself");
355 if ((extflags
& LK_CANRECURSE
) != 0) {
356 lkp
->lk_exclusivecount
++;
362 * If we are just polling, check to see if we will sleep.
364 if ((extflags
& LK_NOWAIT
) &&
365 (lkp
->lk_flags
& (LK_HAVE_EXCL
| LK_WANT_EXCL
| LK_WANT_UPGRADE
| LK_SHARE_NONZERO
))) {
370 * Try to acquire the want_exclusive flag.
372 error
= acquire(lkp
, extflags
, (LK_HAVE_EXCL
| LK_WANT_EXCL
));
375 lkp
->lk_flags
|= LK_WANT_EXCL
;
377 * Wait for shared locks and upgrades to finish.
379 error
= acquire(lkp
, extflags
, LK_WANT_UPGRADE
| LK_SHARE_NONZERO
);
380 lkp
->lk_flags
&= ~LK_WANT_EXCL
;
383 lkp
->lk_flags
|= LK_HAVE_EXCL
;
384 lkp
->lk_lockholder
= td
;
385 if (lkp
->lk_exclusivecount
!= 0) {
386 spin_unlock(&lkp
->lk_spinlock
);
387 panic("lockmgr: non-zero exclusive count");
389 lkp
->lk_exclusivecount
= 1;
390 #if defined(DEBUG_LOCKS)
391 lkp
->lk_filename
= file
;
392 lkp
->lk_lineno
= line
;
393 lkp
->lk_lockername
= name
;
399 if (lkp
->lk_exclusivecount
!= 0) {
400 if (lkp
->lk_lockholder
!= td
&&
401 lkp
->lk_lockholder
!= LK_KERNTHREAD
) {
402 spin_unlock(&lkp
->lk_spinlock
);
403 panic("lockmgr: pid %d, not %s thr %p unlocking",
404 (td
->td_proc
? td
->td_proc
->p_pid
: -99),
405 "exclusive lock holder",
408 if (lkp
->lk_lockholder
!= LK_KERNTHREAD
) {
411 if (lkp
->lk_exclusivecount
== 1) {
412 lkp
->lk_flags
&= ~LK_HAVE_EXCL
;
413 lkp
->lk_lockholder
= LK_NOTHREAD
;
414 lkp
->lk_exclusivecount
= 0;
416 lkp
->lk_exclusivecount
--;
418 } else if (lkp
->lk_flags
& LK_SHARE_NONZERO
) {
422 if (lkp
->lk_flags
& LK_WAIT_NONZERO
)
427 spin_unlock(&lkp
->lk_spinlock
);
428 panic("lockmgr: unknown locktype request %d",
429 flags
& LK_TYPE_MASK
);
432 spin_unlock(&lkp
->lk_spinlock
);
437 lockmgr_kernproc(struct lock
*lp
)
439 struct thread
*td
= curthread
;
441 KASSERT(lp
->lk_lockholder
== td
, ("lockmgr_kernproc: lock not owned by curthread %p", td
));
443 lp
->lk_lockholder
= LK_KERNTHREAD
;
447 * Initialize a lock; required before use.
450 lockinit(struct lock
*lkp
, char *wmesg
, int timo
, int flags
)
452 spin_init(&lkp
->lk_spinlock
);
453 lkp
->lk_flags
= (flags
& LK_EXTFLG_MASK
);
454 lkp
->lk_sharecount
= 0;
455 lkp
->lk_waitcount
= 0;
456 lkp
->lk_exclusivecount
= 0;
457 lkp
->lk_wmesg
= wmesg
;
459 lkp
->lk_lockholder
= LK_NOTHREAD
;
463 * Reinitialize a lock that is being reused for a different purpose, but
464 * which may have pending (blocked) threads sitting on it. The caller
465 * must already hold the interlock.
468 lockreinit(struct lock
*lkp
, char *wmesg
, int timo
, int flags
)
470 lkp
->lk_flags
= (lkp
->lk_flags
& ~LK_EXTFLG_MASK
) |
471 (flags
& LK_EXTFLG_MASK
);
472 lkp
->lk_wmesg
= wmesg
;
477 * Determine the status of a lock.
480 lockstatus(struct lock
*lkp
, struct thread
*td
)
484 spin_lock(&lkp
->lk_spinlock
);
485 if (lkp
->lk_exclusivecount
!= 0) {
486 if (td
== NULL
|| lkp
->lk_lockholder
== td
)
487 lock_type
= LK_EXCLUSIVE
;
489 lock_type
= LK_EXCLOTHER
;
490 } else if (lkp
->lk_sharecount
!= 0) {
491 lock_type
= LK_SHARED
;
493 spin_unlock(&lkp
->lk_spinlock
);
498 * Determine the number of holders of a lock.
500 * The non-blocking version can usually be used for assertions.
503 lockcount(struct lock
*lkp
)
507 spin_lock(&lkp
->lk_spinlock
);
508 count
= lkp
->lk_exclusivecount
+ lkp
->lk_sharecount
;
509 spin_unlock(&lkp
->lk_spinlock
);
514 lockcountnb(struct lock
*lkp
)
516 return (lkp
->lk_exclusivecount
+ lkp
->lk_sharecount
);
520 * Print out information about state of a lock. Used by VOP_PRINT
521 * routines to display status about contained locks.
524 lockmgr_printinfo(struct lock
*lkp
)
526 struct thread
*td
= lkp
->lk_lockholder
;
529 if (td
&& td
!= LK_KERNTHREAD
&& td
!= LK_NOTHREAD
)
534 if (lkp
->lk_sharecount
)
535 printf(" lock type %s: SHARED (count %d)", lkp
->lk_wmesg
,
537 else if (lkp
->lk_flags
& LK_HAVE_EXCL
)
538 printf(" lock type %s: EXCL (count %d) by td %p pid %d",
539 lkp
->lk_wmesg
, lkp
->lk_exclusivecount
, td
,
541 if (lkp
->lk_waitcount
> 0)
542 printf(" with %d pending", lkp
->lk_waitcount
);