sys/sys/mutex2.h

   1 /*
   2  * Copyright (c) 2009 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #ifndef _SYS_MUTEX2_H_
  36 #define _SYS_MUTEX2_H_
  37
  38 #ifndef _SYS_MUTEX_H_
  39 #include <sys/mutex.h>
  40 #endif
  41 #ifndef _SYS_THREAD2_H_
  42 #include <sys/thread2.h>
  43 #endif
  44 #ifndef _SYS_GLOBALDATA_H_
  45 #include <sys/globaldata.h>
  46 #endif
  47 #include <machine/atomic.h>
  48
  49 /*
  50  * Initialize a new mutex, placing it in an unlocked state with no refs.
  51  */
  52 static __inline void
  53 mtx_init(mtx_t *mtx, const char *ident)
  54 {
  55         mtx->mtx_lock = 0;
  56         mtx->mtx_owner = NULL;
  57         mtx->mtx_exlink = NULL;
  58         mtx->mtx_shlink = NULL;
  59         mtx->mtx_ident = ident;
  60 }
  61
  62 /*
  63  * Initialize a mtx link structure for deeper control over the mutex
  64  * operation.
  65  */
  66 static __inline void
  67 mtx_link_init(mtx_link_t *link)
  68 {
  69         link->state = MTX_LINK_IDLE;
  70         link->callback = NULL;
  71         link->arg = NULL;
  72 }
  73
  74 /*
  75  * A link structure initialized this way causes mutex operations to not block,
  76  * caller must specify a callback.  Caller may still abort the mutex via
  77  * the link.
  78  */
  79 static __inline void
  80 mtx_link_init_async(mtx_link_t *link,
  81                     void (*callback)(mtx_link_t *link, void *arg, int error),
  82                     void *arg)
  83 {
  84         link->state = MTX_LINK_IDLE;
  85         link->callback = callback;
  86         link->arg = arg;
  87 }
  88
  89 /*
  90  * Deinitialize a mutex
  91  */
  92 static __inline void
  93 mtx_uninit(mtx_t *mtx)
  94 {
  95         /* empty */
  96 }
  97
  98 /*
  99  * Exclusive-lock a mutex, block until acquired or aborted.  Recursion
 100  * is allowed.
 101  *
 102  * This version of the function allows the mtx_link to be passed in, thus
 103  * giving the caller visibility for the link structure which is required
 104  * when calling mtx_abort_ex_link() or when requesting an asynchronous lock.
 105  *
 106  * The mutex may be aborted at any time while the passed link structure
 107  * is valid.
 108  */
 109 static __inline int
 110 mtx_lock_ex_link(mtx_t *mtx, mtx_link_t *link, int flags, int to)
 111 {
 112         if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
 113                 return(_mtx_lock_ex_link(mtx, link, flags, to));
 114         mtx->mtx_owner = curthread;
 115         link->state = MTX_LINK_ACQUIRED;
 116
 117         return(0);
 118 }
 119
 120 /*
 121  * Short-form exclusive-lock a mutex, block until acquired.  Recursion is
 122  * allowed.  This is equivalent to mtx_lock_ex(mtx, "mtxex", 0, 0).
 123  */
 124 static __inline void
 125 mtx_lock(mtx_t *mtx)
 126 {
 127         if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) {
 128                 _mtx_lock_ex(mtx, 0, 0);
 129                 return;
 130         }
 131         mtx->mtx_owner = curthread;
 132 }
 133
 134 /*
 135  * Exclusive-lock a mutex, block until acquired.  Recursion is allowed.
 136  *
 137  * Returns 0 on success, or the tsleep() return code on failure.
 138  * An error can only be returned if PCATCH is specified in the flags.
 139  */
 140 static __inline int
 141 mtx_lock_ex(mtx_t *mtx, int flags, int to)
 142 {
 143         if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
 144                 return(_mtx_lock_ex(mtx, flags, to));
 145         mtx->mtx_owner = curthread;
 146         return(0);
 147 }
 148
 149 static __inline int
 150 mtx_lock_ex_quick(mtx_t *mtx)
 151 {
 152         if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
 153                 return(_mtx_lock_ex_quick(mtx));
 154         mtx->mtx_owner = curthread;
 155         return(0);
 156 }
 157
 158 static __inline int
 159 mtx_lock_sh_link(mtx_t *mtx, mtx_link_t *link, int flags, int to)
 160 {
 161         if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0)
 162                 return(_mtx_lock_sh_link(mtx, link, flags, to));
 163         link->state = MTX_LINK_ACQUIRED;
 164         return(0);
 165 }
 166
 167 /*
 168  * Share-lock a mutex, block until acquired.  Recursion is allowed.
 169  *
 170  * Returns 0 on success, or the tsleep() return code on failure.
 171  * An error can only be returned if PCATCH is specified in the flags.
 172  */
 173 static __inline int
 174 mtx_lock_sh(mtx_t *mtx, int flags, int to)
 175 {
 176         if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0)
 177                 return(_mtx_lock_sh(mtx, flags, to));
 178         return(0);
 179 }
 180
 181 static __inline int
 182 mtx_lock_sh_quick(mtx_t *mtx)
 183 {
 184         if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0)
 185                 return(_mtx_lock_sh_quick(mtx));
 186         return(0);
 187 }
 188
 189 /*
 190  * Adds a shared lock reference to a lock already locked shared,
 191  * does not block on pending exclusive request.
 192  */
 193 static __inline void
 194 mtx_lock_sh_again(mtx_t *mtx)
 195 {
 196         KKASSERT((mtx->mtx_lock & MTX_EXCLUSIVE) == 0 &&
 197                  (mtx->mtx_lock & MTX_MASK) > 0);
 198         atomic_add_int(&mtx->mtx_lock, 1);
 199 }
 200
 201 /*
 202  * Short-form exclusive spinlock a mutex.  Must be paired with
 203  * mtx_spinunlock().
 204  */
 205 static __inline void
 206 mtx_spinlock(mtx_t *mtx)
 207 {
 208         globaldata_t gd = mycpu;
 209
 210         /*
 211          * Predispose a hard critical section
 212          */
 213         ++gd->gd_curthread->td_critcount;
 214         cpu_ccfence();
 215         ++gd->gd_spinlocks;
 216
 217         /*
 218          * If we cannot get it trivially get it the hard way.
 219          *
 220          * Note that mtx_owner will be set twice if we fail to get it
 221          * trivially, but there's no point conditionalizing it as a
 222          * conditional will be slower.
 223          */
 224         if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
 225                 _mtx_spinlock(mtx);
 226         mtx->mtx_owner = gd->gd_curthread;
 227 }
 228
 229 static __inline int
 230 mtx_spinlock_try(mtx_t *mtx)
 231 {
 232         globaldata_t gd = mycpu;
 233
 234         /*
 235          * Predispose a hard critical section
 236          */
 237         ++gd->gd_curthread->td_critcount;
 238         cpu_ccfence();
 239         ++gd->gd_spinlocks;
 240
 241         /*
 242          * If we cannot get it trivially call _mtx_spinlock_try().  This
 243          * function will clean up the hard critical section if it fails.
 244          */
 245         if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
 246                 return(_mtx_spinlock_try(mtx));
 247         mtx->mtx_owner = gd->gd_curthread;
 248         return (0);
 249 }
 250
 251 /*
 252  * Short-form exclusive-lock a mutex, spin until acquired.  Recursion is
 253  * allowed.  This form is identical to mtx_spinlock_ex().
 254  *
 255  * Attempt to exclusive-lock a mutex, return 0 on success and
 256  * EAGAIN on failure.
 257  */
 258 static __inline int
 259 mtx_lock_ex_try(mtx_t *mtx)
 260 {
 261         if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
 262                 return (_mtx_lock_ex_try(mtx));
 263         mtx->mtx_owner = curthread;
 264         return (0);
 265 }
 266
 267 /*
 268  * Attempt to share-lock a mutex, return 0 on success and
 269  * EAGAIN on failure.
 270  */
 271 static __inline int
 272 mtx_lock_sh_try(mtx_t *mtx)
 273 {
 274         if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0)
 275                 return (_mtx_lock_sh_try(mtx));
 276         return (0);
 277 }
 278
 279 /*
 280  * If the lock is held exclusively it must be owned by the caller.  If the
 281  * lock is already a shared lock this operation is a NOP.    A panic will
 282  * occur if the lock is not held either shared or exclusive.
 283  *
 284  * The exclusive count is converted to a shared count.
 285  */
 286 static __inline void
 287 mtx_downgrade(mtx_t *mtx)
 288 {
 289         mtx->mtx_owner = NULL;
 290         if (atomic_cmpset_int(&mtx->mtx_lock, MTX_EXCLUSIVE | 1, 1) == 0)
 291                 _mtx_downgrade(mtx);
 292 }
 293
 294 /*
 295  * Upgrade a shared lock to an exclusive lock.  The upgrade will fail if
 296  * the shared lock has a count other then 1.  Optimize the most likely case
 297  * but note that a single cmpset can fail due to WANTED races.
 298  *
 299  * If the lock is held exclusively it must be owned by the caller and
 300  * this function will simply return without doing anything.  A panic will
 301  * occur if the lock is held exclusively by someone other then the caller.
 302  *
 303  * Returns 0 on success, EDEADLK on failure.
 304  */
 305 static __inline int
 306 mtx_upgrade_try(mtx_t *mtx)
 307 {
 308         if (atomic_cmpset_int(&mtx->mtx_lock, 1, MTX_EXCLUSIVE | 1)) {
 309                 mtx->mtx_owner = curthread;
 310                 return(0);
 311         }
 312         return (_mtx_upgrade_try(mtx));
 313 }
 314
 315 /*
 316  * Optimized unlock cases.
 317  *
 318  * NOTE: mtx_unlock() handles any type of mutex: exclusive, shared, and
 319  *       both blocking and spin methods.
 320  *
 321  *       The mtx_unlock_ex/sh() forms are optimized for exclusive or shared
 322  *       mutexes and produce less code, but it is ok for code to just use
 323  *       mtx_unlock() and, in fact, if code uses the short-form mtx_lock()
 324  *       or mtx_spinlock() to lock it should also use mtx_unlock() to unlock.
 325  */
 326 static __inline void
 327 mtx_unlock(mtx_t *mtx)
 328 {
 329         u_int lock = mtx->mtx_lock;
 330
 331         if (lock == (MTX_EXCLUSIVE | 1)) {
 332                 mtx->mtx_owner = NULL;
 333                 if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0)
 334                         _mtx_unlock(mtx);
 335         } else if (lock == 1) {
 336                 if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0)
 337                         _mtx_unlock(mtx);
 338         } else {
 339                 _mtx_unlock(mtx);
 340         }
 341 }
 342
 343 static __inline void
 344 mtx_unlock_ex(mtx_t *mtx)
 345 {
 346         u_int lock = mtx->mtx_lock;
 347
 348         if (lock == (MTX_EXCLUSIVE | 1)) {
 349                 mtx->mtx_owner = NULL;
 350                 if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0)
 351                         _mtx_unlock(mtx);
 352         } else {
 353                 _mtx_unlock(mtx);
 354         }
 355 }
 356
 357 static __inline void
 358 mtx_unlock_sh(mtx_t *mtx)
 359 {
 360         if (atomic_cmpset_int(&mtx->mtx_lock, 1, 0) == 0)
 361                 _mtx_unlock(mtx);
 362 }
 363
 364 /*
 365  * NOTE: spinlocks are exclusive-only
 366  */
 367 static __inline void
 368 mtx_spinunlock(mtx_t *mtx)
 369 {
 370         globaldata_t gd = mycpu;
 371
 372         mtx_unlock(mtx);
 373
 374         --gd->gd_spinlocks;
 375         cpu_ccfence();
 376         --gd->gd_curthread->td_critcount;
 377 }
 378
 379 /*
 380  * Return TRUE (non-zero) if the mutex is locked shared or exclusive by
 381  * anyone, including the owner.
 382  */
 383 static __inline int
 384 mtx_islocked(mtx_t *mtx)
 385 {
 386         return(mtx->mtx_lock != 0);
 387 }
 388
 389 /*
 390  * Return TRUE (non-zero) if the mutex is locked exclusively by anyone,
 391  * including the owner.  Returns FALSE (0) if the mutex is unlocked or
 392  * if it is locked shared by one or more entities.
 393  *
 394  * A caller wishing to check whether a lock is owned exclusively by it
 395  * should use mtx_owned().
 396  */
 397 static __inline int
 398 mtx_islocked_ex(mtx_t *mtx)
 399 {
 400         return((mtx->mtx_lock & MTX_EXCLUSIVE) != 0);
 401 }
 402
 403 /*
 404  * Return TRUE (non-zero) if the mutex is not locked.
 405  */
 406 static __inline int
 407 mtx_notlocked(mtx_t *mtx)
 408 {
 409         return(mtx->mtx_lock == 0);
 410 }
 411
 412 /*
 413  * Return TRUE (non-zero) if the mutex is not locked exclusively.
 414  * The mutex may in an unlocked or shared lock state.
 415  */
 416 static __inline int
 417 mtx_notlocked_ex(mtx_t *mtx)
 418 {
 419         return((mtx->mtx_lock & MTX_EXCLUSIVE) != 0);
 420 }
 421
 422 /*
 423  * Return TRUE (non-zero) if the mutex is exclusively locked by
 424  * the caller.
 425  */
 426 static __inline int
 427 mtx_owned(mtx_t *mtx)
 428 {
 429         return((mtx->mtx_lock & MTX_EXCLUSIVE) && mtx->mtx_owner == curthread);
 430 }
 431
 432 /*
 433  * Return TRUE (non-zero) if the mutex is not exclusively locked by
 434  * the caller.
 435  */
 436 static __inline int
 437 mtx_notowned(mtx_t *mtx)
 438 {
 439         return((mtx->mtx_lock & MTX_EXCLUSIVE) == 0 ||
 440                mtx->mtx_owner != curthread);
 441 }
 442
 443 /*
 444  * Return the shared or exclusive lock count.  A return value of 0
 445  * indicate that the mutex is not locked.
 446  *
 447  * NOTE: If the mutex is held exclusively by someone other then the
 448  *       caller the lock count for the other owner is still returned.
 449  */
 450 static __inline
 451 int
 452 mtx_lockrefs(mtx_t *mtx)
 453 {
 454         return(mtx->mtx_lock & MTX_MASK);
 455 }
 456
 457 /*
 458  * Lock must held and will be released on return.  Returns state
 459  * which can be passed to mtx_lock_temp_restore() to return the
 460  * lock to its previous state.
 461  */
 462 static __inline
 463 mtx_state_t
 464 mtx_lock_temp_release(mtx_t *mtx)
 465 {
 466         mtx_state_t state;
 467
 468         state = (mtx->mtx_lock & MTX_EXCLUSIVE);
 469         mtx_unlock(mtx);
 470
 471         return state;
 472 }
 473
 474 /*
 475  * Restore the previous state of a lock released with
 476  * mtx_lock_temp_release() or mtx_lock_upgrade().
 477  */
 478 static __inline
 479 void
 480 mtx_lock_temp_restore(mtx_t *mtx, mtx_state_t state)
 481 {
 482         if (state & MTX_EXCLUSIVE)
 483                 mtx_lock_ex_quick(mtx);
 484         else
 485                 mtx_lock_sh_quick(mtx);
 486 }
 487
 488 #endif