include/asm-i386/mutex.h

   1 /*
   2  * Assembly implementation of the mutex fastpath, based on atomic
   3  * decrement/increment.
   4  *
   5  * started by Ingo Molnar:
   6  *
   7  *  Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
   8  */
   9 #ifndef _ASM_MUTEX_H
  10 #define _ASM_MUTEX_H
  11
  12 #include "asm/alternative.h"
  13
  14 /**
  15  *  __mutex_fastpath_lock - try to take the lock by moving the count
  16  *                          from 1 to a 0 value
  17  *  @count: pointer of type atomic_t
  18  *  @fn: function to call if the original value was not 1
  19  *
  20  * Change the count from 1 to a value lower than 1, and call <fn> if it
  21  * wasn't 1 originally. This function MUST leave the value lower than 1
  22  * even when the "1" assertion wasn't true.
  23  */
  24 #define __mutex_fastpath_lock(count, fail_fn)                           \
  25 do {                                                                    \
  26         unsigned int dummy;                                             \
  27                                                                         \
  28         typecheck(atomic_t *, count);                                   \
  29         typecheck_fn(fastcall void (*)(atomic_t *), fail_fn);           \
  30                                                                         \
  31         __asm__ __volatile__(                                           \
  32                 LOCK_PREFIX "   decl (%%eax)    \n"                     \
  33                         "   jns 1f              \n"                     \
  34                         "   call "#fail_fn"     \n"                     \
  35                         "1:                     \n"                     \
  36                                                                         \
  37                 :"=a" (dummy)                                           \
  38                 : "a" (count)                                           \
  39                 : "memory", "ecx", "edx");                              \
  40 } while (0)
  41
  42
  43 /**
  44  *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
  45  *                                 from 1 to a 0 value
  46  *  @count: pointer of type atomic_t
  47  *  @fail_fn: function to call if the original value was not 1
  48  *
  49  * Change the count from 1 to a value lower than 1, and call <fail_fn> if it
  50  * wasn't 1 originally. This function returns 0 if the fastpath succeeds,
  51  * or anything the slow path function returns
  52  */
  53 static inline int
  54 __mutex_fastpath_lock_retval(atomic_t *count,
  55                              int fastcall (*fail_fn)(atomic_t *))
  56 {
  57         if (unlikely(atomic_dec_return(count) < 0))
  58                 return fail_fn(count);
  59         else
  60                 return 0;
  61 }
  62
  63 /**
  64  *  __mutex_fastpath_unlock - try to promote the mutex from 0 to 1
  65  *  @count: pointer of type atomic_t
  66  *  @fail_fn: function to call if the original value was not 0
  67  *
  68  * try to promote the mutex from 0 to 1. if it wasn't 0, call <fail_fn>.
  69  * In the failure case, this function is allowed to either set the value
  70  * to 1, or to set it to a value lower than 1.
  71  *
  72  * If the implementation sets it to a value of lower than 1, the
  73  * __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs
  74  * to return 0 otherwise.
  75  */
  76 #define __mutex_fastpath_unlock(count, fail_fn)                         \
  77 do {                                                                    \
  78         unsigned int dummy;                                             \
  79                                                                         \
  80         typecheck(atomic_t *, count);                                   \
  81         typecheck_fn(fastcall void (*)(atomic_t *), fail_fn);           \
  82                                                                         \
  83         __asm__ __volatile__(                                           \
  84                 LOCK_PREFIX "   incl (%%eax)    \n"                     \
  85                         "   jg  1f              \n"                     \
  86                         "   call "#fail_fn"     \n"                     \
  87                         "1:                     \n"                     \
  88                                                                         \
  89                 :"=a" (dummy)                                           \
  90                 : "a" (count)                                           \
  91                 : "memory", "ecx", "edx");                              \
  92 } while (0)
  93
  94 #define __mutex_slowpath_needs_to_unlock()      1
  95
  96 /**
  97  * __mutex_fastpath_trylock - try to acquire the mutex, without waiting
  98  *
  99  *  @count: pointer of type atomic_t
 100  *  @fail_fn: fallback function
 101  *
 102  * Change the count from 1 to a value lower than 1, and return 0 (failure)
 103  * if it wasn't 1 originally, or return 1 (success) otherwise. This function
 104  * MUST leave the value lower than 1 even when the "1" assertion wasn't true.
 105  * Additionally, if the value was < 0 originally, this function must not leave
 106  * it to 0 on failure.
 107  */
 108 static inline int
 109 __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
 110 {
 111         /*
 112          * We have two variants here. The cmpxchg based one is the best one
 113          * because it never induce a false contention state.  It is included
 114          * here because architectures using the inc/dec algorithms over the
 115          * xchg ones are much more likely to support cmpxchg natively.
 116          *
 117          * If not we fall back to the spinlock based variant - that is
 118          * just as efficient (and simpler) as a 'destructive' probing of
 119          * the mutex state would be.
 120          */
 121 #ifdef __HAVE_ARCH_CMPXCHG
 122         if (likely(atomic_cmpxchg(count, 1, 0) == 1))
 123                 return 1;
 124         return 0;
 125 #else
 126         return fail_fn(count);
 127 #endif
 128 }
 129
 130 #endif