include/asm-arm/mutex.h

   1 /*
   2  * include/asm-arm/mutex.h
   3  *
   4  * ARM optimized mutex locking primitives
   5  *
   6  * Please look into asm-generic/mutex-xchg.h for a formal definition.
   7  */
   8 #ifndef _ASM_MUTEX_H
   9 #define _ASM_MUTEX_H
  10
  11 #if __LINUX_ARM_ARCH__ < 6
  12 /* On pre-ARMv6 hardware the swp based implementation is the most efficient. */
  13 # include <asm-generic/mutex-xchg.h>
  14 #else
  15
  16 /*
  17  * Attempting to lock a mutex on ARMv6+ can be done with a bastardized
  18  * atomic decrement (it is not a reliable atomic decrement but it satisfies
  19  * the defined semantics for our purpose, while being smaller and faster
  20  * than a real atomic decrement or atomic swap.  The idea is to attempt
  21  * decrementing the lock value only once.  If once decremented it isn't zero,
  22  * or if its store-back fails due to a dispute on the exclusive store, we
  23  * simply bail out immediately through the slow path where the lock will be
  24  * reattempted until it succeeds.
  25  */
  26 static inline void
  27 __mutex_fastpath_lock(atomic_t *count, fastcall void (*fail_fn)(atomic_t *))
  28 {
  29         int __ex_flag, __res;
  30
  31         __asm__ (
  32
  33                 "ldrex  %0, [%2]        \n\t"
  34                 "sub    %0, %0, #1      \n\t"
  35                 "strex  %1, %0, [%2]    "
  36
  37                 : "=&r" (__res), "=&r" (__ex_flag)
  38                 : "r" (&(count)->counter)
  39                 : "cc","memory" );
  40
  41         __res |= __ex_flag;
  42         if (unlikely(__res != 0))
  43                 fail_fn(count);
  44 }
  45
  46 static inline int
  47 __mutex_fastpath_lock_retval(atomic_t *count, fastcall int (*fail_fn)(atomic_t *))
  48 {
  49         int __ex_flag, __res;
  50
  51         __asm__ (
  52
  53                 "ldrex  %0, [%2]        \n\t"
  54                 "sub    %0, %0, #1      \n\t"
  55                 "strex  %1, %0, [%2]    "
  56
  57                 : "=&r" (__res), "=&r" (__ex_flag)
  58                 : "r" (&(count)->counter)
  59                 : "cc","memory" );
  60
  61         __res |= __ex_flag;
  62         if (unlikely(__res != 0))
  63                 __res = fail_fn(count);
  64         return __res;
  65 }
  66
  67 /*
  68  * Same trick is used for the unlock fast path. However the original value,
  69  * rather than the result, is used to test for success in order to have
  70  * better generated assembly.
  71  */
  72 static inline void
  73 __mutex_fastpath_unlock(atomic_t *count, fastcall void (*fail_fn)(atomic_t *))
  74 {
  75         int __ex_flag, __res, __orig;
  76
  77         __asm__ (
  78
  79                 "ldrex  %0, [%3]        \n\t"
  80                 "add    %1, %0, #1      \n\t"
  81                 "strex  %2, %1, [%3]    "
  82
  83                 : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
  84                 : "r" (&(count)->counter)
  85                 : "cc","memory" );
  86
  87         __orig |= __ex_flag;
  88         if (unlikely(__orig != 0))
  89                 fail_fn(count);
  90 }
  91
  92 /*
  93  * If the unlock was done on a contended lock, or if the unlock simply fails
  94  * then the mutex remains locked.
  95  */
  96 #define __mutex_slowpath_needs_to_unlock()      1
  97
  98 /*
  99  * For __mutex_fastpath_trylock we use another construct which could be
 100  * described as a "single value cmpxchg".
 101  *
 102  * This provides the needed trylock semantics like cmpxchg would, but it is
 103  * lighter and less generic than a true cmpxchg implementation.
 104  */
 105 static inline int
 106 __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
 107 {
 108         int __ex_flag, __res, __orig;
 109
 110         __asm__ (
 111
 112                 "1: ldrex       %0, [%3]        \n\t"
 113                 "subs           %1, %0, #1      \n\t"
 114                 "strexeq        %2, %1, [%3]    \n\t"
 115                 "movlt          %0, #0          \n\t"
 116                 "cmpeq          %2, #0          \n\t"
 117                 "bgt            1b              "
 118
 119                 : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
 120                 : "r" (&count->counter)
 121                 : "cc", "memory" );
 122
 123         return __orig;
 124 }
 125
 126 #endif
 127 #endif