workbench/libs/mesa/src/gallium/auxiliary/util/u_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef U_ATOMIC_H
  10 #define U_ATOMIC_H
  11
  12 #include "pipe/p_compiler.h"
  13 #include "pipe/p_defines.h"
  14
  15 /* Favor OS-provided implementations.
  16  *
  17  * Where no OS-provided implementation is available, fall back to
  18  * locally coded assembly, compiler intrinsic or ultimately a
  19  * mutex-based implementation.
  20  */
  21 #if (defined(PIPE_SUBSYSTEM_WINDOWS_DISPLAY) || \
  22      defined(PIPE_SUBSYSTEM_WINDOWS_MINIPORT))
  23 #define PIPE_ATOMIC_OS_UNLOCKED
  24 #elif defined(PIPE_OS_SOLARIS)
  25 #define PIPE_ATOMIC_OS_SOLARIS
  26 #elif defined(PIPE_CC_MSVC)
  27 #define PIPE_ATOMIC_MSVC_INTRINSIC
  28 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
  29 #define PIPE_ATOMIC_ASM_MSVC_X86
  30 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
  31 #define PIPE_ATOMIC_ASM_GCC_X86
  32 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_ARM))
  33 #define PIPE_ATOMIC_ASM_GCC_ARM
  34 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
  35 #define PIPE_ATOMIC_ASM_GCC_X86_64
  36 #elif defined(PIPE_OS_AROS) && defined(PIPE_ARCH_M68K)
  37 #define PIPE_ATOMIC_OS_AROS_CPU_M68K
  38 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
  39 #define PIPE_ATOMIC_GCC_INTRINSIC
  40 #else
  41 #error "Unsupported platform"
  42 #endif
  43
  44
  45 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
  46 #define PIPE_ATOMIC "GCC x86_64 assembly"
  47
  48 #ifdef __cplusplus
  49 extern "C" {
  50 #endif
  51
  52 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  53 #define p_atomic_read(_v) (*(_v))
  54
  55 static INLINE boolean
  56 p_atomic_dec_zero(int32_t *v)
  57 {
  58    unsigned char c;
  59
  60    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
  61                         ::"memory");
  62
  63    return c != 0;
  64 }
  65
  66 static INLINE void
  67 p_atomic_inc(int32_t *v)
  68 {
  69    __asm__ __volatile__("lock; incl %0":"+m"(*v));
  70 }
  71
  72 static INLINE void
  73 p_atomic_dec(int32_t *v)
  74 {
  75    __asm__ __volatile__("lock; decl %0":"+m"(*v));
  76 }
  77
  78 static INLINE int32_t
  79 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
  80 {
  81    return __sync_val_compare_and_swap(v, old, _new);
  82 }
  83
  84 #ifdef __cplusplus
  85 }
  86 #endif
  87
  88 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
  89
  90
  91 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
  92
  93 #define PIPE_ATOMIC "GCC x86 assembly"
  94
  95 #ifdef __cplusplus
  96 extern "C" {
  97 #endif
  98
  99 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 100 #define p_atomic_read(_v) (*(_v))
 101
 102 static INLINE boolean
 103 p_atomic_dec_zero(int32_t *v)
 104 {
 105    unsigned char c;
 106
 107    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
 108                         ::"memory");
 109
 110    return c != 0;
 111 }
 112
 113 static INLINE void
 114 p_atomic_inc(int32_t *v)
 115 {
 116    __asm__ __volatile__("lock; incl %0":"+m"(*v));
 117 }
 118
 119 static INLINE void
 120 p_atomic_dec(int32_t *v)
 121 {
 122    __asm__ __volatile__("lock; decl %0":"+m"(*v));
 123 }
 124
 125 static INLINE int32_t
 126 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 127 {
 128    return __sync_val_compare_and_swap(v, old, _new);
 129 }
 130
 131 #ifdef __cplusplus
 132 }
 133 #endif
 134
 135 #endif
 136
 137 #if defined(PIPE_ATOMIC_ASM_GCC_ARM)
 138
 139 #define PIPE_ATOMIC "GCC ARM assembly"
 140
 141 #ifdef __cplusplus
 142 extern "C" {
 143 #endif
 144
 145 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 146 #define p_atomic_read(_v) (*(_v))
 147
 148 static INLINE boolean
 149 p_atomic_dec_zero(int32_t *v)
 150 {
 151    unsigned long temp;
 152    int result;
 153    unsigned long cc;
 154    __asm__ __volatile__("\n1: ldrex %0, [%3]; subs %0, %0, #1; moveq %2, #1; movne %2, #0; strex %1, %0, [%3]; teq %1, #0; bne 1b"
 155                                                    :"=&r"(result), "=&r"(temp), "=&r"(cc)
 156                                                    :"r"(v)
 157                                                    :"cc");
 158    return cc;
 159 }
 160
 161 static INLINE void
 162 p_atomic_inc(int32_t *v)
 163 {
 164    unsigned long temp;
 165    int result;
 166    __asm__ __volatile__("\n1: ldrex %0, [%2]; add %0, %0, #1; strex %1, %0, [%2]; teq %1, #0; bne 1b"
 167                                                    :"=&r"(result), "=&r"(temp)
 168                                                    :"r"(v)
 169                                                    :"cc");
 170 }
 171
 172 static INLINE void
 173 p_atomic_dec(int32_t *v)
 174 {
 175    unsigned long temp;
 176    int result;
 177    __asm__ __volatile__("\n1: ldrex %0, [%2]; sub %0, %0, #1; strex %1, %0, [%2]; teq %1, #0; bne 1b"
 178                                                    :"=&r"(result), "=&r"(temp)
 179                                                    :"r"(v)
 180                                                    :"cc");
 181 }
 182
 183 static INLINE int32_t
 184 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 185 {
 186           int32_t oldval;
 187            unsigned long temp;
 188            __asm__ __volatile__("\n1: ldrex %0,[%2]; teq %0, %3; strexeq %1, %4, [%2]; teq %1, #0; bne 1b"
 189                                                            :"=&r"(oldval), "=&r"(temp)
 190                                                            :"r"(v), "Ir"(old), "r"(_new)
 191                                                            :"cc");
 192            return oldval;
 193 }
 194
 195 #ifdef __cplusplus
 196 }
 197 #endif
 198
 199 #endif
 200
 201
 202 /* Implementation using GCC-provided synchronization intrinsics
 203  */
 204 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
 205
 206 #define PIPE_ATOMIC "GCC Sync Intrinsics"
 207
 208 #ifdef __cplusplus
 209 extern "C" {
 210 #endif
 211
 212 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 213 #define p_atomic_read(_v) (*(_v))
 214
 215 static INLINE boolean
 216 p_atomic_dec_zero(int32_t *v)
 217 {
 218    return (__sync_sub_and_fetch(v, 1) == 0);
 219 }
 220
 221 static INLINE void
 222 p_atomic_inc(int32_t *v)
 223 {
 224    (void) __sync_add_and_fetch(v, 1);
 225 }
 226
 227 static INLINE void
 228 p_atomic_dec(int32_t *v)
 229 {
 230    (void) __sync_sub_and_fetch(v, 1);
 231 }
 232
 233 static INLINE int32_t
 234 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 235 {
 236    return __sync_val_compare_and_swap(v, old, _new);
 237 }
 238
 239 #ifdef __cplusplus
 240 }
 241 #endif
 242
 243 #endif
 244
 245
 246
 247 /* Unlocked version for single threaded environments, such as some
 248  * windows kernel modules.
 249  */
 250 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
 251
 252 #define PIPE_ATOMIC "Unlocked"
 253
 254 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 255 #define p_atomic_read(_v) (*(_v))
 256 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
 257 #define p_atomic_inc(_v) ((void) (*(_v))++)
 258 #define p_atomic_dec(_v) ((void) (*(_v))--)
 259 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
 260
 261 #endif
 262
 263
 264 /* Locally coded assembly for MSVC on x86:
 265  */
 266 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
 267
 268 #define PIPE_ATOMIC "MSVC x86 assembly"
 269
 270 #ifdef __cplusplus
 271 extern "C" {
 272 #endif
 273
 274 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 275 #define p_atomic_read(_v) (*(_v))
 276
 277 static INLINE boolean
 278 p_atomic_dec_zero(int32_t *v)
 279 {
 280    unsigned char c;
 281
 282    __asm {
 283       mov       eax, [v]
 284       lock dec  dword ptr [eax]
 285       sete      byte ptr [c]
 286    }
 287
 288    return c != 0;
 289 }
 290
 291 static INLINE void
 292 p_atomic_inc(int32_t *v)
 293 {
 294    __asm {
 295       mov       eax, [v]
 296       lock inc  dword ptr [eax]
 297    }
 298 }
 299
 300 static INLINE void
 301 p_atomic_dec(int32_t *v)
 302 {
 303    __asm {
 304       mov       eax, [v]
 305       lock dec  dword ptr [eax]
 306    }
 307 }
 308
 309 static INLINE int32_t
 310 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 311 {
 312    int32_t orig;
 313
 314    __asm {
 315       mov ecx, [v]
 316       mov eax, [old]
 317       mov edx, [_new]
 318       lock cmpxchg [ecx], edx
 319       mov [orig], eax
 320    }
 321
 322    return orig;
 323 }
 324
 325 #ifdef __cplusplus
 326 }
 327 #endif
 328
 329 #endif
 330
 331
 332 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
 333
 334 #define PIPE_ATOMIC "MSVC Intrinsics"
 335
 336 #include <intrin.h>
 337
 338 #pragma intrinsic(_InterlockedIncrement)
 339 #pragma intrinsic(_InterlockedDecrement)
 340 #pragma intrinsic(_InterlockedCompareExchange)
 341
 342 #ifdef __cplusplus
 343 extern "C" {
 344 #endif
 345
 346 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 347 #define p_atomic_read(_v) (*(_v))
 348
 349 static INLINE boolean
 350 p_atomic_dec_zero(int32_t *v)
 351 {
 352    return _InterlockedDecrement((long *)v) == 0;
 353 }
 354
 355 static INLINE void
 356 p_atomic_inc(int32_t *v)
 357 {
 358    _InterlockedIncrement((long *)v);
 359 }
 360
 361 static INLINE void
 362 p_atomic_dec(int32_t *v)
 363 {
 364    _InterlockedDecrement((long *)v);
 365 }
 366
 367 static INLINE int32_t
 368 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 369 {
 370    return _InterlockedCompareExchange((long *)v, _new, old);
 371 }
 372
 373 #ifdef __cplusplus
 374 }
 375 #endif
 376
 377 #endif
 378
 379 #if defined(PIPE_ATOMIC_OS_SOLARIS)
 380
 381 #define PIPE_ATOMIC "Solaris OS atomic functions"
 382
 383 #include <atomic.h>
 384
 385 #ifdef __cplusplus
 386 extern "C" {
 387 #endif
 388
 389 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 390 #define p_atomic_read(_v) (*(_v))
 391
 392 static INLINE boolean
 393 p_atomic_dec_zero(int32_t *v)
 394 {
 395    uint32_t n = atomic_dec_32_nv((uint32_t *) v);
 396
 397    return n != 0;
 398 }
 399
 400 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
 401 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
 402
 403 #define p_atomic_cmpxchg(_v, _old, _new) \
 404         atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
 405
 406 #ifdef __cplusplus
 407 }
 408 #endif
 409
 410 #endif
 411
 412
 413 #if defined(PIPE_ATOMIC_OS_AROS_CPU_M68K)
 414
 415 #define PIPE_ATOMIC "AROS OS atomic functions"
 416
 417 #include <aros/atomic.h>
 418
 419 #ifdef __cplusplus
 420 extern "C" {
 421 #endif
 422
 423 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 424 #define p_atomic_read(_v) (*(_v))
 425
 426 static INLINE boolean
 427 p_atomic_dec_zero(int32_t *v)
 428 {
 429    boolean n;
 430
 431    /* FIXME: AROS needs an atomic decrement and return... */
 432    Disable();
 433    AROS_ATOMIC_DEC(*(LONG *)v);
 434    n = (*v == 0) ? TRUE : FALSE;
 435    Enable();
 436
 437    return n;
 438 }
 439
 440 #define p_atomic_inc(_v) AROS_ATOMIC_INC(*(LONG *)_v)
 441 #define p_atomic_dec(_v) AROS_ATOMIC_DEC(*(LONG *)_v)
 442
 443 static INLINE int32_t
 444 p_atomic_cmpxchg(int32_t *v, int32_t o, int32_t n)
 445 {
 446         int32_t ret;
 447
 448         /* FIXME: AROS needs an atomic cmpxchg, using CAS.
 449          * However we can't do this if:
 450          *  a) We are on a 68000 or
 451          *  b) The 'v' points to Chip RAM (no r/m/w possible)
 452          *
 453          *  Settle for Disable()/Enable() for now.
 454          */
 455         Disable();
 456         if (*v == o)
 457                 *v = (n);
 458         ret = *v;
 459         Enable();
 460
 461         return ret;
 462 }
 463
 464 #ifdef __cplusplus
 465 }
 466 #endif
 467
 468 #endif
 469
 470
 471 #ifndef PIPE_ATOMIC
 472 #error "No pipe_atomic implementation selected"
 473 #endif
 474
 475
 476
 477 #endif /* U_ATOMIC_H */