3 * Copyright (C) Igor Sysoev
4 * Copyright (C) Nginx, Inc.
9 * The ppc assembler treats ";" as comment, so we have to use "\n".
10 * The minus in "bne-" is a hint for the branch prediction unit that
11 * this branch is unlikely to be taken.
12 * The "1b" means the nearest backward label "1" and the "1f" means
13 * the nearest forward label "1".
15 * The "b" means that the base registers can be used only, i.e.
16 * any register except r0. The r0 register always has a zero value and
17 * could not be used in "addi r0, r0, 1".
18 * The "=&b" means that no input registers can be used.
20 * "sync" read and write barriers
21 * "isync" read barrier, is faster than "sync"
22 * "eieio" write barrier, is faster than "sync"
23 * "lwsync" write barrier, is faster than "eieio" on ppc64
26 #if (NGX_PTR_SIZE == 8)
28 static ngx_inline ngx_atomic_uint_t
29 ngx_atomic_cmp_set(ngx_atomic_t
*lock
, ngx_atomic_uint_t old
,
30 ngx_atomic_uint_t set
)
32 ngx_atomic_uint_t res
, temp
;
36 " li %0, 0 \n" /* preset "0" to "res" */
37 " lwsync \n" /* write barrier */
39 " ldarx %1, 0, %2 \n" /* load from [lock] into "temp" */
40 /* and store reservation */
41 " cmpd %1, %3 \n" /* compare "temp" and "old" */
42 " bne- 2f \n" /* not equal */
43 " stdcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */
45 " bne- 1b \n" /* the reservation was cleared */
46 " isync \n" /* read barrier */
47 " li %0, 1 \n" /* set "1" to "res" */
50 : "=&b" (res
), "=&b" (temp
)
51 : "b" (lock
), "b" (old
), "b" (set
)
58 static ngx_inline ngx_atomic_int_t
59 ngx_atomic_fetch_add(ngx_atomic_t
*value
, ngx_atomic_int_t add
)
61 ngx_atomic_uint_t res
, temp
;
65 " lwsync \n" /* write barrier */
66 "1: ldarx %0, 0, %2 \n" /* load from [value] into "res" */
67 /* and store reservation */
68 " add %1, %0, %3 \n" /* "res" + "add" store in "temp" */
69 " stdcx. %1, 0, %2 \n" /* store "temp" into [value] if reservation */
71 " bne- 1b \n" /* try again if reservation was cleared */
72 " isync \n" /* read barrier */
74 : "=&b" (res
), "=&b" (temp
)
75 : "b" (value
), "b" (add
)
83 #define ngx_memory_barrier() \
84 __asm__ volatile ("isync \n lwsync \n" ::: "memory")
86 #define ngx_memory_barrier() __asm__ volatile ("" ::: "memory")
91 static ngx_inline ngx_atomic_uint_t
92 ngx_atomic_cmp_set(ngx_atomic_t
*lock
, ngx_atomic_uint_t old
,
93 ngx_atomic_uint_t set
)
95 ngx_atomic_uint_t res
, temp
;
99 " li %0, 0 \n" /* preset "0" to "res" */
100 " eieio \n" /* write barrier */
102 " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */
103 /* and store reservation */
104 " cmpw %1, %3 \n" /* compare "temp" and "old" */
105 " bne- 2f \n" /* not equal */
106 " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */
108 " bne- 1b \n" /* the reservation was cleared */
109 " isync \n" /* read barrier */
110 " li %0, 1 \n" /* set "1" to "res" */
113 : "=&b" (res
), "=&b" (temp
)
114 : "b" (lock
), "b" (old
), "b" (set
)
121 static ngx_inline ngx_atomic_int_t
122 ngx_atomic_fetch_add(ngx_atomic_t
*value
, ngx_atomic_int_t add
)
124 ngx_atomic_uint_t res
, temp
;
128 " eieio \n" /* write barrier */
129 "1: lwarx %0, 0, %2 \n" /* load from [value] into "res" */
130 /* and store reservation */
131 " add %1, %0, %3 \n" /* "res" + "add" store in "temp" */
132 " stwcx. %1, 0, %2 \n" /* store "temp" into [value] if reservation */
134 " bne- 1b \n" /* try again if reservation was cleared */
135 " isync \n" /* read barrier */
137 : "=&b" (res
), "=&b" (temp
)
138 : "b" (value
), "b" (add
)
146 #define ngx_memory_barrier() \
147 __asm__ volatile ("isync \n eieio \n" ::: "memory")
149 #define ngx_memory_barrier() __asm__ volatile ("" ::: "memory")
155 #define ngx_cpu_pause()