1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_AtomicOperations_h
8 #define jit_AtomicOperations_h
10 #include "mozilla/Types.h"
14 #include "jit/AtomicOperationsGenerated.h"
15 #include "vm/SharedMem.h"
21 * [SMDOC] Atomic Operations
23 * The atomic operations layer defines types and functions for
24 * JIT-compatible atomic operation.
26 * The fundamental constraints on the functions are:
28 * - That their realization here MUST be compatible with code the JIT
29 * generates for its Atomics operations, so that an atomic access
30 * from the interpreter or runtime - from any C++ code - really is
31 * atomic relative to a concurrent, compatible atomic access from
32 * jitted code. That is, these primitives expose JIT-compatible
33 * atomicity functionality to C++.
35 * - That accesses may race without creating C++ undefined behavior:
36 * atomic accesses (marked "SeqCst") may race with non-atomic
37 * accesses (marked "SafeWhenRacy"); overlapping but non-matching,
38 * and hence incompatible, atomic accesses may race; and non-atomic
39 * accesses may race. The effects of races need not be predictable,
40 * so garbage can be produced by a read or written by a write, but
41 * the effects must be benign: the program must continue to run, and
42 * only the memory in the union of addresses named in the racing
43 * accesses may be affected.
45 * The compatibility constraint means that if the JIT makes dynamic
46 * decisions about how to implement atomic operations then
47 * corresponding dynamic decisions MUST be made in the implementations
48 * of the functions below.
50 * The safe-for-races constraint means that by and large, it is hard
51 * to implement these primitives in C++. See "Implementation notes"
54 * The "SeqCst" suffix on operations means "sequentially consistent"
55 * and means such a function's operation must have "sequentially
56 * consistent" memory ordering. See mfbt/Atomics.h for an explanation
57 * of this memory ordering.
59 * Note that a "SafeWhenRacy" access does not provide the atomicity of
60 * a "relaxed atomic" access: it can read or write garbage if there's
64 * Implementation notes.
66 * It's not a requirement that these functions be inlined; performance
67 * is not a great concern. On some platforms these functions may call
68 * functions that use inline assembly. See GenerateAtomicOperations.py.
70 * In principle these functions will not be written in C++, thus
71 * making races defined behavior if all racy accesses from C++ go via
72 * these functions. (Jitted code will always be safe for races and
73 * provides the same guarantees as these functions.)
75 * The appropriate implementations will be platform-specific and
76 * there are some obvious implementation strategies to choose
77 * from, sometimes a combination is appropriate:
79 * - generating the code at run-time with the JIT;
80 * - hand-written assembler (maybe inline); or
81 * - using special compiler intrinsics or directives.
83 * Trusting the compiler not to generate code that blows up on a
84 * race definitely won't work in the presence of TSan, or even of
85 * optimizing compilers in seemingly-"innocuous" conditions. (See
86 * https://www.usenix.org/legacy/event/hotpar11/tech/final_files/Boehm.pdf
89 class AtomicOperations
{
90 // The following functions are defined for T = int8_t, uint8_t,
91 // int16_t, uint16_t, int32_t, uint32_t, int64_t, and uint64_t.
93 // Atomically read *addr.
95 static inline T
loadSeqCst(T
* addr
);
97 // Atomically store val in *addr.
99 static inline void storeSeqCst(T
* addr
, T val
);
101 // Atomically store val in *addr and return the old value of *addr.
102 template <typename T
>
103 static inline T
exchangeSeqCst(T
* addr
, T val
);
105 // Atomically check that *addr contains oldval and if so replace it
106 // with newval, in any case returning the old contents of *addr.
107 template <typename T
>
108 static inline T
compareExchangeSeqCst(T
* addr
, T oldval
, T newval
);
110 // Atomically add, subtract, bitwise-AND, bitwise-OR, or bitwise-XOR
111 // val into *addr and return the old value of *addr.
112 template <typename T
>
113 static inline T
fetchAddSeqCst(T
* addr
, T val
);
115 template <typename T
>
116 static inline T
fetchSubSeqCst(T
* addr
, T val
);
118 template <typename T
>
119 static inline T
fetchAndSeqCst(T
* addr
, T val
);
121 template <typename T
>
122 static inline T
fetchOrSeqCst(T
* addr
, T val
);
124 template <typename T
>
125 static inline T
fetchXorSeqCst(T
* addr
, T val
);
127 // The SafeWhenRacy functions are to be used when C++ code has to access
128 // memory without synchronization and can't guarantee that there won't be a
129 // race on the access. But they are access-atomic for integer data so long
130 // as any racing writes are of the same size and to the same address.
132 // Defined for all the integral types as well as for float32 and float64,
133 // but not access-atomic for floats, nor for int64 and uint64 on 32-bit
135 template <typename T
>
136 static inline T
loadSafeWhenRacy(T
* addr
);
138 // Defined for all the integral types as well as for float32 and float64,
139 // but not access-atomic for floats, nor for int64 and uint64 on 32-bit
141 template <typename T
>
142 static inline void storeSafeWhenRacy(T
* addr
, T val
);
144 // Replacement for memcpy(). No access-atomicity guarantees.
145 static inline void memcpySafeWhenRacy(void* dest
, const void* src
,
148 // Replacement for memmove(). No access-atomicity guarantees.
149 static inline void memmoveSafeWhenRacy(void* dest
, const void* src
,
153 // Test lock-freedom for any int32 value. This implements the
154 // Atomics::isLockFree() operation in the ECMAScript Shared Memory and
155 // Atomics specification, as follows:
157 // 4-byte accesses are always lock free (in the spec).
158 // 1-, 2-, and 8-byte accesses are always lock free (in SpiderMonkey).
160 // There is no lock-freedom for JS for any other values on any platform.
161 static constexpr inline bool isLockfreeJS(int32_t n
);
163 // If the return value is true then the templated functions below are
164 // supported for int64_t and uint64_t. If the return value is false then
165 // those functions will MOZ_CRASH. The value of this call does not change
167 static inline bool hasAtomic8();
169 // If the return value is true then hasAtomic8() is true and the atomic
170 // operations are indeed lock-free. The value of this call does not change
172 static inline bool isLockfree8();
174 // Execute a full memory barrier (LoadLoad+LoadStore+StoreLoad+StoreStore).
175 static inline void fenceSeqCst();
177 // All clients should use the APIs that take SharedMem pointers.
178 // See above for semantics and acceptable types.
180 template <typename T
>
181 static T
loadSeqCst(SharedMem
<T
*> addr
) {
182 return loadSeqCst(addr
.unwrap());
185 template <typename T
>
186 static void storeSeqCst(SharedMem
<T
*> addr
, T val
) {
187 return storeSeqCst(addr
.unwrap(), val
);
190 template <typename T
>
191 static T
exchangeSeqCst(SharedMem
<T
*> addr
, T val
) {
192 return exchangeSeqCst(addr
.unwrap(), val
);
195 template <typename T
>
196 static T
compareExchangeSeqCst(SharedMem
<T
*> addr
, T oldval
, T newval
) {
197 return compareExchangeSeqCst(addr
.unwrap(), oldval
, newval
);
200 template <typename T
>
201 static T
fetchAddSeqCst(SharedMem
<T
*> addr
, T val
) {
202 return fetchAddSeqCst(addr
.unwrap(), val
);
205 template <typename T
>
206 static T
fetchSubSeqCst(SharedMem
<T
*> addr
, T val
) {
207 return fetchSubSeqCst(addr
.unwrap(), val
);
210 template <typename T
>
211 static T
fetchAndSeqCst(SharedMem
<T
*> addr
, T val
) {
212 return fetchAndSeqCst(addr
.unwrap(), val
);
215 template <typename T
>
216 static T
fetchOrSeqCst(SharedMem
<T
*> addr
, T val
) {
217 return fetchOrSeqCst(addr
.unwrap(), val
);
220 template <typename T
>
221 static T
fetchXorSeqCst(SharedMem
<T
*> addr
, T val
) {
222 return fetchXorSeqCst(addr
.unwrap(), val
);
225 template <typename T
>
226 static T
loadSafeWhenRacy(SharedMem
<T
*> addr
) {
227 return loadSafeWhenRacy(addr
.unwrap());
230 template <typename T
>
231 static void storeSafeWhenRacy(SharedMem
<T
*> addr
, T val
) {
232 return storeSafeWhenRacy(addr
.unwrap(), val
);
235 template <typename T
>
236 static void memcpySafeWhenRacy(SharedMem
<T
*> dest
, SharedMem
<T
*> src
,
238 memcpySafeWhenRacy(dest
.template cast
<void*>().unwrap(),
239 src
.template cast
<void*>().unwrap(), nbytes
);
242 template <typename T
>
243 static void memcpySafeWhenRacy(SharedMem
<T
*> dest
, T
* src
, size_t nbytes
) {
244 memcpySafeWhenRacy(dest
.template cast
<void*>().unwrap(),
245 static_cast<void*>(src
), nbytes
);
248 template <typename T
>
249 static void memcpySafeWhenRacy(T
* dest
, SharedMem
<T
*> src
, size_t nbytes
) {
250 memcpySafeWhenRacy(static_cast<void*>(dest
),
251 src
.template cast
<void*>().unwrap(), nbytes
);
254 template <typename T
>
255 static void memmoveSafeWhenRacy(SharedMem
<T
*> dest
, SharedMem
<T
*> src
,
257 memmoveSafeWhenRacy(dest
.template cast
<void*>().unwrap(),
258 src
.template cast
<void*>().unwrap(), nbytes
);
261 static void memsetSafeWhenRacy(SharedMem
<uint8_t*> dest
, int value
,
264 size_t iterations
= nbytes
/ sizeof(buf
);
265 size_t tail
= nbytes
% sizeof(buf
);
267 if (iterations
> 0) {
268 memset(buf
, value
, sizeof(buf
));
269 while (iterations
--) {
270 memcpySafeWhenRacy(dest
+ offs
, SharedMem
<uint8_t*>::unshared(buf
),
275 memset(buf
, value
, tail
);
277 memcpySafeWhenRacy(dest
+ offs
, SharedMem
<uint8_t*>::unshared(buf
), tail
);
280 template <typename T
>
281 static void podCopySafeWhenRacy(SharedMem
<T
*> dest
, SharedMem
<T
*> src
,
283 memcpySafeWhenRacy(dest
, src
, nelem
* sizeof(T
));
286 template <typename T
>
287 static void podMoveSafeWhenRacy(SharedMem
<T
*> dest
, SharedMem
<T
*> src
,
289 memmoveSafeWhenRacy(dest
, src
, nelem
* sizeof(T
));
293 constexpr inline bool AtomicOperations::isLockfreeJS(int32_t size
) {
294 // Keep this in sync with atomicIsLockFreeJS() in jit/MacroAssembler.cpp.
302 // The spec requires Atomics.isLockFree(4) to return true.
314 // As explained above, our atomic operations are not portable even in principle,
315 // so we must include platform+compiler specific definitions here.
317 // x86, x64, arm, and arm64 are maintained by Mozilla. All other platform
318 // setups are by platform maintainers' request and are not maintained by
321 // If you are using a platform+compiler combination that causes an error below
322 // (and if the problem isn't just that the compiler uses a different name for a
323 // known architecture), you have basically three options:
325 // - find an already-supported compiler for the platform and use that instead
327 // - write your own support code for the platform+compiler and create a new
330 // - include jit/shared/AtomicOperations-feeling-lucky.h in a case for the
331 // platform below, if you have a gcc-compatible compiler and truly feel
332 // lucky. You may have to add a little code to that file, too.
334 // Simulators are confusing. These atomic primitives must be compatible with
335 // the code that the JIT emits, but of course for an ARM simulator running on
336 // x86 the primitives here will be for x86, not for ARM, while the JIT emits ARM
337 // code. Our ARM simulator solves that the easy way: by using these primitives
338 // to implement its atomic operations. For other simulators there may need to
339 // be special cases below to provide simulator-compatible primitives, for
340 // example, for our ARM64 simulator the primitives could in principle
341 // participate in the memory exclusivity monitors implemented by the simulator.
342 // Such a solution is likely to be difficult.
344 #ifdef JS_HAVE_GENERATED_ATOMIC_OPS
345 # include "jit/shared/AtomicOperations-shared-jit.h"
346 #elif defined(JS_SIMULATOR_MIPS32) || defined(__mips__)
347 # include "jit/mips-shared/AtomicOperations-mips-shared.h"
349 # include "jit/shared/AtomicOperations-feeling-lucky.h"
352 #endif // jit_AtomicOperations_h