mfbt/AtomicBitfields.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef mozilla_AtomicBitfields_h
   8 #define mozilla_AtomicBitfields_h
   9
  10 #include "mozilla/Assertions.h"
  11 #include "mozilla/MacroArgs.h"
  12 #include "mozilla/MacroForEach.h"
  13
  14 #include <limits>
  15 #include <stdint.h>
  16 #include <type_traits>
  17
  18 #ifdef __wasi__
  19 #  include "mozilla/WasiAtomic.h"
  20 #else
  21 #  include <atomic>
  22 #endif  // __wasi__
  23
  24 namespace mozilla {
  25
  26 // Creates a series of atomic bitfields.
  27 //
  28 // |aBitfields| is the name of the underlying storage for the bitfields.
  29 // |aBitFieldsSize| is the size of the underlying storage (8, 16, 32, or 64).
  30 //
  31 // Bitfields are specified as a triplet of (type, name, size), which mirrors
  32 // the way you declare native C++ bitfields (bool mMyField1: 1). Trailing
  33 // commas are not supported in the list of bitfields.
  34 //
  35 // Signed integer types are not supported by this Macro to avoid dealing with
  36 // packing/unpacking the sign bit and C++'s general messiness around signed
  37 // integer representations not being fully defined.
  38 //
  39 // You cannot request a single field that's the
  40 // size of the the entire bitfield storage. Just use a normal atomic integer!
  41 //
  42 //
  43 // ========================== SEMANTICS AND SAFETY ============================
  44 //
  45 // All fields are default-initialized to 0.
  46 //
  47 // In debug builds, storing a value to a bitfield that's larger than its bits
  48 // can fit will trigger an assertion. In release builds, the value will just be
  49 // masked off.
  50 //
  51 // If you request anything unsupported by this macro it should result in
  52 // a compile-time error (either a static assert or just weird macro errors).
  53 // For instance, this macro will statically prevent using more bits than
  54 // |aBitFieldsSize|, so specifying the size is just to prevent accidentally
  55 // making the storage bigger.
  56 //
  57 // Each field will get a Load$NAME and Store$Name method which will atomically
  58 // load and store the requested value with a Sequentially Consistent memory
  59 // order (to be on the safe side). Storing a field requires a compare-exchange,
  60 // so a thread may get stalled if there's a lot of contention on the bitfields.
  61 //
  62 //
  63 // ============================== MOTIVATION ==================================
  64 //
  65 // You might be wondering: why would I need atomic bitfields? Well as it turns
  66 // out, bitfields and concurrency mess a lot of people up!
  67 //
  68 // CPUs don't have operations to write to a handful of bits -- they generally
  69 // only have the precision of a byte. So when you use C++'s native bitfields,
  70 // the compiler generates code to mask and shift the values in for you. This
  71 // means writing to a single field will actually overwrite all the other
  72 // bitfields that are packed in with it!
  73 //
  74 // In single-threaded code this is fine; the old values are loaded and written
  75 // back by the compiler's generated code. But in concurrent code, it means
  76 // that accessing two different fields can be an unexpected Data Race (which is
  77 // Undefined Behavior!).
  78 //
  79 // By using MOZ_ATOMIC_BITFIELDS, you protect yourself from these Data Races,
  80 // and don't have to worry about writes getting lost.
  81 //
  82 //
  83 // ================================ EXAMPLE ===================================
  84 //
  85 //   #include "mozilla/AtomicBitfields.h"
  86 //   #include <stdint.h>
  87 //
  88 //
  89 //   struct MyType {
  90 //     MOZ_ATOMIC_BITFIELDS(mAtomicFields, 8, (
  91 //      (bool, IsDownloaded, 1),
  92 //      (uint32_t, SomeData, 2),
  93 //      (uint8_t, OtherData, 5)
  94 //     ))
  95 //
  96 //     int32_t aNormalInteger;
  97 //
  98 //     explicit MyType(uint32_t aSomeData): aNormalInteger(7) {
  99 //       StoreSomeData(aSomeData);
 100 //       // Other bitfields were already default initialized to 0/false
 101 //     }
 102 //   };
 103 //
 104 //
 105 //   int main() {
 106 //     MyType val(3);
 107 //
 108 //     if (!val.LoadIsDownloaded()) {
 109 //       val.StoreOtherData(2);
 110 //       val.StoreIsDownloaded(true);
 111 //     }
 112 //   }
 113 //
 114 //
 115 // ============================== GENERATED ===================================
 116 //
 117 // This macro is a real mess to read because, well, it's a macro. So for the
 118 // sake of anyone who has to review or modify its internals, here's a rough
 119 // sketch of what the above example would expand to:
 120 //
 121 //   struct MyType {
 122 //     // The actual storage of the bitfields, initialized to 0.
 123 //     std::atomic_uint8_t mAtomicFields{0};
 124 //
 125 //     // How many bits were actually used (in this case, all of them).
 126 //     static const size_t mAtomicFields_USED_BITS = 8;
 127 //
 128 //     // The offset values for each field.
 129 //     static const size_t mAtomicFieldsIsDownloaded = 0;
 130 //     static const size_t mAtomicFieldsSomeData = 1;
 131 //     static const size_t mAtomicFieldsOtherData = 3;
 132 //
 133 //     // Quick safety guard to prevent capacity overflow.
 134 //     static_assert(mAtomicFields_USED_BITS <= 8);
 135 //
 136 //     // Asserts that fields are reasonable.
 137 //     static_assert(8>1, "mAtomicFields: MOZ_ATOMIC_BITFIELDS field too big");
 138 //     static_assert(std::is_unsigned<bool>(), "mAtomicFields:
 139 //     MOZ_ATOMIC_BITFIELDS doesn't support signed payloads");
 140 //     // ...and so on
 141 //
 142 //     // Load/Store methods for all the fields.
 143 //
 144 //     bool LoadIsDownloaded() { ... }
 145 //     void StoreIsDownloaded(bool aValue) { ... }
 146 //
 147 //     uint32_t LoadSomeData() { ... }
 148 //     void StoreSomeData(uint32_t aValue) { ... }
 149 //
 150 //     uint8_t LoadOtherData() { ... }
 151 //     void StoreOtherData(uint8_t aValue) { ... }
 152 //
 153 //
 154 //     // Remainder of the struct body continues normally.
 155 //     int32_t aNormalInteger;
 156 //     explicit MyType(uint32_t aSomeData): aNormalInteger(7) {
 157 //       StoreSomeData(aSomeData);
 158 //       // Other bitfields were already default initialized to 0/false.
 159 //     }
 160 //   }
 161 //
 162 // Also if you're wondering why there's so many MOZ_CONCAT's -- it's because
 163 // the preprocessor sometimes gets confused if we use ## on certain arguments.
 164 // MOZ_CONCAT reliably kept the preprocessor happy, sorry it's so ugly!
 165 //
 166 //
 167 // ==================== FIXMES / FUTURE WORK ==================================
 168 //
 169 // * It would be nice if LoadField could be IsField for booleans.
 170 //
 171 // * For the case of setting something to all 1's or 0's, we can use
 172 //   |fetch_or| or |fetch_and| instead of |compare_exchange_weak|. Is this
 173 //   worth providing? (Possibly for 1-bit boolean fields?)
 174 //
 175 // * Try harder to hide the atomic/enum/array internals from
 176 //   the outer struct?
 177 //
 178 #define MOZ_ATOMIC_BITFIELDS(aBitfields, aBitfieldsSize, aFields)             \
 179   std::atomic_uint##aBitfieldsSize##_t aBitfields{0};                         \
 180                                                                               \
 181   static const size_t MOZ_CONCAT(aBitfields, _USED_BITS) =                    \
 182       MOZ_FOR_EACH_SEPARATED(MOZ_ATOMIC_BITFIELDS_FIELD_SIZE, (+), (),        \
 183                              aFields);                                        \
 184                                                                               \
 185   MOZ_ROLL_EACH(MOZ_ATOMIC_BITFIELDS_OFFSET_HELPER1, (aBitfields, ), aFields) \
 186                                                                               \
 187   static_assert(MOZ_CONCAT(aBitfields, _USED_BITS) <= aBitfieldsSize,         \
 188                 #aBitfields ": Maximum bits (" #aBitfieldsSize                \
 189                             ") exceeded for MOZ_ATOMIC_BITFIELDS instance");  \
 190                                                                               \
 191   MOZ_FOR_EACH(MOZ_ATOMIC_BITFIELDS_FIELD_HELPER,                             \
 192                (aBitfields, aBitfieldsSize, ), aFields)
 193
 194 // Just a helper to unpack the head of the list.
 195 #define MOZ_ATOMIC_BITFIELDS_OFFSET_HELPER1(aBitfields, aFields) \
 196   MOZ_ATOMIC_BITFIELDS_OFFSET_HELPER2(aBitfields, MOZ_ARG_1 aFields, aFields);
 197
 198 // Just a helper to unpack the name and call the real function.
 199 #define MOZ_ATOMIC_BITFIELDS_OFFSET_HELPER2(aBitfields, aField, aFields) \
 200   MOZ_ATOMIC_BITFIELDS_OFFSET(aBitfields, MOZ_ARG_2 aField, aFields)
 201
 202 // To compute the offset of a field, why sum up all the offsets after it
 203 // (inclusive) and subtract that from the total sum itself. We do this to swap
 204 // the rolling sum that |MOZ_ROLL_EACH| gets us from descending to ascending.
 205 #define MOZ_ATOMIC_BITFIELDS_OFFSET(aBitfields, aFieldName, aFields)    \
 206   static const size_t MOZ_CONCAT(aBitfields, aFieldName) =              \
 207       MOZ_CONCAT(aBitfields, _USED_BITS) -                              \
 208       (MOZ_FOR_EACH_SEPARATED(MOZ_ATOMIC_BITFIELDS_FIELD_SIZE, (+), (), \
 209                               aFields));
 210
 211 // Just a more clearly named way of unpacking the size.
 212 #define MOZ_ATOMIC_BITFIELDS_FIELD_SIZE(aArgs) MOZ_ARG_3 aArgs
 213
 214 // Just a helper to unpack the tuple and call the real function.
 215 #define MOZ_ATOMIC_BITFIELDS_FIELD_HELPER(aBitfields, aBitfieldsSize, aArgs) \
 216   MOZ_ATOMIC_BITFIELDS_FIELD(aBitfields, aBitfieldsSize, MOZ_ARG_1 aArgs,    \
 217                              MOZ_ARG_2 aArgs, MOZ_ARG_3 aArgs)
 218
 219 // We need to disable this with coverity because it doesn't like checking that
 220 // booleans are < 2 (because they always are).
 221 #ifdef __COVERITY__
 222 #  define MOZ_ATOMIC_BITFIELDS_STORE_GUARD(aValue, aFieldSize)
 223 #else
 224 #  define MOZ_ATOMIC_BITFIELDS_STORE_GUARD(aValue, aFieldSize) \
 225     MOZ_ASSERT(((uint64_t)aValue) < (1ull << aFieldSize),      \
 226                "Stored value exceeded capacity of bitfield!")
 227 #endif
 228
 229 // Generates the Load and Store methods for each field.
 230 //
 231 // Some comments here because inline macro comments are a pain in the neck:
 232 //
 233 // Most of the locals are forward declared to minimize messy macroified
 234 // type declaration. Also a lot of locals are used to try to make things
 235 // a little more clear, while also avoiding integer promotion issues.
 236 // This is why some locals are literally just copying a value we already have:
 237 // to force it to the right size.
 238 //
 239 // There's an annoying overflow case where a bitfields instance has a field
 240 // that is the same size as the bitfields. Rather than trying to handle that,
 241 // we just static_assert against it.
 242 //
 243 //
 244 // BITMATH EXPLAINED:
 245 //
 246 // For |Load$Name|:
 247 //
 248 //    mask = ((1 << fieldSize) - 1) << offset
 249 //
 250 // If you subtract 1 from a value with 1 bit set you get all 1's below that bit.
 251 // This is perfect for ANDing out |fieldSize| bits. We shift by |offset| to get
 252 // it in the right place.
 253 //
 254 //    value = (aBitfields.load() & mask) >> offset
 255 //
 256 // This sets every bit we're not interested in to 0. Shifting the result by
 257 // |offset| converts the value back to its native format, ready to be cast
 258 // up to an integer type.
 259 //
 260 //
 261 // For |Store$Name|:
 262 //
 263 //    packedValue = (resizedValue << offset) & mask
 264 //
 265 // This converts a native value to the packed format. If the value is in bounds,
 266 // the AND will do nothing. If it's out of bounds (not checked in release),
 267 // then it will cause the value to wrap around by modulo 2^aFieldSize, just like
 268 // a normal uint.
 269 //
 270 //    clearedValue = oldValue & ~mask;
 271 //
 272 // This clears the bits where our field is stored on our bitfield storage by
 273 // ANDing it with an inverted (NOTed) mask.
 274 //
 275 //    newValue = clearedValue | packedValue;
 276 //
 277 // Once we have |packedValue| and |clearedValue| they just need to be ORed
 278 // together to merge the new field value with the old values of all the other
 279 // fields.
 280 //
 281 // This last step is done in a while loop because someone else can modify
 282 // the bits before we have a chance to. If we didn't guard against this,
 283 // our write would undo the write the other thread did. |compare_exchange_weak|
 284 // is specifically designed to handle this. We give it what we expect the
 285 // current value to be, and what we want it to be. If someone else modifies
 286 // the bitfields before us, then we will reload the value and try again.
 287 //
 288 // Note that |compare_exchange_weak| writes back the actual value to the
 289 // "expected" argument (it's passed by-reference), so we don't need to do
 290 // another load in the body of the loop when we fail to write our result.
 291 #define MOZ_ATOMIC_BITFIELDS_FIELD(aBitfields, aBitfieldsSize, aFieldType, \
 292                                    aFieldName, aFieldSize)                 \
 293   static_assert(aBitfieldsSize > aFieldSize,                               \
 294                 #aBitfields ": MOZ_ATOMIC_BITFIELDS field too big");       \
 295   static_assert(std::is_unsigned<aFieldType>(), #aBitfields                \
 296                 ": MOZ_ATOMIC_BITFIELDS doesn't support signed payloads"); \
 297                                                                            \
 298   aFieldType MOZ_CONCAT(Load, aFieldName)() const {                        \
 299     uint##aBitfieldsSize##_t fieldSize, mask, masked, value;               \
 300     size_t offset = MOZ_CONCAT(aBitfields, aFieldName);                    \
 301     fieldSize = aFieldSize;                                                \
 302     mask = ((1ull << fieldSize) - 1ull) << offset;                         \
 303     masked = aBitfields.load() & mask;                                     \
 304     value = (masked >> offset);                                            \
 305     return value;                                                          \
 306   }                                                                        \
 307                                                                            \
 308   void MOZ_CONCAT(Store, aFieldName)(aFieldType aValue) {                  \
 309     MOZ_ATOMIC_BITFIELDS_STORE_GUARD(aValue, aFieldSize);                  \
 310     uint##aBitfieldsSize##_t fieldSize, mask, resizedValue, packedValue,   \
 311         oldValue, clearedValue, newValue;                                  \
 312     size_t offset = MOZ_CONCAT(aBitfields, aFieldName);                    \
 313     fieldSize = aFieldSize;                                                \
 314     mask = ((1ull << fieldSize) - 1ull) << offset;                         \
 315     resizedValue = aValue;                                                 \
 316     packedValue = (resizedValue << offset) & mask;                         \
 317     oldValue = aBitfields.load();                                          \
 318     do {                                                                   \
 319       clearedValue = oldValue & ~mask;                                     \
 320       newValue = clearedValue | packedValue;                               \
 321     } while (!aBitfields.compare_exchange_weak(oldValue, newValue));       \
 322   }
 323
 324 // OK SO THIS IS A GROSS HACK. GCC 10.2 (and below) has a bug[1] where it
 325 // doesn't allow a static array to reference itself in its initializer, so we
 326 // need to create a hacky way to produce a rolling sum of all the offsets.
 327 //
 328 // To do this, we make a tweaked version of |MOZ_FOR_EACH| which instead of
 329 // passing just one argument to |aMacro| it passes the remaining values of
 330 // |aArgs|.
 331 //
 332 // This allows us to expand an input (a, b, c, d) quadratically to:
 333 //
 334 // int sum1 = a + b + c + d;
 335 // int sum2 = b + c + d;
 336 // int sum3 = c + d;
 337 // int sum4 = d;
 338 //
 339 // So all of this is a copy-paste of |MOZ_FOR_EACH| except the definition
 340 // of |MOZ_FOR_EACH_HELPER| no longer extracts an argument with |MOZ_ARG_1|.
 341 // Also this is restricted to 32 arguments just to reduce footprint a little.
 342 //
 343 // If the GCC bug is ever fixed, then this hack can be removed, and we can
 344 // use the non-quadratic version that was originally written[2]. In case
 345 // that link dies, a brief summary of that implementation:
 346 //
 347 // * Associate each field with an index by creating an `enum class` with
 348 //   entries for each field (an existing gecko patten).
 349 //
 350 // * Calculate offsets with a constexpr static array whose initializer
 351 //   self-referentially adds the contents of the previous index to the
 352 //   compute the current one.
 353 //
 354 // * Index into this array with the enum.
 355 //
 356 // [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=97234
 357 // [2]: https://phabricator.services.mozilla.com/D91622?id=346499
 358 #define MOZ_ROLL_EACH_EXPAND_HELPER(...) __VA_ARGS__
 359 #define MOZ_ROLL_EACH_GLUE(a, b) a b
 360 #define MOZ_ROLL_EACH_SEPARATED(aMacro, aSeparator, aFixedArgs, aArgs)       \
 361   MOZ_ROLL_EACH_GLUE(MOZ_PASTE_PREFIX_AND_ARG_COUNT(                         \
 362                          MOZ_ROLL_EACH_, MOZ_ROLL_EACH_EXPAND_HELPER aArgs), \
 363                      (aMacro, aSeparator, aFixedArgs, aArgs))
 364 #define MOZ_ROLL_EACH(aMacro, aFixedArgs, aArgs) \
 365   MOZ_ROLL_EACH_SEPARATED(aMacro, (), aFixedArgs, aArgs)
 366
 367 #define MOZ_ROLL_EACH_HELPER_GLUE(a, b) a b
 368 #define MOZ_ROLL_EACH_HELPER(aMacro, aFixedArgs, aArgs) \
 369   MOZ_ROLL_EACH_HELPER_GLUE(aMacro,                     \
 370                             (MOZ_ROLL_EACH_EXPAND_HELPER aFixedArgs aArgs))
 371
 372 #define MOZ_ROLL_EACH_0(m, s, fa, a)
 373 #define MOZ_ROLL_EACH_1(m, s, fa, a) MOZ_ROLL_EACH_HELPER(m, fa, a)
 374 #define MOZ_ROLL_EACH_2(m, s, fa, a) \
 375   MOZ_ROLL_EACH_HELPER(m, fa, a)     \
 376   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_1(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 377 #define MOZ_ROLL_EACH_3(m, s, fa, a) \
 378   MOZ_ROLL_EACH_HELPER(m, fa, a)     \
 379   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_2(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 380 #define MOZ_ROLL_EACH_4(m, s, fa, a) \
 381   MOZ_ROLL_EACH_HELPER(m, fa, a)     \
 382   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_3(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 383 #define MOZ_ROLL_EACH_5(m, s, fa, a) \
 384   MOZ_ROLL_EACH_HELPER(m, fa, a)     \
 385   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_4(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 386 #define MOZ_ROLL_EACH_6(m, s, fa, a) \
 387   MOZ_ROLL_EACH_HELPER(m, fa, a)     \
 388   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_5(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 389 #define MOZ_ROLL_EACH_7(m, s, fa, a) \
 390   MOZ_ROLL_EACH_HELPER(m, fa, a)     \
 391   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_6(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 392 #define MOZ_ROLL_EACH_8(m, s, fa, a) \
 393   MOZ_ROLL_EACH_HELPER(m, fa, a)     \
 394   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_7(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 395 #define MOZ_ROLL_EACH_9(m, s, fa, a) \
 396   MOZ_ROLL_EACH_HELPER(m, fa, a)     \
 397   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_8(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 398 #define MOZ_ROLL_EACH_10(m, s, fa, a) \
 399   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 400   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_9(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 401 #define MOZ_ROLL_EACH_11(m, s, fa, a) \
 402   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 403   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_10(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 404 #define MOZ_ROLL_EACH_12(m, s, fa, a) \
 405   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 406   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_11(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 407 #define MOZ_ROLL_EACH_13(m, s, fa, a) \
 408   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 409   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_12(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 410 #define MOZ_ROLL_EACH_14(m, s, fa, a) \
 411   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 412   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_13(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 413 #define MOZ_ROLL_EACH_15(m, s, fa, a) \
 414   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 415   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_14(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 416 #define MOZ_ROLL_EACH_16(m, s, fa, a) \
 417   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 418   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_15(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 419 #define MOZ_ROLL_EACH_17(m, s, fa, a) \
 420   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 421   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_16(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 422 #define MOZ_ROLL_EACH_18(m, s, fa, a) \
 423   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 424   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_17(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 425 #define MOZ_ROLL_EACH_19(m, s, fa, a) \
 426   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 427   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_18(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 428 #define MOZ_ROLL_EACH_20(m, s, fa, a) \
 429   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 430   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_19(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 431 #define MOZ_ROLL_EACH_21(m, s, fa, a) \
 432   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 433   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_20(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 434 #define MOZ_ROLL_EACH_22(m, s, fa, a) \
 435   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 436   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_21(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 437 #define MOZ_ROLL_EACH_23(m, s, fa, a) \
 438   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 439   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_22(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 440 #define MOZ_ROLL_EACH_24(m, s, fa, a) \
 441   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 442   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_23(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 443 #define MOZ_ROLL_EACH_25(m, s, fa, a) \
 444   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 445   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_24(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 446 #define MOZ_ROLL_EACH_26(m, s, fa, a) \
 447   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 448   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_25(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 449 #define MOZ_ROLL_EACH_27(m, s, fa, a) \
 450   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 451   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_26(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 452 #define MOZ_ROLL_EACH_28(m, s, fa, a) \
 453   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 454   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_27(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 455 #define MOZ_ROLL_EACH_29(m, s, fa, a) \
 456   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 457   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_28(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 458 #define MOZ_ROLL_EACH_30(m, s, fa, a) \
 459   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 460   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_29(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 461 #define MOZ_ROLL_EACH_31(m, s, fa, a) \
 462   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 463   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_30(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 464 #define MOZ_ROLL_EACH_32(m, s, fa, a) \
 465   MOZ_ROLL_EACH_HELPER(m, fa, a)      \
 466   MOZ_ROLL_EACH_EXPAND_HELPER s MOZ_ROLL_EACH_31(m, s, fa, (MOZ_ARGS_AFTER_1 a))
 467 }  // namespace mozilla
 468 #endif /* mozilla_AtomicBitfields_h */