Atomics.h 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800
  1. /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
  2. /* This Source Code Form is subject to the terms of the Mozilla Public
  3. * License, v. 2.0. If a copy of the MPL was not distributed with this
  4. * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
  5. /*
  6. * Implements (almost always) lock-free atomic operations. The operations here
  7. * are a subset of that which can be found in C++11's <atomic> header, with a
  8. * different API to enforce consistent memory ordering constraints.
  9. *
  10. * Anyone caught using |volatile| for inter-thread memory safety needs to be
  11. * sent a copy of this header and the C++11 standard.
  12. */
  13. #ifndef mozilla_Atomics_h
  14. #define mozilla_Atomics_h
  15. #include "mozilla/Assertions.h"
  16. #include "mozilla/Attributes.h"
  17. #include "mozilla/Compiler.h"
  18. #include "mozilla/TypeTraits.h"
  19. #include <stdint.h>
  20. /*
  21. * Our minimum deployment target on clang/OS X is OS X 10.6, whose SDK
  22. * does not have <atomic>. So be sure to check for <atomic> support
  23. * along with C++0x support.
  24. */
  25. #if defined(_MSC_VER)
  26. # define MOZ_HAVE_CXX11_ATOMICS
  27. #elif defined(__clang__) || defined(__GNUC__)
  28. /*
  29. * Clang doesn't like <atomic> from libstdc++ before 4.7 due to the
  30. * loose typing of the atomic builtins. GCC 4.5 and 4.6 lacks inline
  31. * definitions for unspecialized std::atomic and causes linking errors.
  32. * Therefore, we require at least 4.7.0 for using libstdc++.
  33. *
  34. * libc++ <atomic> is only functional with clang.
  35. */
  36. # if MOZ_USING_LIBSTDCXX && MOZ_LIBSTDCXX_VERSION_AT_LEAST(4, 7, 0)
  37. # define MOZ_HAVE_CXX11_ATOMICS
  38. # elif MOZ_USING_LIBCXX && defined(__clang__)
  39. # define MOZ_HAVE_CXX11_ATOMICS
  40. # endif
  41. #endif
  42. namespace mozilla {
  43. /**
  44. * An enum of memory ordering possibilities for atomics.
  45. *
  46. * Memory ordering is the observable state of distinct values in memory.
  47. * (It's a separate concept from atomicity, which concerns whether an
  48. * operation can ever be observed in an intermediate state. Don't
  49. * conflate the two!) Given a sequence of operations in source code on
  50. * memory, it is *not* always the case that, at all times and on all
  51. * cores, those operations will appear to have occurred in that exact
  52. * sequence. First, the compiler might reorder that sequence, if it
  53. * thinks another ordering will be more efficient. Second, the CPU may
  54. * not expose so consistent a view of memory. CPUs will often perform
  55. * their own instruction reordering, above and beyond that performed by
  56. * the compiler. And each core has its own memory caches, and accesses
  57. * (reads and writes both) to "memory" may only resolve to out-of-date
  58. * cache entries -- not to the "most recently" performed operation in
  59. * some global sense. Any access to a value that may be used by
  60. * multiple threads, potentially across multiple cores, must therefore
  61. * have a memory ordering imposed on it, for all code on all
  62. * threads/cores to have a sufficiently coherent worldview.
  63. *
  64. * http://gcc.gnu.org/wiki/Atomic/GCCMM/AtomicSync and
  65. * http://en.cppreference.com/w/cpp/atomic/memory_order go into more
  66. * detail on all this, including examples of how each mode works.
  67. *
  68. * Note that for simplicity and practicality, not all of the modes in
  69. * C++11 are supported. The missing C++11 modes are either subsumed by
  70. * the modes we provide below, or not relevant for the CPUs we support
  71. * in Gecko. These three modes are confusing enough as it is!
  72. */
  73. enum MemoryOrdering {
  74. /*
  75. * Relaxed ordering is the simplest memory ordering: none at all.
  76. * When the result of a write is observed, nothing may be inferred
  77. * about other memory. Writes ostensibly performed "before" on the
  78. * writing thread may not yet be visible. Writes performed "after" on
  79. * the writing thread may already be visible, if the compiler or CPU
  80. * reordered them. (The latter can happen if reads and/or writes get
  81. * held up in per-processor caches.) Relaxed ordering means
  82. * operations can always use cached values (as long as the actual
  83. * updates to atomic values actually occur, correctly, eventually), so
  84. * it's usually the fastest sort of atomic access. For this reason,
  85. * *it's also the most dangerous kind of access*.
  86. *
  87. * Relaxed ordering is good for things like process-wide statistics
  88. * counters that don't need to be consistent with anything else, so
  89. * long as updates themselves are atomic. (And so long as any
  90. * observations of that value can tolerate being out-of-date -- if you
  91. * need some sort of up-to-date value, you need some sort of other
  92. * synchronizing operation.) It's *not* good for locks, mutexes,
  93. * reference counts, etc. that mediate access to other memory, or must
  94. * be observably consistent with other memory.
  95. *
  96. * x86 architectures don't take advantage of the optimization
  97. * opportunities that relaxed ordering permits. Thus it's possible
  98. * that using relaxed ordering will "work" on x86 but fail elsewhere
  99. * (ARM, say, which *does* implement non-sequentially-consistent
  100. * relaxed ordering semantics). Be extra-careful using relaxed
  101. * ordering if you can't easily test non-x86 architectures!
  102. */
  103. Relaxed,
  104. /*
  105. * When an atomic value is updated with ReleaseAcquire ordering, and
  106. * that new value is observed with ReleaseAcquire ordering, prior
  107. * writes (atomic or not) are also observable. What ReleaseAcquire
  108. * *doesn't* give you is any observable ordering guarantees for
  109. * ReleaseAcquire-ordered operations on different objects. For
  110. * example, if there are two cores that each perform ReleaseAcquire
  111. * operations on separate objects, each core may or may not observe
  112. * the operations made by the other core. The only way the cores can
  113. * be synchronized with ReleaseAcquire is if they both
  114. * ReleaseAcquire-access the same object. This implies that you can't
  115. * necessarily describe some global total ordering of ReleaseAcquire
  116. * operations.
  117. *
  118. * ReleaseAcquire ordering is good for (as the name implies) atomic
  119. * operations on values controlling ownership of things: reference
  120. * counts, mutexes, and the like. However, if you are thinking about
  121. * using these to implement your own locks or mutexes, you should take
  122. * a good, hard look at actual lock or mutex primitives first.
  123. */
  124. ReleaseAcquire,
  125. /*
  126. * When an atomic value is updated with SequentiallyConsistent
  127. * ordering, all writes observable when the update is observed, just
  128. * as with ReleaseAcquire ordering. But, furthermore, a global total
  129. * ordering of SequentiallyConsistent operations *can* be described.
  130. * For example, if two cores perform SequentiallyConsistent operations
  131. * on separate objects, one core will observably perform its update
  132. * (and all previous operations will have completed), then the other
  133. * core will observably perform its update (and all previous
  134. * operations will have completed). (Although those previous
  135. * operations aren't themselves ordered -- they could be intermixed,
  136. * or ordered if they occur on atomic values with ordering
  137. * requirements.) SequentiallyConsistent is the *simplest and safest*
  138. * ordering of atomic operations -- it's always as if one operation
  139. * happens, then another, then another, in some order -- and every
  140. * core observes updates to happen in that single order. Because it
  141. * has the most synchronization requirements, operations ordered this
  142. * way also tend to be slowest.
  143. *
  144. * SequentiallyConsistent ordering can be desirable when multiple
  145. * threads observe objects, and they all have to agree on the
  146. * observable order of changes to them. People expect
  147. * SequentiallyConsistent ordering, even if they shouldn't, when
  148. * writing code, atomic or otherwise. SequentiallyConsistent is also
  149. * the ordering of choice when designing lockless data structures. If
  150. * you don't know what order to use, use this one.
  151. */
  152. SequentiallyConsistent,
  153. };
  154. } // namespace mozilla
  155. // Build up the underlying intrinsics.
  156. #ifdef MOZ_HAVE_CXX11_ATOMICS
  157. # include <atomic>
  158. namespace mozilla {
  159. namespace detail {
  160. /*
  161. * We provide CompareExchangeFailureOrder to work around a bug in some
  162. * versions of GCC's <atomic> header. See bug 898491.
  163. */
  164. template<MemoryOrdering Order> struct AtomicOrderConstraints;
  165. template<>
  166. struct AtomicOrderConstraints<Relaxed>
  167. {
  168. static const std::memory_order AtomicRMWOrder = std::memory_order_relaxed;
  169. static const std::memory_order LoadOrder = std::memory_order_relaxed;
  170. static const std::memory_order StoreOrder = std::memory_order_relaxed;
  171. static const std::memory_order CompareExchangeFailureOrder =
  172. std::memory_order_relaxed;
  173. };
  174. template<>
  175. struct AtomicOrderConstraints<ReleaseAcquire>
  176. {
  177. static const std::memory_order AtomicRMWOrder = std::memory_order_acq_rel;
  178. static const std::memory_order LoadOrder = std::memory_order_acquire;
  179. static const std::memory_order StoreOrder = std::memory_order_release;
  180. static const std::memory_order CompareExchangeFailureOrder =
  181. std::memory_order_acquire;
  182. };
  183. template<>
  184. struct AtomicOrderConstraints<SequentiallyConsistent>
  185. {
  186. static const std::memory_order AtomicRMWOrder = std::memory_order_seq_cst;
  187. static const std::memory_order LoadOrder = std::memory_order_seq_cst;
  188. static const std::memory_order StoreOrder = std::memory_order_seq_cst;
  189. static const std::memory_order CompareExchangeFailureOrder =
  190. std::memory_order_seq_cst;
  191. };
  192. template<typename T, MemoryOrdering Order>
  193. struct IntrinsicBase
  194. {
  195. typedef std::atomic<T> ValueType;
  196. typedef AtomicOrderConstraints<Order> OrderedOp;
  197. };
  198. template<typename T, MemoryOrdering Order>
  199. struct IntrinsicMemoryOps : public IntrinsicBase<T, Order>
  200. {
  201. typedef IntrinsicBase<T, Order> Base;
  202. static T load(const typename Base::ValueType& aPtr)
  203. {
  204. return aPtr.load(Base::OrderedOp::LoadOrder);
  205. }
  206. static void store(typename Base::ValueType& aPtr, T aVal)
  207. {
  208. aPtr.store(aVal, Base::OrderedOp::StoreOrder);
  209. }
  210. static T exchange(typename Base::ValueType& aPtr, T aVal)
  211. {
  212. return aPtr.exchange(aVal, Base::OrderedOp::AtomicRMWOrder);
  213. }
  214. static bool compareExchange(typename Base::ValueType& aPtr,
  215. T aOldVal, T aNewVal)
  216. {
  217. return aPtr.compare_exchange_strong(aOldVal, aNewVal,
  218. Base::OrderedOp::AtomicRMWOrder,
  219. Base::OrderedOp::CompareExchangeFailureOrder);
  220. }
  221. };
  222. template<typename T, MemoryOrdering Order>
  223. struct IntrinsicAddSub : public IntrinsicBase<T, Order>
  224. {
  225. typedef IntrinsicBase<T, Order> Base;
  226. static T add(typename Base::ValueType& aPtr, T aVal)
  227. {
  228. return aPtr.fetch_add(aVal, Base::OrderedOp::AtomicRMWOrder);
  229. }
  230. static T sub(typename Base::ValueType& aPtr, T aVal)
  231. {
  232. return aPtr.fetch_sub(aVal, Base::OrderedOp::AtomicRMWOrder);
  233. }
  234. };
  235. template<typename T, MemoryOrdering Order>
  236. struct IntrinsicAddSub<T*, Order> : public IntrinsicBase<T*, Order>
  237. {
  238. typedef IntrinsicBase<T*, Order> Base;
  239. static T* add(typename Base::ValueType& aPtr, ptrdiff_t aVal)
  240. {
  241. return aPtr.fetch_add(aVal, Base::OrderedOp::AtomicRMWOrder);
  242. }
  243. static T* sub(typename Base::ValueType& aPtr, ptrdiff_t aVal)
  244. {
  245. return aPtr.fetch_sub(aVal, Base::OrderedOp::AtomicRMWOrder);
  246. }
  247. };
  248. template<typename T, MemoryOrdering Order>
  249. struct IntrinsicIncDec : public IntrinsicAddSub<T, Order>
  250. {
  251. typedef IntrinsicBase<T, Order> Base;
  252. static T inc(typename Base::ValueType& aPtr)
  253. {
  254. return IntrinsicAddSub<T, Order>::add(aPtr, 1);
  255. }
  256. static T dec(typename Base::ValueType& aPtr)
  257. {
  258. return IntrinsicAddSub<T, Order>::sub(aPtr, 1);
  259. }
  260. };
  261. template<typename T, MemoryOrdering Order>
  262. struct AtomicIntrinsics : public IntrinsicMemoryOps<T, Order>,
  263. public IntrinsicIncDec<T, Order>
  264. {
  265. typedef IntrinsicBase<T, Order> Base;
  266. static T or_(typename Base::ValueType& aPtr, T aVal)
  267. {
  268. return aPtr.fetch_or(aVal, Base::OrderedOp::AtomicRMWOrder);
  269. }
  270. static T xor_(typename Base::ValueType& aPtr, T aVal)
  271. {
  272. return aPtr.fetch_xor(aVal, Base::OrderedOp::AtomicRMWOrder);
  273. }
  274. static T and_(typename Base::ValueType& aPtr, T aVal)
  275. {
  276. return aPtr.fetch_and(aVal, Base::OrderedOp::AtomicRMWOrder);
  277. }
  278. };
  279. template<typename T, MemoryOrdering Order>
  280. struct AtomicIntrinsics<T*, Order>
  281. : public IntrinsicMemoryOps<T*, Order>, public IntrinsicIncDec<T*, Order>
  282. {
  283. };
  284. template<typename T>
  285. struct ToStorageTypeArgument
  286. {
  287. static constexpr T convert (T aT) { return aT; }
  288. };
  289. } // namespace detail
  290. } // namespace mozilla
  291. #elif defined(__GNUC__)
  292. namespace mozilla {
  293. namespace detail {
  294. /*
  295. * The __sync_* family of intrinsics is documented here:
  296. *
  297. * http://gcc.gnu.org/onlinedocs/gcc-4.6.4/gcc/Atomic-Builtins.html
  298. *
  299. * While these intrinsics are deprecated in favor of the newer __atomic_*
  300. * family of intrincs:
  301. *
  302. * http://gcc.gnu.org/onlinedocs/gcc-4.7.3/gcc/_005f_005fatomic-Builtins.html
  303. *
  304. * any GCC version that supports the __atomic_* intrinsics will also support
  305. * the <atomic> header and so will be handled above. We provide a version of
  306. * atomics using the __sync_* intrinsics to support older versions of GCC.
  307. *
  308. * All __sync_* intrinsics that we use below act as full memory barriers, for
  309. * both compiler and hardware reordering, except for __sync_lock_test_and_set,
  310. * which is a only an acquire barrier. When we call __sync_lock_test_and_set,
  311. * we add a barrier above it as appropriate.
  312. */
  313. template<MemoryOrdering Order> struct Barrier;
  314. /*
  315. * Some processors (in particular, x86) don't require quite so many calls to
  316. * __sync_sychronize as our specializations of Barrier produce. If
  317. * performance turns out to be an issue, defining these specializations
  318. * on a per-processor basis would be a good first tuning step.
  319. */
  320. template<>
  321. struct Barrier<Relaxed>
  322. {
  323. static void beforeLoad() {}
  324. static void afterLoad() {}
  325. static void beforeStore() {}
  326. static void afterStore() {}
  327. };
  328. template<>
  329. struct Barrier<ReleaseAcquire>
  330. {
  331. static void beforeLoad() {}
  332. static void afterLoad() { __sync_synchronize(); }
  333. static void beforeStore() { __sync_synchronize(); }
  334. static void afterStore() {}
  335. };
  336. template<>
  337. struct Barrier<SequentiallyConsistent>
  338. {
  339. static void beforeLoad() { __sync_synchronize(); }
  340. static void afterLoad() { __sync_synchronize(); }
  341. static void beforeStore() { __sync_synchronize(); }
  342. static void afterStore() { __sync_synchronize(); }
  343. };
  344. template<typename T, bool TIsEnum = IsEnum<T>::value>
  345. struct AtomicStorageType
  346. {
  347. // For non-enums, just use the type directly.
  348. typedef T Type;
  349. };
  350. template<typename T>
  351. struct AtomicStorageType<T, true>
  352. : Conditional<sizeof(T) == 4, uint32_t, uint64_t>
  353. {
  354. static_assert(sizeof(T) == 4 || sizeof(T) == 8,
  355. "wrong type computed in conditional above");
  356. };
  357. template<typename T, MemoryOrdering Order>
  358. struct IntrinsicMemoryOps
  359. {
  360. typedef typename AtomicStorageType<T>::Type ValueType;
  361. static T load(const ValueType& aPtr)
  362. {
  363. Barrier<Order>::beforeLoad();
  364. T val = T(aPtr);
  365. Barrier<Order>::afterLoad();
  366. return val;
  367. }
  368. static void store(ValueType& aPtr, T aVal)
  369. {
  370. Barrier<Order>::beforeStore();
  371. aPtr = ValueType(aVal);
  372. Barrier<Order>::afterStore();
  373. }
  374. static T exchange(ValueType& aPtr, T aVal)
  375. {
  376. // __sync_lock_test_and_set is only an acquire barrier; loads and stores
  377. // can't be moved up from after to before it, but they can be moved down
  378. // from before to after it. We may want a stricter ordering, so we need
  379. // an explicit barrier.
  380. Barrier<Order>::beforeStore();
  381. return T(__sync_lock_test_and_set(&aPtr, ValueType(aVal)));
  382. }
  383. static bool compareExchange(ValueType& aPtr, T aOldVal, T aNewVal)
  384. {
  385. return __sync_bool_compare_and_swap(&aPtr, ValueType(aOldVal), ValueType(aNewVal));
  386. }
  387. };
  388. template<typename T, MemoryOrdering Order>
  389. struct IntrinsicAddSub
  390. : public IntrinsicMemoryOps<T, Order>
  391. {
  392. typedef IntrinsicMemoryOps<T, Order> Base;
  393. typedef typename Base::ValueType ValueType;
  394. static T add(ValueType& aPtr, T aVal)
  395. {
  396. return T(__sync_fetch_and_add(&aPtr, ValueType(aVal)));
  397. }
  398. static T sub(ValueType& aPtr, T aVal)
  399. {
  400. return T(__sync_fetch_and_sub(&aPtr, ValueType(aVal)));
  401. }
  402. };
  403. template<typename T, MemoryOrdering Order>
  404. struct IntrinsicAddSub<T*, Order>
  405. : public IntrinsicMemoryOps<T*, Order>
  406. {
  407. typedef IntrinsicMemoryOps<T*, Order> Base;
  408. typedef typename Base::ValueType ValueType;
  409. /*
  410. * The reinterpret_casts are needed so that
  411. * __sync_fetch_and_{add,sub} will properly type-check.
  412. *
  413. * Also, these functions do not provide standard semantics for
  414. * pointer types, so we need to adjust the addend.
  415. */
  416. static ValueType add(ValueType& aPtr, ptrdiff_t aVal)
  417. {
  418. ValueType amount = reinterpret_cast<ValueType>(aVal * sizeof(T));
  419. return __sync_fetch_and_add(&aPtr, amount);
  420. }
  421. static ValueType sub(ValueType& aPtr, ptrdiff_t aVal)
  422. {
  423. ValueType amount = reinterpret_cast<ValueType>(aVal * sizeof(T));
  424. return __sync_fetch_and_sub(&aPtr, amount);
  425. }
  426. };
  427. template<typename T, MemoryOrdering Order>
  428. struct IntrinsicIncDec : public IntrinsicAddSub<T, Order>
  429. {
  430. typedef IntrinsicAddSub<T, Order> Base;
  431. typedef typename Base::ValueType ValueType;
  432. static T inc(ValueType& aPtr) { return Base::add(aPtr, 1); }
  433. static T dec(ValueType& aPtr) { return Base::sub(aPtr, 1); }
  434. };
  435. template<typename T, MemoryOrdering Order>
  436. struct AtomicIntrinsics : public IntrinsicIncDec<T, Order>
  437. {
  438. static T or_( T& aPtr, T aVal) { return __sync_fetch_and_or(&aPtr, aVal); }
  439. static T xor_(T& aPtr, T aVal) { return __sync_fetch_and_xor(&aPtr, aVal); }
  440. static T and_(T& aPtr, T aVal) { return __sync_fetch_and_and(&aPtr, aVal); }
  441. };
  442. template<typename T, MemoryOrdering Order>
  443. struct AtomicIntrinsics<T*, Order> : public IntrinsicIncDec<T*, Order>
  444. {
  445. };
  446. template<typename T, bool TIsEnum = IsEnum<T>::value>
  447. struct ToStorageTypeArgument
  448. {
  449. typedef typename AtomicStorageType<T>::Type ResultType;
  450. static constexpr ResultType convert (T aT) { return ResultType(aT); }
  451. };
  452. template<typename T>
  453. struct ToStorageTypeArgument<T, false>
  454. {
  455. static constexpr T convert (T aT) { return aT; }
  456. };
  457. } // namespace detail
  458. } // namespace mozilla
  459. #else
  460. # error "Atomic compiler intrinsics are not supported on your platform"
  461. #endif
  462. namespace mozilla {
  463. namespace detail {
  464. template<typename T, MemoryOrdering Order>
  465. class AtomicBase
  466. {
  467. static_assert(sizeof(T) == 4 || sizeof(T) == 8,
  468. "mozilla/Atomics.h only supports 32-bit and 64-bit types");
  469. protected:
  470. typedef typename detail::AtomicIntrinsics<T, Order> Intrinsics;
  471. typedef typename Intrinsics::ValueType ValueType;
  472. ValueType mValue;
  473. public:
  474. constexpr AtomicBase() : mValue() {}
  475. explicit constexpr AtomicBase(T aInit)
  476. : mValue(ToStorageTypeArgument<T>::convert(aInit))
  477. {}
  478. // Note: we can't provide operator T() here because Atomic<bool> inherits
  479. // from AtomcBase with T=uint32_t and not T=bool. If we implemented
  480. // operator T() here, it would cause errors when comparing Atomic<bool> with
  481. // a regular bool.
  482. T operator=(T aVal)
  483. {
  484. Intrinsics::store(mValue, aVal);
  485. return aVal;
  486. }
  487. /**
  488. * Performs an atomic swap operation. aVal is stored and the previous
  489. * value of this variable is returned.
  490. */
  491. T exchange(T aVal)
  492. {
  493. return Intrinsics::exchange(mValue, aVal);
  494. }
  495. /**
  496. * Performs an atomic compare-and-swap operation and returns true if it
  497. * succeeded. This is equivalent to atomically doing
  498. *
  499. * if (mValue == aOldValue) {
  500. * mValue = aNewValue;
  501. * return true;
  502. * } else {
  503. * return false;
  504. * }
  505. */
  506. bool compareExchange(T aOldValue, T aNewValue)
  507. {
  508. return Intrinsics::compareExchange(mValue, aOldValue, aNewValue);
  509. }
  510. private:
  511. template<MemoryOrdering AnyOrder>
  512. AtomicBase(const AtomicBase<T, AnyOrder>& aCopy) = delete;
  513. };
  514. template<typename T, MemoryOrdering Order>
  515. class AtomicBaseIncDec : public AtomicBase<T, Order>
  516. {
  517. typedef typename detail::AtomicBase<T, Order> Base;
  518. public:
  519. constexpr AtomicBaseIncDec() : Base() {}
  520. explicit constexpr AtomicBaseIncDec(T aInit) : Base(aInit) {}
  521. using Base::operator=;
  522. operator T() const { return Base::Intrinsics::load(Base::mValue); }
  523. T operator++(int) { return Base::Intrinsics::inc(Base::mValue); }
  524. T operator--(int) { return Base::Intrinsics::dec(Base::mValue); }
  525. T operator++() { return Base::Intrinsics::inc(Base::mValue) + 1; }
  526. T operator--() { return Base::Intrinsics::dec(Base::mValue) - 1; }
  527. private:
  528. template<MemoryOrdering AnyOrder>
  529. AtomicBaseIncDec(const AtomicBaseIncDec<T, AnyOrder>& aCopy) = delete;
  530. };
  531. } // namespace detail
  532. /**
  533. * A wrapper for a type that enforces that all memory accesses are atomic.
  534. *
  535. * In general, where a variable |T foo| exists, |Atomic<T> foo| can be used in
  536. * its place. Implementations for integral and pointer types are provided
  537. * below.
  538. *
  539. * Atomic accesses are sequentially consistent by default. You should
  540. * use the default unless you are tall enough to ride the
  541. * memory-ordering roller coaster (if you're not sure, you aren't) and
  542. * you have a compelling reason to do otherwise.
  543. *
  544. * There is one exception to the case of atomic memory accesses: providing an
  545. * initial value of the atomic value is not guaranteed to be atomic. This is a
  546. * deliberate design choice that enables static atomic variables to be declared
  547. * without introducing extra static constructors.
  548. */
  549. template<typename T,
  550. MemoryOrdering Order = SequentiallyConsistent,
  551. typename Enable = void>
  552. class Atomic;
  553. /**
  554. * Atomic<T> implementation for integral types.
  555. *
  556. * In addition to atomic store and load operations, compound assignment and
  557. * increment/decrement operators are implemented which perform the
  558. * corresponding read-modify-write operation atomically. Finally, an atomic
  559. * swap method is provided.
  560. */
  561. template<typename T, MemoryOrdering Order>
  562. class Atomic<T, Order, typename EnableIf<IsIntegral<T>::value &&
  563. !IsSame<T, bool>::value>::Type>
  564. : public detail::AtomicBaseIncDec<T, Order>
  565. {
  566. typedef typename detail::AtomicBaseIncDec<T, Order> Base;
  567. public:
  568. constexpr Atomic() : Base() {}
  569. explicit constexpr Atomic(T aInit) : Base(aInit) {}
  570. using Base::operator=;
  571. T operator+=(T aDelta)
  572. {
  573. return Base::Intrinsics::add(Base::mValue, aDelta) + aDelta;
  574. }
  575. T operator-=(T aDelta)
  576. {
  577. return Base::Intrinsics::sub(Base::mValue, aDelta) - aDelta;
  578. }
  579. T operator|=(T aVal)
  580. {
  581. return Base::Intrinsics::or_(Base::mValue, aVal) | aVal;
  582. }
  583. T operator^=(T aVal)
  584. {
  585. return Base::Intrinsics::xor_(Base::mValue, aVal) ^ aVal;
  586. }
  587. T operator&=(T aVal)
  588. {
  589. return Base::Intrinsics::and_(Base::mValue, aVal) & aVal;
  590. }
  591. private:
  592. Atomic(Atomic<T, Order>& aOther) = delete;
  593. };
  594. /**
  595. * Atomic<T> implementation for pointer types.
  596. *
  597. * An atomic compare-and-swap primitive for pointer variables is provided, as
  598. * are atomic increment and decement operators. Also provided are the compound
  599. * assignment operators for addition and subtraction. Atomic swap (via
  600. * exchange()) is included as well.
  601. */
  602. template<typename T, MemoryOrdering Order>
  603. class Atomic<T*, Order> : public detail::AtomicBaseIncDec<T*, Order>
  604. {
  605. typedef typename detail::AtomicBaseIncDec<T*, Order> Base;
  606. public:
  607. constexpr Atomic() : Base() {}
  608. explicit constexpr Atomic(T* aInit) : Base(aInit) {}
  609. using Base::operator=;
  610. T* operator+=(ptrdiff_t aDelta)
  611. {
  612. return Base::Intrinsics::add(Base::mValue, aDelta) + aDelta;
  613. }
  614. T* operator-=(ptrdiff_t aDelta)
  615. {
  616. return Base::Intrinsics::sub(Base::mValue, aDelta) - aDelta;
  617. }
  618. private:
  619. Atomic(Atomic<T*, Order>& aOther) = delete;
  620. };
  621. /**
  622. * Atomic<T> implementation for enum types.
  623. *
  624. * The atomic store and load operations and the atomic swap method is provided.
  625. */
  626. template<typename T, MemoryOrdering Order>
  627. class Atomic<T, Order, typename EnableIf<IsEnum<T>::value>::Type>
  628. : public detail::AtomicBase<T, Order>
  629. {
  630. typedef typename detail::AtomicBase<T, Order> Base;
  631. public:
  632. constexpr Atomic() : Base() {}
  633. explicit constexpr Atomic(T aInit) : Base(aInit) {}
  634. operator T() const { return T(Base::Intrinsics::load(Base::mValue)); }
  635. using Base::operator=;
  636. private:
  637. Atomic(Atomic<T, Order>& aOther) = delete;
  638. };
  639. /**
  640. * Atomic<T> implementation for boolean types.
  641. *
  642. * The atomic store and load operations and the atomic swap method is provided.
  643. *
  644. * Note:
  645. *
  646. * - sizeof(Atomic<bool>) != sizeof(bool) for some implementations of
  647. * bool and/or some implementations of std::atomic. This is allowed in
  648. * [atomic.types.generic]p9.
  649. *
  650. * - It's not obvious whether the 8-bit atomic functions on Windows are always
  651. * inlined or not. If they are not inlined, the corresponding functions in the
  652. * runtime library are not available on Windows XP. This is why we implement
  653. * Atomic<bool> with an underlying type of uint32_t.
  654. */
  655. template<MemoryOrdering Order>
  656. class Atomic<bool, Order>
  657. : protected detail::AtomicBase<uint32_t, Order>
  658. {
  659. typedef typename detail::AtomicBase<uint32_t, Order> Base;
  660. public:
  661. constexpr Atomic() : Base() {}
  662. explicit constexpr Atomic(bool aInit) : Base(aInit) {}
  663. // We provide boolean wrappers for the underlying AtomicBase methods.
  664. MOZ_IMPLICIT operator bool() const
  665. {
  666. return Base::Intrinsics::load(Base::mValue);
  667. }
  668. bool operator=(bool aVal)
  669. {
  670. return Base::operator=(aVal);
  671. }
  672. bool exchange(bool aVal)
  673. {
  674. return Base::exchange(aVal);
  675. }
  676. bool compareExchange(bool aOldValue, bool aNewValue)
  677. {
  678. return Base::compareExchange(aOldValue, aNewValue);
  679. }
  680. private:
  681. Atomic(Atomic<bool, Order>& aOther) = delete;
  682. };
  683. } // namespace mozilla
  684. #endif /* mozilla_Atomics_h */