26#pragma GCC diagnostic push
27#pragma GCC diagnostic ignored "-Wconversion"
29#ifdef MCDBGQ_USE_RELACY
30#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
35#include "TargetConditionals.h"
38#ifdef MCDBGQ_USE_RELACY
39#include "relacy/relacy_std.hpp"
40#include "relacy_shims.h"
71#if defined(MCDBGQ_USE_RELACY)
78#elif defined(_WIN32) || defined(__WINDOWS__) || defined(__WIN32__)
81extern "C" __declspec(dllimport)
unsigned long __stdcall GetCurrentThreadId(
void);
83 static_assert(
sizeof(
unsigned long) ==
sizeof(std::uint32_t),
"Expected size of unsigned long to be 32 bits on Windows");
89#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE)
91 static_assert(
sizeof(std::thread::id) == 4 ||
sizeof(std::thread::id) == 8,
"std::thread::id is expected to be either 4 or 8 bytes");
101 template<std::
size_t>
struct thread_id_size { };
103 template<>
struct thread_id_size<4> {
typedef std::uint32_t numeric_t; };
105 template<>
struct thread_id_size<8> {
typedef std::uint64_t numeric_t; };
108 template<>
struct thread_id_converter<
thread_id_t> {
124 return std::hash<std::thread::id>()(x);
136#if defined(__GNUC__) || defined(__INTEL_COMPILER)
137#define MOODYCAMEL_THREADLOCAL __thread
138#elif defined(_MSC_VER)
139#define MOODYCAMEL_THREADLOCAL __declspec(thread)
142#define MOODYCAMEL_THREADLOCAL thread_local
153#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED
154#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__))
155#define MOODYCAMEL_EXCEPTIONS_ENABLED
163#if (defined(LIBCARLA_NO_EXCEPTIONS) && defined(MOODYCAMEL_EXCEPTIONS_ENABLED))
164# undef MOODYCAMEL_EXCEPTIONS_ENABLED
168#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED
169#define MOODYCAMEL_TRY try
171#define MOODYCAMEL_CATCH(...) catch(__VA_ARGS__)
172#define MOODYCAMEL_RETHROW throw
173#define MOODYCAMEL_THROW(expr) ::carla::throw_exception(expr)
175#define MOODYCAMEL_TRY if (true)
177#define MOODYCAMEL_CATCH(...) else if (false)
179#define MOODYCAMEL_RETHROW
180#define MOODYCAMEL_THROW(expr) ::carla::throw_exception(expr)
186#ifndef MOODYCAMEL_NOEXCEPT
187#if !defined(MOODYCAMEL_EXCEPTIONS_ENABLED)
188#define MOODYCAMEL_NOEXCEPT
189#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) true
190#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) true
191#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1800
194#define MOODYCAMEL_NOEXCEPT _NOEXCEPT
195#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference<valueType>::value && std::is_move_constructible<type>::value ? std::is_trivially_move_constructible<type>::value : std::is_trivially_copy_constructible<type>::value)
196#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference<valueType>::value && std::is_move_assignable<type>::value ? std::is_trivially_move_assignable<type>::value || std::is_nothrow_move_assignable<type>::value : std::is_trivially_copy_assignable<type>::value || std::is_nothrow_copy_assignable<type>::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr))
197#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1900
198#define MOODYCAMEL_NOEXCEPT _NOEXCEPT
199#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference<valueType>::value && std::is_move_constructible<type>::value ? std::is_trivially_move_constructible<type>::value || std::is_nothrow_move_constructible<type>::value : std::is_trivially_copy_constructible<type>::value || std::is_nothrow_copy_constructible<type>::value)
200#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference<valueType>::value && std::is_move_assignable<type>::value ? std::is_trivially_move_assignable<type>::value || std::is_nothrow_move_assignable<type>::value : std::is_trivially_copy_assignable<type>::value || std::is_nothrow_copy_assignable<type>::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr))
202#define MOODYCAMEL_NOEXCEPT noexcept
203#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) noexcept(expr)
204#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) noexcept(expr)
208#ifndef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
209#ifdef MCDBGQ_USE_RELACY
210#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
215#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__)
224#ifndef MOODYCAMEL_DELETE_FUNCTION
225#if defined(_MSC_VER) && _MSC_VER < 1800
226#define MOODYCAMEL_DELETE_FUNCTION
228#define MOODYCAMEL_DELETE_FUNCTION = delete
235 static inline bool (
likely)(
bool x) {
return __builtin_expect((x),
true); }
238 static inline bool (
unlikely)(
bool x) {
return __builtin_expect((x),
false); }
241 static inline bool (
likely)(
bool x) {
return x; }
242 static inline bool (
unlikely)(
bool x) {
return x; }
246#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
247#include "internal/concurrentqueue_internal_debug.h"
256 static_assert(std::is_integral<T>::value,
"const_numeric_max can only be used with integers");
258 static const T
value = std::numeric_limits<T>::is_signed
260 ? (
static_cast<T
>(1) << (
sizeof(T) * CHAR_BIT - 1)) -
static_cast<T
>(1)
261 : static_cast<T>(-1);
266#if defined(__GLIBCXX__)
333#ifndef MCDBGQ_USE_RELACY
336#if defined(malloc) || defined(free)
339 static inline void* WORKAROUND_malloc(
size_t size) {
return malloc(size); }
340 static inline void WORKAROUND_free(
void* ptr) {
return free(ptr); }
341 static inline void* (
malloc)(
size_t size) {
return WORKAROUND_malloc(size); }
342 static inline void (
free)(
void* ptr) {
return WORKAROUND_free(ptr); }
349#ifndef MCDBGQ_USE_RELACY
352#if defined(malloc) || defined(free)
355 static inline void* WORKAROUND_malloc(
size_t size) {
return malloc(size); }
356 static inline void WORKAROUND_free(
void* ptr) {
return free(ptr); }
357 static inline void* (
malloc)(
size_t size) {
return WORKAROUND_malloc(size); }
358 static inline void (
free)(
void* ptr) {
return WORKAROUND_free(ptr); }
360 static inline void*
malloc(
size_t size) {
return std::malloc(size); }
361 static inline void free(
void* ptr) {
return std::free(ptr); }
366 static inline void*
malloc(
size_t size) {
return rl::rl_malloc(size, $); }
367 static inline void free(
void* ptr) {
return rl::rl_free(ptr, $); }
402 static inline std::uint32_t
hash(std::uint32_t h)
412 return h ^ (h >> 16);
416 static inline std::uint64_t
hash(std::uint64_t h)
419 h *= 0xff51afd7ed558ccd;
421 h *= 0xc4ceb9fe1a85ec53;
422 return h ^ (h >> 33);
429 static_assert(
sizeof(
thread_id_t) <= 8,
"Expected a platform where thread IDs are at most 64-bit values");
440#pragma warning(disable: 4554)
442 static_assert(std::is_integral<T>::value && !std::numeric_limits<T>::is_signed,
"circular_less_than is intended to be used only with unsigned integer types");
443 return static_cast<T
>(a - b) >
static_cast<T
>(
static_cast<T
>(1) <<
static_cast<T
>(
sizeof(T) * CHAR_BIT - 1));
453 const std::size_t alignment = std::alignment_of<U>::value;
454 return ptr + (alignment - (
reinterpret_cast<std::uintptr_t
>(ptr) % alignment)) % alignment;
460 static_assert(std::is_integral<T>::value && !std::numeric_limits<T>::is_signed,
"ceil_to_pow_2 is intended to be used only with unsigned integer types");
467 for (std::size_t i = 1; i <
sizeof(T); i <<= 1) {
475 static inline void swap_relaxed(std::atomic<T>& left, std::atomic<T>& right)
477 T temp = std::move(left.load(std::memory_order_relaxed));
478 left.store(std::move(right.load(std::memory_order_relaxed)), std::memory_order_relaxed);
479 right.store(std::move(temp), std::memory_order_relaxed);
483 static inline T
const&
nomove(T
const& x)
488 template<
bool Enable>
492 static inline T
const&
eval(T
const& x)
504 ->
decltype(std::forward<U>(x))
506 return std::forward<U>(x);
510 template<
typename It>
517#if defined(__clang__) || !defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)
524#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
525#ifdef MCDBGQ_USE_RELACY
526 typedef RelacyThreadExitListener ThreadExitListener;
527 typedef RelacyThreadExitNotifier ThreadExitNotifier;
529 struct ThreadExitListener
531 typedef void (*callback_t)(
void*);
535 ThreadExitListener* next;
540 class ThreadExitNotifier
545 static void subscribe(ThreadExitListener* listener)
547 auto& tlsInst = instance();
548 listener->next = tlsInst.tail;
549 tlsInst.tail = listener;
553 static void unsubscribe(ThreadExitListener* listener)
555 auto& tlsInst = instance();
556 ThreadExitListener** prev = &tlsInst.tail;
557 for (
auto ptr = tlsInst.tail; ptr !=
nullptr; ptr = ptr->next) {
558 if (ptr == listener) {
568 ThreadExitNotifier() : tail(nullptr) { }
572 ~ThreadExitNotifier()
575 assert(
this == &instance() &&
"If this assert fails, you likely have a buggy compiler! Change the preprocessor conditions such that MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is no longer defined.");
576 for (
auto ptr = tail; ptr !=
nullptr; ptr = ptr->next) {
577 ptr->callback(ptr->userData);
582 static inline ThreadExitNotifier& instance()
584 static thread_local ThreadExitNotifier notifier;
589 ThreadExitListener* tail;
618 template<
typename T,
typename Traits>
623 template<
typename T,
typename Traits>
633 other.producer =
nullptr;
650 std::swap(
producer, other.producer);
654 if (other.producer !=
nullptr) {
693 template<
typename T,
typename Traits>
697 template<
typename T,
typename Traits>
742template<
typename T,
typename Traits>
748template<
typename T,
typename Traits = ConcurrentQueueDefaultTraits>
762 static const size_t BLOCK_SIZE =
static_cast<size_t>(Traits::BLOCK_SIZE);
776#pragma warning(disable: 4307)
777#pragma warning(disable: 4309)
788 static_assert(!std::numeric_limits<size_t>::is_signed && std::is_integral<size_t>::value,
"Traits::size_t must be an unsigned integral type");
790 static_assert(!std::numeric_limits<index_t>::is_signed && std::is_integral<index_t>::value,
"Traits::index_t must be an unsigned integral type");
792 static_assert(
sizeof(
index_t) >=
sizeof(
size_t),
"Traits::index_t must be at least as wide as Traits::size_t");
826#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
831 explicitProducers.store(
nullptr, std::memory_order_relaxed);
832 implicitProducers.store(
nullptr, std::memory_order_relaxed);
839 ConcurrentQueue(
size_t minCapacity,
size_t maxExplicitProducers,
size_t maxImplicitProducers)
853 size_t blocks = (((minCapacity +
BLOCK_SIZE - 1) /
BLOCK_SIZE) - 1) * (maxExplicitProducers + 1) + 2 * (maxExplicitProducers + maxImplicitProducers);
858#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
859 explicitProducers.store(
nullptr, std::memory_order_relaxed);
862 implicitProducers.store(
nullptr, std::memory_order_relaxed);
874 while (ptr !=
nullptr) {
875 auto next = ptr->next_prod();
876 if (ptr->token !=
nullptr) {
877 ptr->token->producer =
nullptr;
886 while (hash !=
nullptr) {
887 auto prev = hash->prev;
888 if (prev !=
nullptr) {
889 for (
size_t i = 0; i != hash->capacity; ++i) {
890 hash->entries[i].~ImplicitProducerKVP();
892 hash->~ImplicitProducerHash();
893 (Traits::free)(hash);
900 auto block =
freeList.head_unsafe();
901 while (block !=
nullptr) {
902 auto next = block->freeListNext.load(std::memory_order_relaxed);
903 if (block->dynamicallyAllocated) {
923 producerCount(other.producerCount.load(std::memory_order_relaxed)),
927 freeList(std::move(other.freeList)),
936 other.producerListTail.store(
nullptr, std::memory_order_relaxed);
937 other.producerCount.store(0, std::memory_order_relaxed);
938 other.nextExplicitConsumerId.store(0, std::memory_order_relaxed);
939 other.globalExplicitConsumerOffset.store(0, std::memory_order_relaxed);
941#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
942 explicitProducers.store(other.explicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed);
943 other.explicitProducers.store(
nullptr, std::memory_order_relaxed);
944 implicitProducers.store(other.implicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed);
945 other.implicitProducers.store(
nullptr, std::memory_order_relaxed);
948 other.initialBlockPoolIndex.store(0, std::memory_order_relaxed);
949 other.initialBlockPoolSize = 0;
950 other.initialBlockPool =
nullptr;
973 if (
this == &other) {
989 other.reown_producers();
991#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
1009 return inner_enqueue<CanAlloc>(item);
1020 return inner_enqueue<CanAlloc>(std::move(item));
1029 return inner_enqueue<CanAlloc>(token, item);
1038 return inner_enqueue<CanAlloc>(token, std::move(item));
1046 template<
typename It>
1050 return inner_enqueue_bulk<CanAlloc>(itemFirst, count);
1059 template<
typename It>
1062 return inner_enqueue_bulk<CanAlloc>(token, itemFirst, count);
1073 return inner_enqueue<CannotAlloc>(item);
1084 return inner_enqueue<CannotAlloc>(std::move(item));
1092 return inner_enqueue<CannotAlloc>(token, item);
1100 return inner_enqueue<CannotAlloc>(token, std::move(item));
1110 template<
typename It>
1114 return inner_enqueue_bulk<CannotAlloc>(itemFirst, count);
1122 template<
typename It>
1125 return inner_enqueue_bulk<CannotAlloc>(token, itemFirst, count);
1133 template<
typename U>
1139 size_t nonEmptyCount = 0;
1141 size_t bestSize = 0;
1142 for (
auto ptr =
producerListTail.load(std::memory_order_acquire); nonEmptyCount < 3 && ptr !=
nullptr; ptr = ptr->next_prod()) {
1145 if (size > bestSize) {
1154 if (nonEmptyCount > 0) {
1158 for (
auto ptr =
producerListTail.load(std::memory_order_acquire); ptr !=
nullptr; ptr = ptr->next_prod()) {
1159 if (ptr != best && ptr->
dequeue(item)) {
1173 template<
typename U>
1176 for (
auto ptr =
producerListTail.load(std::memory_order_acquire); ptr !=
nullptr; ptr = ptr->next_prod()) {
1177 if (ptr->dequeue(item)) {
1188 template<
typename U>
1213 if (ptr ==
nullptr) {
1217 if (ptr->dequeue(item)) {
1222 ptr = ptr->next_prod();
1223 if (ptr ==
nullptr) {
1234 template<
typename It>
1238 for (
auto ptr =
producerListTail.load(std::memory_order_acquire); ptr !=
nullptr; ptr = ptr->next_prod()) {
1239 count += ptr->dequeue_bulk(itemFirst, max - count);
1252 template<
typename It>
1284 if (ptr ==
nullptr) {
1294 if (dequeued != 0) {
1300 if (dequeued == max) {
1305 ptr = ptr->next_prod();
1306 if (ptr ==
nullptr) {
1322 template<
typename U>
1338 template<
typename It>
1358 for (
auto ptr =
producerListTail.load(std::memory_order_acquire); ptr !=
nullptr; ptr = ptr->next_prod()) {
1362 size += ptr->size_approx();
1401 template<AllocationMode canAlloc,
typename U>
1404 return static_cast<ExplicitProducer*
>(token.
producer)->ConcurrentQueue::ExplicitProducer::template enqueue<canAlloc>(std::forward<U>(element));
1409 template<AllocationMode canAlloc,
typename U>
1413 return producer ==
nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue<canAlloc>(std::forward<U>(element));
1416 template<AllocationMode canAlloc,
typename It>
1419 return static_cast<ExplicitProducer*
>(token.
producer)->ConcurrentQueue::ExplicitProducer::template enqueue_bulk<canAlloc>(itemFirst, count);
1422 template<AllocationMode canAlloc,
typename It>
1426 return producer ==
nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue_bulk<canAlloc>(itemFirst, count);
1436 auto prodCount =
producerCount.load(std::memory_order_relaxed);
1443 std::uint32_t offset = prodCount - 1 - (token.
initialOffset % prodCount);
1445 for (std::uint32_t i = 0; i != offset; ++i) {
1456 if (delta >= prodCount) {
1457 delta = delta % prodCount;
1459 for (std::uint32_t i = 0; i != delta; ++i) {
1479 template <
typename N>
1491 template<
typename N>
1503#if MCDBGQ_NOLOCKFREE_FREELIST
1504 debug::DebugLock lock(mutex);
1517#if MCDBGQ_NOLOCKFREE_FREELIST
1518 debug::DebugLock lock(mutex);
1520 auto head =
freeListHead.load(std::memory_order_acquire);
1521 while (head !=
nullptr) {
1522 auto prevHead = head;
1523 auto refs = head->freeListRefs.load(std::memory_order_relaxed);
1524 if ((refs &
REFS_MASK) == 0 || !head->freeListRefs.compare_exchange_strong(refs, refs + 1, std::memory_order_acquire, std::memory_order_relaxed)) {
1530 auto next = head->freeListNext.load(std::memory_order_relaxed);
1531 if (
freeListHead.compare_exchange_strong(head, next, std::memory_order_acquire, std::memory_order_relaxed)) {
1537 head->freeListRefs.fetch_sub(2, std::memory_order_release);
1543 refs = prevHead->freeListRefs.fetch_sub(1, std::memory_order_acq_rel);
1565 auto head =
freeListHead.load(std::memory_order_relaxed);
1567 node->freeListNext.store(head, std::memory_order_relaxed);
1568 node->freeListRefs.store(1, std::memory_order_release);
1569 if (!
freeListHead.compare_exchange_strong(head, node, std::memory_order_release, std::memory_order_relaxed)) {
1586#if MCDBGQ_NOLOCKFREE_FREELIST
1587 debug::DebugMutex mutex;
1608 template<InnerQueueContext context>
1614 if (!
emptyFlags[i].load(std::memory_order_relaxed)) {
1620 std::atomic_thread_fence(std::memory_order_acquire);
1626 std::atomic_thread_fence(std::memory_order_acquire);
1635 template<InnerQueueContext context>
1654 template<InnerQueueContext context>
1659 std::atomic_thread_fence(std::memory_order_release);
1661 for (
size_t j = 0; j != count; ++j) {
1662 assert(!
emptyFlags[i + j].load(std::memory_order_relaxed));
1663 emptyFlags[i + j].store(
true, std::memory_order_relaxed);
1675 template<InnerQueueContext context>
1681 emptyFlags[i].store(
true, std::memory_order_relaxed);
1690 template<InnerQueueContext context>
1696 emptyFlags[i].store(
false, std::memory_order_relaxed);
1713 static_assert(std::alignment_of<T>::value <= std::alignment_of<details::max_align_t>::value,
"The queue does not support super-aligned types at this time");
1734 static_assert(std::alignment_of<Block>::value >= std::alignment_of<details::max_align_t>::value,
"Internal error: Blocks must be at least as aligned as the type they are wrapping");
1762 template<
typename U>
1773 template<
typename It>
1788 auto tail =
tailIndex.load(std::memory_order_relaxed);
1789 auto head =
headIndex.load(std::memory_order_relaxed);
1809 friend struct MemStats;
1848 Block* halfDequeuedBlock =
nullptr;
1863 block = block->
next;
1864 if (block->ConcurrentQueue::Block::template is_empty<explicit_context>()) {
1869 if (block == halfDequeuedBlock) {
1876 (*block)[i++]->~T();
1885 auto nextBlock = block->
next;
1886 if (block->dynamicallyAllocated) {
1890 this->
parent->add_block_to_free_list(block);
1898 while (header !=
nullptr) {
1899 auto prev =
static_cast<BlockIndexHeader*
>(header->prev);
1900 header->~BlockIndexHeader();
1901 (Traits::free)(header);
1906 template<AllocationMode allocMode,
typename U>
1909 index_t currentTailIndex = this->
tailIndex.load(std::memory_order_relaxed);
1910 index_t newTailIndex = 1 + currentTailIndex;
1915 if (this->
tailBlock !=
nullptr && this->
tailBlock->
next->ConcurrentQueue::Block::template is_empty<explicit_context>()) {
1918 this->
tailBlock->ConcurrentQueue::Block::template reset_empty<explicit_context>();
1925 auto head = this->
headIndex.load(std::memory_order_relaxed);
1926 assert(!details::circular_less_than<index_t>(currentTailIndex, head));
1927 if (!details::circular_less_than<index_t>(head, currentTailIndex +
BLOCK_SIZE)
1942 auto newBlock = this->
parent->ConcurrentQueue::template requisition_block<allocMode>();
1943 if (newBlock ==
nullptr) {
1947 newBlock->owner =
this;
1949 newBlock->ConcurrentQueue::Block::template reset_empty<explicit_context>();
1951 newBlock->next = newBlock;
1964 new ((*this->
tailBlock)[currentTailIndex]) T(std::forward<U>(element));
1975 (void)originalBlockIndexSlotsUsed;
1980 entry.base = currentTailIndex;
1986 this->
tailIndex.store(newTailIndex, std::memory_order_release);
1992 new ((*this->
tailBlock)[currentTailIndex]) T(std::forward<U>(element));
1994 this->
tailIndex.store(newTailIndex, std::memory_order_release);
1998 template<
typename U>
2001 auto tail = this->
tailIndex.load(std::memory_order_relaxed);
2003 if (details::circular_less_than<index_t>(this->
dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) {
2019 std::atomic_thread_fence(std::memory_order_acquire);
2034 tail = this->
tailIndex.load(std::memory_order_acquire);
2035 if ((
details::likely)(details::circular_less_than<index_t>(myDequeueCount - overcommit, tail))) {
2044 auto index = this->
headIndex.fetch_add(1, std::memory_order_acq_rel);
2049 auto localBlockIndex =
blockIndex.load(std::memory_order_acquire);
2050 auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire);
2055 auto headBase = localBlockIndex->entries[localBlockIndexHead].base;
2056 auto blockBaseIndex = index & ~static_cast<index_t>(
BLOCK_SIZE - 1);
2057 auto offset =
static_cast<size_t>(
static_cast<typename std::make_signed<index_t>::type
>(blockBaseIndex - headBase) /
BLOCK_SIZE);
2058 auto block = localBlockIndex->entries[(localBlockIndexHead + offset) & (localBlockIndex->size - 1)].block;
2061 auto& el = *((*block)[index]);
2070 (*block)[index]->~T();
2071 block->ConcurrentQueue::Block::template set_empty<explicit_context>(index);
2073 } guard = { block, index };
2075 element = std::move(el);
2078 element = std::move(el);
2080 block->ConcurrentQueue::Block::template set_empty<explicit_context>(index);
2094 template<AllocationMode allocMode,
typename It>
2104 Block* firstAllocatedBlock =
nullptr;
2107 size_t blockBaseDiff = ((startTailIndex + count - 1) & ~
static_cast<index_t>(
BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast<index_t>(
BLOCK_SIZE - 1));
2109 if (blockBaseDiff > 0) {
2111 while (blockBaseDiff > 0 && this->
tailBlock !=
nullptr && this->
tailBlock->
next != firstAllocatedBlock && this->tailBlock->
next->ConcurrentQueue::Block::template is_empty<explicit_context>()) {
2116 firstAllocatedBlock = firstAllocatedBlock ==
nullptr ? this->
tailBlock : firstAllocatedBlock;
2119 entry.base = currentTailIndex;
2125 while (blockBaseDiff > 0) {
2129 auto head = this->
headIndex.load(std::memory_order_relaxed);
2130 assert(!details::circular_less_than<index_t>(currentTailIndex, head));
2137 this->
tailBlock = startBlock ==
nullptr ? firstAllocatedBlock : startBlock;
2141 originalBlockIndexFront = originalBlockIndexSlotsUsed;
2145 auto newBlock = this->
parent->ConcurrentQueue::template requisition_block<allocMode>();
2146 if (newBlock ==
nullptr) {
2149 this->
tailBlock = startBlock ==
nullptr ? firstAllocatedBlock : startBlock;
2154 newBlock->owner =
this;
2157 newBlock->ConcurrentQueue::Block::template set_all_empty<explicit_context>();
2159 newBlock->next = newBlock;
2168 firstAllocatedBlock = firstAllocatedBlock ==
nullptr ? this->
tailBlock : firstAllocatedBlock;
2174 entry.base = currentTailIndex;
2183 auto block = firstAllocatedBlock;
2185 block->ConcurrentQueue::Block::template reset_empty<explicit_context>();
2191 block = block->next;
2200 index_t newTailIndex = startTailIndex +
static_cast<index_t>(count);
2201 currentTailIndex = startTailIndex;
2205 assert((startTailIndex &
static_cast<index_t>(
BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock !=
nullptr || count == 0);
2206 if ((startTailIndex &
static_cast<index_t>(
BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock !=
nullptr) {
2212 if (details::circular_less_than<index_t>(newTailIndex, stopIndex)) {
2213 stopIndex = newTailIndex;
2216 while (currentTailIndex != stopIndex) {
2217 new ((*this->
tailBlock)[currentTailIndex++]) T(*itemFirst++);
2222 while (currentTailIndex != stopIndex) {
2236 auto constructedStopIndex = currentTailIndex;
2237 auto lastBlockEnqueued = this->
tailBlock;
2241 this->
tailBlock = startBlock ==
nullptr ? firstAllocatedBlock : startBlock;
2244 auto block = startBlock;
2246 block = firstAllocatedBlock;
2248 currentTailIndex = startTailIndex;
2251 if (details::circular_less_than<index_t>(constructedStopIndex, stopIndex)) {
2252 stopIndex = constructedStopIndex;
2254 while (currentTailIndex != stopIndex) {
2255 (*block)[currentTailIndex++]->~T();
2257 if (block == lastBlockEnqueued) {
2260 block = block->
next;
2268 assert(currentTailIndex == newTailIndex);
2278 this->
tailIndex.store(newTailIndex, std::memory_order_release);
2282 template<
typename It>
2285 auto tail = this->
tailIndex.load(std::memory_order_relaxed);
2287 auto desiredCount =
static_cast<size_t>(tail - (this->
dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit));
2288 if (details::circular_less_than<size_t>(0, desiredCount)) {
2289 desiredCount = desiredCount < max ? desiredCount : max;
2290 std::atomic_thread_fence(std::memory_order_acquire);
2294 tail = this->
tailIndex.load(std::memory_order_acquire);
2295 auto actualCount =
static_cast<size_t>(tail - (myDequeueCount - overcommit));
2296 if (details::circular_less_than<size_t>(0, actualCount)) {
2297 actualCount = desiredCount < actualCount ? desiredCount : actualCount;
2298 if (actualCount < desiredCount) {
2299 this->
dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release);
2304 auto firstIndex = this->
headIndex.fetch_add(actualCount, std::memory_order_acq_rel);
2307 auto localBlockIndex =
blockIndex.load(std::memory_order_acquire);
2308 auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire);
2310 auto headBase = localBlockIndex->entries[localBlockIndexHead].base;
2311 auto firstBlockBaseIndex = firstIndex & ~static_cast<index_t>(
BLOCK_SIZE - 1);
2312 auto offset =
static_cast<size_t>(
static_cast<typename std::make_signed<index_t>::type
>(firstBlockBaseIndex - headBase) /
BLOCK_SIZE);
2313 auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1);
2316 auto index = firstIndex;
2318 auto firstIndexInBlock = index;
2320 endIndex = details::circular_less_than<index_t>(firstIndex +
static_cast<index_t>(actualCount), endIndex) ? firstIndex +
static_cast<index_t>(actualCount) : endIndex;
2321 auto block = localBlockIndex->entries[indexIndex].block;
2323 while (index != endIndex) {
2324 auto& el = *((*block)[index]);
2325 *itemFirst++ = std::move(el);
2332 while (index != endIndex) {
2333 auto& el = *((*block)[index]);
2334 *itemFirst = std::move(el);
2344 block = localBlockIndex->entries[indexIndex].block;
2345 while (index != endIndex) {
2346 (*block)[index++]->~T();
2348 block->ConcurrentQueue::Block::template set_many_empty<explicit_context>(firstIndexInBlock,
static_cast<size_t>(endIndex - firstIndexInBlock));
2349 indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1);
2351 firstIndexInBlock = index;
2353 endIndex = details::circular_less_than<index_t>(firstIndex +
static_cast<index_t>(actualCount), endIndex) ? firstIndex +
static_cast<index_t>(actualCount) : endIndex;
2354 }
while (index != firstIndex + actualCount);
2359 block->ConcurrentQueue::Block::template set_many_empty<explicit_context>(firstIndexInBlock,
static_cast<size_t>(endIndex - firstIndexInBlock));
2360 indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1);
2361 }
while (index != firstIndex + actualCount);
2399 if (newRawPtr ==
nullptr) {
2412 i = (i + 1) & prevBlockSizeMask;
2419 header->front.store(numberOfFilledSlotsToExpose - 1, std::memory_order_relaxed);
2420 header->entries = newBlockIndexEntries;
2427 blockIndex.store(header, std::memory_order_release);
2442#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
2449 friend struct MemStats;
2474#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
2477 if (!this->
inactive.load(std::memory_order_relaxed)) {
2478 details::ThreadExitNotifier::unsubscribe(&threadExitListener);
2483 auto tail = this->
tailIndex.load(std::memory_order_relaxed);
2484 auto index = this->
headIndex.load(std::memory_order_relaxed);
2485 Block* block =
nullptr;
2487 bool forceFreeLastBlock = index != tail;
2488 while (index != tail) {
2490 if (block !=
nullptr) {
2492 this->
parent->add_block_to_free_list(block);
2498 ((*block)[index])->~T();
2508 auto localBlockIndex =
blockIndex.load(std::memory_order_relaxed);
2509 if (localBlockIndex !=
nullptr) {
2510 for (
size_t i = 0; i != localBlockIndex->capacity; ++i) {
2511 localBlockIndex->index[i]->~BlockIndexEntry();
2514 auto prev = localBlockIndex->prev;
2515 localBlockIndex->~BlockIndexHeader();
2516 (Traits::free)(localBlockIndex);
2517 localBlockIndex = prev;
2518 }
while (localBlockIndex !=
nullptr);
2522 template<AllocationMode allocMode,
typename U>
2525 index_t currentTailIndex = this->
tailIndex.load(std::memory_order_relaxed);
2526 index_t newTailIndex = 1 + currentTailIndex;
2529 auto head = this->
headIndex.load(std::memory_order_relaxed);
2530 assert(!details::circular_less_than<index_t>(currentTailIndex, head));
2534#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
2535 debug::DebugLock lock(mutex);
2539 if (!insert_block_index_entry<allocMode>(idxEntry, currentTailIndex)) {
2544 auto newBlock = this->
parent->ConcurrentQueue::template requisition_block<allocMode>();
2545 if (newBlock ==
nullptr) {
2547 idxEntry->
value.store(
nullptr, std::memory_order_relaxed);
2551 newBlock->owner =
this;
2553 newBlock->ConcurrentQueue::Block::template reset_empty<implicit_context>();
2558 new ((*newBlock)[currentTailIndex]) T(std::forward<U>(element));
2562 idxEntry->
value.store(
nullptr, std::memory_order_relaxed);
2563 this->
parent->add_block_to_free_list(newBlock);
2569 idxEntry->
value.store(newBlock, std::memory_order_relaxed);
2574 this->
tailIndex.store(newTailIndex, std::memory_order_release);
2580 new ((*this->
tailBlock)[currentTailIndex]) T(std::forward<U>(element));
2582 this->
tailIndex.store(newTailIndex, std::memory_order_release);
2586 template<
typename U>
2592 if (details::circular_less_than<index_t>(this->
dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) {
2593 std::atomic_thread_fence(std::memory_order_acquire);
2596 tail = this->
tailIndex.load(std::memory_order_acquire);
2597 if ((
details::likely)(details::circular_less_than<index_t>(myDequeueCount - overcommit, tail))) {
2604 auto block = entry->value.load(std::memory_order_relaxed);
2605 auto& el = *((*block)[index]);
2608#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
2610 debug::DebugLock lock(producer->mutex);
2620 (*block)[index]->~T();
2621 if (block->ConcurrentQueue::Block::template set_empty<implicit_context>(index)) {
2622 entry->
value.store(
nullptr, std::memory_order_relaxed);
2623 parent->add_block_to_free_list(block);
2626 } guard = { block, index, entry, this->
parent };
2628 element = std::move(el);
2631 element = std::move(el);
2634 if (block->ConcurrentQueue::Block::template set_empty<implicit_context>(index)) {
2636#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
2637 debug::DebugLock lock(mutex);
2640 entry->value.store(
nullptr, std::memory_order_relaxed);
2642 this->
parent->add_block_to_free_list(block);
2656 template<AllocationMode allocMode,
typename It>
2668 Block* firstAllocatedBlock =
nullptr;
2672 size_t blockBaseDiff = ((startTailIndex + count - 1) & ~
static_cast<index_t>(
BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast<index_t>(
BLOCK_SIZE - 1));
2674 if (blockBaseDiff > 0) {
2675#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
2676 debug::DebugLock lock(mutex);
2685 bool indexInserted =
false;
2686 auto head = this->
headIndex.load(std::memory_order_relaxed);
2687 assert(!details::circular_less_than<index_t>(currentTailIndex, head));
2689 if (full || !(indexInserted = insert_block_index_entry<allocMode>(idxEntry, currentTailIndex)) || (newBlock = this->
parent->ConcurrentQueue::template requisition_block<allocMode>()) ==
nullptr) {
2692 if (indexInserted) {
2694 idxEntry->
value.store(
nullptr, std::memory_order_relaxed);
2696 currentTailIndex = (startTailIndex - 1) & ~
static_cast<index_t>(
BLOCK_SIZE - 1);
2697 for (
auto block = firstAllocatedBlock; block !=
nullptr; block = block->
next) {
2700 idxEntry->
value.store(
nullptr, std::memory_order_relaxed);
2703 this->
parent->add_blocks_to_free_list(firstAllocatedBlock);
2710 newBlock->owner =
this;
2712 newBlock->ConcurrentQueue::Block::template reset_empty<implicit_context>();
2713 newBlock->
next =
nullptr;
2716 idxEntry->
value.store(newBlock, std::memory_order_relaxed);
2720 if ((startTailIndex &
static_cast<index_t>(
BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock !=
nullptr) {
2725 endBlock = newBlock;
2726 firstAllocatedBlock = firstAllocatedBlock ==
nullptr ? newBlock : firstAllocatedBlock;
2727 }
while (blockBaseDiff > 0);
2731 index_t newTailIndex = startTailIndex +
static_cast<index_t>(count);
2732 currentTailIndex = startTailIndex;
2734 assert((startTailIndex &
static_cast<index_t>(
BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock !=
nullptr || count == 0);
2735 if ((startTailIndex &
static_cast<index_t>(
BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock !=
nullptr) {
2740 if (details::circular_less_than<index_t>(newTailIndex, stopIndex)) {
2741 stopIndex = newTailIndex;
2744 while (currentTailIndex != stopIndex) {
2745 new ((*this->
tailBlock)[currentTailIndex++]) T(*itemFirst++);
2750 while (currentTailIndex != stopIndex) {
2757 auto constructedStopIndex = currentTailIndex;
2758 auto lastBlockEnqueued = this->
tailBlock;
2761 auto block = startBlock;
2763 block = firstAllocatedBlock;
2765 currentTailIndex = startTailIndex;
2768 if (details::circular_less_than<index_t>(constructedStopIndex, stopIndex)) {
2769 stopIndex = constructedStopIndex;
2771 while (currentTailIndex != stopIndex) {
2772 (*block)[currentTailIndex++]->~T();
2774 if (block == lastBlockEnqueued) {
2777 block = block->
next;
2781 currentTailIndex = (startTailIndex - 1) & ~
static_cast<index_t>(
BLOCK_SIZE - 1);
2782 for (
auto block = firstAllocatedBlock; block !=
nullptr; block = block->
next) {
2785 idxEntry->value.store(
nullptr, std::memory_order_relaxed);
2788 this->
parent->add_blocks_to_free_list(firstAllocatedBlock);
2795 assert(currentTailIndex == newTailIndex);
2800 this->
tailIndex.store(newTailIndex, std::memory_order_release);
2804 template<
typename It>
2807 auto tail = this->
tailIndex.load(std::memory_order_relaxed);
2809 auto desiredCount =
static_cast<size_t>(tail - (this->
dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit));
2810 if (details::circular_less_than<size_t>(0, desiredCount)) {
2811 desiredCount = desiredCount < max ? desiredCount : max;
2812 std::atomic_thread_fence(std::memory_order_acquire);
2816 tail = this->
tailIndex.load(std::memory_order_acquire);
2817 auto actualCount =
static_cast<size_t>(tail - (myDequeueCount - overcommit));
2818 if (details::circular_less_than<size_t>(0, actualCount)) {
2819 actualCount = desiredCount < actualCount ? desiredCount : actualCount;
2820 if (actualCount < desiredCount) {
2821 this->
dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release);
2826 auto firstIndex = this->
headIndex.fetch_add(actualCount, std::memory_order_acq_rel);
2829 auto index = firstIndex;
2833 auto blockStartIndex = index;
2835 endIndex = details::circular_less_than<index_t>(firstIndex +
static_cast<index_t>(actualCount), endIndex) ? firstIndex +
static_cast<index_t>(actualCount) : endIndex;
2837 auto entry = localBlockIndex->
index[indexIndex];
2838 auto block = entry->
value.load(std::memory_order_relaxed);
2840 while (index != endIndex) {
2841 auto& el = *((*block)[index]);
2842 *itemFirst++ = std::move(el);
2849 while (index != endIndex) {
2850 auto& el = *((*block)[index]);
2851 *itemFirst = std::move(el);
2859 entry = localBlockIndex->
index[indexIndex];
2860 block = entry->
value.load(std::memory_order_relaxed);
2861 while (index != endIndex) {
2862 (*block)[index++]->~T();
2865 if (block->ConcurrentQueue::Block::template set_many_empty<implicit_context>(blockStartIndex,
static_cast<size_t>(endIndex - blockStartIndex))) {
2866#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
2867 debug::DebugLock lock(mutex);
2869 entry->value.store(
nullptr, std::memory_order_relaxed);
2870 this->
parent->add_block_to_free_list(block);
2872 indexIndex = (indexIndex + 1) & (localBlockIndex->
capacity - 1);
2874 blockStartIndex = index;
2876 endIndex = details::circular_less_than<index_t>(firstIndex +
static_cast<index_t>(actualCount), endIndex) ? firstIndex +
static_cast<index_t>(actualCount) : endIndex;
2877 }
while (index != firstIndex + actualCount);
2882 if (block->ConcurrentQueue::Block::template set_many_empty<implicit_context>(blockStartIndex,
static_cast<size_t>(endIndex - blockStartIndex))) {
2884#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
2885 debug::DebugLock lock(mutex);
2889 entry->value.store(
nullptr, std::memory_order_relaxed);
2891 this->
parent->add_block_to_free_list(block);
2893 indexIndex = (indexIndex + 1) & (localBlockIndex->
capacity - 1);
2894 }
while (index != firstIndex + actualCount);
2925 template<AllocationMode allocMode>
2928 auto localBlockIndex =
blockIndex.load(std::memory_order_relaxed);
2929 if (localBlockIndex ==
nullptr) {
2932 auto newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1);
2933 idxEntry = localBlockIndex->index[newTail];
2935 idxEntry->
value.load(std::memory_order_relaxed) ==
nullptr) {
2937 idxEntry->
key.store(blockStartIndex, std::memory_order_relaxed);
2938 localBlockIndex->tail.store(newTail, std::memory_order_release);
2946 localBlockIndex =
blockIndex.load(std::memory_order_relaxed);
2947 newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1);
2948 idxEntry = localBlockIndex->index[newTail];
2950 idxEntry->
key.store(blockStartIndex, std::memory_order_relaxed);
2951 localBlockIndex->tail.store(newTail, std::memory_order_release);
2957 auto localBlockIndex =
blockIndex.load(std::memory_order_relaxed);
2958 localBlockIndex->tail.store((localBlockIndex->tail.load(std::memory_order_relaxed) - 1) & (localBlockIndex->capacity - 1), std::memory_order_relaxed);
2965 return localBlockIndex->
index[idx];
2970#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
2971 debug::DebugLock lock(mutex);
2973 index &= ~static_cast<index_t>(
BLOCK_SIZE - 1);
2974 localBlockIndex =
blockIndex.load(std::memory_order_acquire);
2975 auto tail = localBlockIndex->
tail.load(std::memory_order_acquire);
2976 auto tailBase = localBlockIndex->
index[tail]->
key.load(std::memory_order_relaxed);
2979 auto offset =
static_cast<size_t>(
static_cast<typename std::make_signed<index_t>::type
>(index - tailBase) /
BLOCK_SIZE);
2980 size_t idx = (tail + offset) & (localBlockIndex->
capacity - 1);
2981 assert(localBlockIndex->
index[idx]->
key.load(std::memory_order_relaxed) == index && localBlockIndex->
index[idx]->
value.load(std::memory_order_relaxed) !=
nullptr);
2987 auto prev =
blockIndex.load(std::memory_order_relaxed);
2988 size_t prevCapacity = prev ==
nullptr ? 0 : prev->capacity;
2990 auto raw =
static_cast<char*
>((Traits::malloc)(
2992 std::alignment_of<BlockIndexEntry>::value - 1 +
sizeof(
BlockIndexEntry) * entryCount +
2994 if (raw ==
nullptr) {
3000 auto index =
reinterpret_cast<BlockIndexEntry**
>(details::align_for<BlockIndexEntry*>(
reinterpret_cast<char*
>(entries) +
sizeof(
BlockIndexEntry) * entryCount));
3001 if (prev !=
nullptr) {
3002 auto prevTail = prev->tail.load(std::memory_order_relaxed);
3003 auto prevPos = prevTail;
3006 prevPos = (prevPos + 1) & (prev->capacity - 1);
3007 index[i++] = prev->index[prevPos];
3008 }
while (prevPos != prevTail);
3009 assert(i == prevCapacity);
3011 for (
size_t i = 0; i != entryCount; ++i) {
3014 index[prevCapacity + i] = entries + i;
3016 header->prev = prev;
3017 header->entries = entries;
3018 header->index = index;
3022 blockIndex.store(header, std::memory_order_release);
3033#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
3035 details::ThreadExitListener threadExitListener;
3039#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
3045#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX
3046 mutable debug::DebugMutex mutex;
3049 friend struct MemStats;
3089 block->owner =
nullptr;
3096 while (block !=
nullptr) {
3097 auto next = block->
next;
3109 template<AllocationMode canAlloc>
3113 if (block !=
nullptr) {
3118 if (block !=
nullptr) {
3123 return create<Block>();
3135 size_t allocatedBlocks;
3141 size_t ownedBlocksExplicit;
3143 size_t ownedBlocksImplicit;
3145 size_t implicitProducers;
3147 size_t explicitProducers;
3149 size_t elementsEnqueued;
3151 size_t blockClassBytes;
3153 size_t queueClassBytes;
3155 size_t implicitBlockIndexBytes;
3157 size_t explicitBlockIndexBytes;
3166 MemStats stats = { 0 };
3168 stats.elementsEnqueued = q->size_approx();
3170 auto block = q->freeList.head_unsafe();
3172 while (block !=
nullptr) {
3174 ++stats.allocatedBlocks;
3178 block = block->freeListNext.load(std::memory_order_relaxed);
3181 for (
auto ptr = q->producerListTail.load(std::memory_order_acquire); ptr !=
nullptr; ptr = ptr->next_prod()) {
3185 stats.implicitProducers += implicit ? 1 : 0;
3187 stats.explicitProducers += implicit ? 0 : 1;
3195 auto head = prod->headIndex.load(std::memory_order_relaxed);
3197 auto tail = prod->tailIndex.load(std::memory_order_relaxed);
3199 auto hash = prod->blockIndex.load(std::memory_order_relaxed);
3201 if (hash !=
nullptr) {
3203 for (
size_t i = 0; i != hash->capacity; ++i) {
3205 if (hash->index[i]->key.load(std::memory_order_relaxed) != ImplicitProducer::INVALID_BLOCK_BASE && hash->index[i]->value.load(std::memory_order_relaxed) !=
nullptr) {
3207 ++stats.allocatedBlocks;
3209 ++stats.ownedBlocksImplicit;
3213 stats.implicitBlockIndexBytes += hash->capacity *
sizeof(
typename ImplicitProducer::BlockIndexEntry);
3215 for (; hash !=
nullptr; hash = hash->prev) {
3216 stats.implicitBlockIndexBytes +=
sizeof(
typename ImplicitProducer::BlockIndexHeader) + hash->capacity *
sizeof(
typename ImplicitProducer::BlockIndexEntry*);
3220 for (; details::circular_less_than<index_t>(head, tail); head +=
BLOCK_SIZE) {
3230 auto tailBlock = prod->tailBlock;
3231 bool wasNonEmpty =
false;
3232 if (tailBlock !=
nullptr) {
3233 auto block = tailBlock;
3235 ++stats.allocatedBlocks;
3236 if (!block->ConcurrentQueue::Block::template is_empty<explicit_context>() || wasNonEmpty) {
3238 wasNonEmpty = wasNonEmpty || block != tailBlock;
3240 ++stats.ownedBlocksExplicit;
3241 block = block->next;
3242 }
while (block != tailBlock);
3244 auto index = prod->blockIndex.load(std::memory_order_relaxed);
3245 while (index !=
nullptr) {
3246 stats.explicitBlockIndexBytes +=
sizeof(
typename ExplicitProducer::BlockIndexHeader) + index->size *
sizeof(
typename ExplicitProducer::BlockIndexEntry);
3247 index =
static_cast<typename ExplicitProducer::BlockIndexHeader*
>(index->prev);
3252 auto freeOnInitialPool = q->initialBlockPoolIndex.load(std::memory_order_relaxed) >= q->initialBlockPoolSize ? 0 : q->initialBlockPoolSize - q->initialBlockPoolIndex.load(std::memory_order_relaxed);
3253 stats.allocatedBlocks += freeOnInitialPool;
3254 stats.freeBlocks += freeOnInitialPool;
3256 stats.blockClassBytes =
sizeof(Block) * stats.allocatedBlocks;
3264 MemStats getMemStats()
3266 return MemStats::getFor(
this);
3269 friend struct MemStats;
3285#if MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH
3286 debug::DebugLock lock(implicitProdMutex);
3289 for (
auto ptr =
producerListTail.load(std::memory_order_acquire); ptr !=
nullptr; ptr = ptr->next_prod()) {
3290 if (ptr->inactive.load(std::memory_order_relaxed) && ptr->isExplicit == isExplicit) {
3291 bool expected =
true;
3292 if (ptr->inactive.compare_exchange_strong(expected,
false, std::memory_order_acquire, std::memory_order_relaxed)) {
3301 return add_producer(isExplicit ?
static_cast<ProducerBase*
>(create<ExplicitProducer>(
this)) : create<ImplicitProducer>(
this));
3307 if (producer ==
nullptr) {
3316 producer->
next = prevTail;
3317 }
while (!
producerListTail.compare_exchange_weak(prevTail, producer, std::memory_order_release, std::memory_order_relaxed));
3319#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
3321 auto prevTailExplicit = explicitProducers.load(std::memory_order_relaxed);
3323 static_cast<ExplicitProducer*
>(producer)->nextExplicitProducer = prevTailExplicit;
3324 }
while (!explicitProducers.compare_exchange_weak(prevTailExplicit,
static_cast<ExplicitProducer*
>(producer), std::memory_order_release, std::memory_order_relaxed));
3327 auto prevTailImplicit = implicitProducers.load(std::memory_order_relaxed);
3329 static_cast<ImplicitProducer*
>(producer)->nextImplicitProducer = prevTailImplicit;
3330 }
while (!implicitProducers.compare_exchange_weak(prevTailImplicit,
static_cast<ImplicitProducer*
>(producer), std::memory_order_release, std::memory_order_relaxed));
3341 for (
auto ptr =
producerListTail.load(std::memory_order_relaxed); ptr !=
nullptr; ptr = ptr->next_prod()) {
3353 std::atomic<details::thread_id_t>
key;
3360 key.store(other.key.load(std::memory_order_relaxed), std::memory_order_relaxed);
3361 value = other.value;
3372 if (
this != &other) {
3374 std::swap(
value, other.value);
3379 template<
typename XT,
typename XTraits>
3380 friend void moodycamel::swap(
typename ConcurrentQueue<XT, XTraits>::ImplicitProducerKVP&,
typename ConcurrentQueue<XT, XTraits>::ImplicitProducerKVP&)
MOODYCAMEL_NOEXCEPT;
3400 hash->prev =
nullptr;
3411 other.initialImplicitProducerHash.entries = &other.initialImplicitProducerHashEntries[0];
3416 if (
implicitProducerHash.load(std::memory_order_relaxed) == &other.initialImplicitProducerHash) {
3421 for (hash =
implicitProducerHash.load(std::memory_order_relaxed); hash->
prev != &other.initialImplicitProducerHash; hash = hash->
prev) {
3427 other.implicitProducerHash.store(&other.initialImplicitProducerHash, std::memory_order_relaxed);
3434 hash->
prev = &other.initialImplicitProducerHash;
3451#if MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH
3452 debug::DebugLock lock(implicitProdMutex);
3459 for (
auto hash = mainHash; hash !=
nullptr; hash = hash->prev) {
3461 auto index = hashedId;
3463 index &= hash->capacity - 1;
3465 auto probedKey = hash->entries[index].key.load(std::memory_order_relaxed);
3466 if (probedKey ==
id) {
3471 auto value = hash->entries[index].value;
3472 if (hash != mainHash) {
3475 index &= mainHash->capacity - 1;
3476 probedKey = mainHash->entries[index].key.load(std::memory_order_relaxed);
3478#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
3480 if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty,
id, std::memory_order_relaxed, std::memory_order_relaxed)) ||
3481 (probedKey == reusable && mainHash->entries[index].key.compare_exchange_strong(reusable,
id, std::memory_order_acquire, std::memory_order_acquire))) {
3483 if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty,
id, std::memory_order_relaxed, std::memory_order_relaxed))) {
3485 mainHash->entries[index].value = value;
3509 if (newCount >= (mainHash->capacity >> 1)) {
3510 auto newCapacity = mainHash->capacity << 1;
3511 while (newCount >= (newCapacity >> 1)) {
3515 if (raw ==
nullptr) {
3525 for (
size_t i = 0; i != newCapacity; ++i) {
3529 newHash->prev = mainHash;
3541 if (newCount < (mainHash->capacity >> 1) + (mainHash->capacity >> 2)) {
3544 if (producer ==
nullptr) {
3552#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
3553 producer->threadExitListener.callback = &ConcurrentQueue::implicit_producer_thread_exited_callback;
3554 producer->threadExitListener.userData = producer;
3555 details::ThreadExitNotifier::subscribe(&producer->threadExitListener);
3558 auto index = hashedId;
3560 index &= mainHash->capacity - 1;
3561 auto probedKey = mainHash->entries[index].key.load(std::memory_order_relaxed);
3564#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
3566 if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty,
id, std::memory_order_relaxed, std::memory_order_relaxed)) ||
3567 (probedKey == reusable && mainHash->entries[index].key.compare_exchange_strong(reusable,
id, std::memory_order_acquire, std::memory_order_acquire))) {
3569 if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty,
id, std::memory_order_relaxed, std::memory_order_relaxed))) {
3571 mainHash->entries[index].value = producer;
3586#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED
3590 details::ThreadExitNotifier::unsubscribe(&producer->threadExitListener);
3593#if MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH
3594 debug::DebugLock lock(implicitProdMutex);
3597 assert(hash !=
nullptr);
3605 for (; hash !=
nullptr; hash = hash->prev) {
3606 auto index = hashedId;
3608 index &= hash->capacity - 1;
3609 probedKey = hash->entries[index].key.load(std::memory_order_relaxed);
3610 if (probedKey ==
id) {
3620 producer->
inactive.store(
true, std::memory_order_release);
3623 static void implicit_producer_thread_exited_callback(
void* userData)
3626 auto queue = producer->
parent;
3627 queue->implicit_producer_thread_exited(producer);
3635 template<
typename U>
3639 auto p =
static_cast<U*
>((Traits::malloc)(
sizeof(U) * count));
3644 for (
size_t i = 0; i != count; ++i) {
3650 template<
typename U>
3655 for (
size_t i = count; i != 0; ) {
3662 template<
typename U>
3665 auto p = (Traits::malloc)(
sizeof(U));
3666 return p !=
nullptr ?
new (p) U :
nullptr;
3669 template<
typename U,
typename A1>
3672 auto p = (Traits::malloc)(
sizeof(U));
3673 return p !=
nullptr ?
new (p) U(std::forward<A1>(a1)) :
nullptr;
3676 template<
typename U>
3693#if !MCDBGQ_USEDEBUGFREELIST
3696 debug::DebugFreeList<Block>
freeList;
3708#if MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH
3709 debug::DebugMutex implicitProdMutex;
3712#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG
3713 std::atomic<ExplicitProducer*> explicitProducers;
3714 std::atomic<ImplicitProducer*> implicitProducers;
3719template<
typename T,
typename Traits>
3728template<
typename T,
typename Traits>
3737template<
typename T,
typename Traits>
3739 : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr)
3741 initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release);
3745template<
typename T,
typename Traits>
3747 : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr)
3753template<
typename T,
typename Traits>
3769template<
typename T,
typename Traits>
3770inline void swap(
typename ConcurrentQueue<T, Traits>::ImplicitProducerKVP& a,
typename ConcurrentQueue<T, Traits>::ImplicitProducerKVP& b)
MOODYCAMEL_NOEXCEPT
3777#if defined(__GNUC__)
3778#pragma GCC diagnostic pop
#define MOODYCAMEL_NOEXCEPT
ConcurrentQueue(size_t capacity=6 *BLOCK_SIZE)
#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)
#define MOODYCAMEL_THREADLOCAL
static const size_t BLOCK_SIZE
#define MOODYCAMEL_DELETE_FUNCTION
ConcurrentQueue & operator=(ConcurrentQueue const &) MOODYCAMEL_DELETE_FUNCTION
#define MOODYCAMEL_CATCH(...)
static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD
#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr)
#define MOODYCAMEL_RETHROW
static const size_t EXPLICIT_INITIAL_INDEX_SIZE
static const size_t IMPLICIT_INITIAL_INDEX_SIZE
static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
static const size_t MAX_SUBQUEUE_SIZE
bool try_enqueue(producer_token_t const &token, T &&item)
ConcurrentQueue & swap_internal(ConcurrentQueue &other)
bool enqueue(producer_token_t const &token, T const &item)
bool enqueue(producer_token_t const &token, T &&item)
bool try_dequeue(U &item)
ConcurrentQueue(ConcurrentQueue const &) MOODYCAMEL_DELETE_FUNCTION
static const size_t MAX_SUBQUEUE_SIZE
ConcurrentQueue(size_t capacity=6 *BLOCK_SIZE)
bool try_enqueue_bulk(producer_token_t const &token, It itemFirst, size_t count)
ConcurrentQueue & operator=(ConcurrentQueue const &) MOODYCAMEL_DELETE_FUNCTION
static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
bool enqueue_bulk(It itemFirst, size_t count)
size_t try_dequeue_bulk(consumer_token_t &token, It itemFirst, size_t max)
bool try_dequeue_from_producer(producer_token_t const &producer, U &item)
bool enqueue_bulk(producer_token_t const &token, It itemFirst, size_t count)
bool try_dequeue(consumer_token_t &token, U &item)
::moodycamel::ProducerToken producer_token_t
void swap(ConcurrentQueue &other) MOODYCAMEL_NOEXCEPT
size_t try_dequeue_bulk(It itemFirst, size_t max)
static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE
static const size_t IMPLICIT_INITIAL_INDEX_SIZE
bool try_enqueue(T const &item)
static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD
ConcurrentQueue & operator=(ConcurrentQueue &&other) MOODYCAMEL_NOEXCEPT
::moodycamel::ConsumerToken consumer_token_t
bool enqueue(T const &item)
bool try_enqueue(producer_token_t const &token, T const &item)
ConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers)
bool try_dequeue_non_interleaved(U &item)
static const size_t EXPLICIT_INITIAL_INDEX_SIZE
static const size_t BLOCK_SIZE
ConcurrentQueue(ConcurrentQueue &&other) MOODYCAMEL_NOEXCEPT
bool try_enqueue_bulk(It itemFirst, size_t count)
bool try_enqueue(T &&item)
static const thread_id_t invalid_thread_id2
static bool unlikely(bool x)
static bool circular_less_than(T a, T b)
static void swap_relaxed(std::atomic< T > &left, std::atomic< T > &right)
std::max_align_t std_max_align_t
static auto deref_noexcept(It &it) MOODYCAMEL_NOEXCEPT -> decltype(*it)
static const thread_id_t invalid_thread_id
static size_t hash_thread_id(thread_id_t id)
static thread_id_t thread_id()
static T const & nomove(T const &x)
static bool likely(bool x)
static char * align_for(char *ptr)
std::uintptr_t thread_id_t
static T ceil_to_pow_2(T x)
ImplicitProducer * get_or_add_implicit_producer()
friend struct ProducerToken
friend struct ConsumerToken
std::atomic< std::uint32_t > nextExplicitConsumerId
void swap_implicit_producer_hashes(ConcurrentQueue &other)
FreeList< Block > freeList
static void destroy(U *p)
enum moodycamel::AllocationMode try_dequeue_bulk_from_producer
bool inner_enqueue_bulk(producer_token_t const &token, It itemFirst, size_t count)
ImplicitProducerHash initialImplicitProducerHash
ProducerBase * recycle_or_create_producer(bool isExplicit)
std::atomic_flag implicitProducerHashResizeInProgress
friend struct ImplicitProducer
std::atomic< size_t > implicitProducerHashCount
friend class ConcurrentQueueTests
std::atomic< ProducerBase * > producerListTail
std::atomic< size_t > initialBlockPoolIndex
void swap(typename ConcurrentQueue< T, Traits >::ImplicitProducerKVP &a, typename ConcurrentQueue< T, Traits >::ImplicitProducerKVP &b) MOODYCAMEL_NOEXCEPT
Block * requisition_block()
void populate_initial_implicit_producer_hash()
ProducerBase * add_producer(ProducerBase *producer)
bool inner_enqueue(producer_token_t const &token, U &&element)
void populate_initial_block_list(size_t blockCount)
Block * try_get_block_from_free_list()
std::atomic< ImplicitProducerHash * > implicitProducerHash
std::atomic< std::uint32_t > globalExplicitConsumerOffset
void add_block_to_free_list(Block *block)
static U * create_array(size_t count)
std::array< ImplicitProducerKVP, INITIAL_IMPLICIT_PRODUCER_HASH_SIZE > initialImplicitProducerHashEntries
void add_blocks_to_free_list(Block *block)
friend struct ExplicitProducer
bool update_current_producer_after_rotation(consumer_token_t &token)
static void destroy_array(U *p, size_t count)
Block * try_get_block_from_initial_pool()
std::atomic< std::uint32_t > producerCount
static bool is_lock_free()
size_t initialBlockPoolSize
size_t size_approx() const
std::atomic< size_t > elementsCompletelyDequeued
bool dynamicallyAllocated
details::max_align_t dummy
std::atomic< Block * > freeListNext
char elements[sizeof(T) *BLOCK_SIZE]
bool set_empty(index_t i)
std::atomic< bool > shouldBeOnFreeList
T const * operator[](index_t idx) const MOODYCAMEL_NOEXCEPT
std::atomic< std::uint32_t > freeListRefs
T * operator[](index_t idx) MOODYCAMEL_NOEXCEPT
std::atomic< bool > emptyFlags[BLOCK_SIZE<=EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE :1]
bool set_many_empty(index_t i, size_t count)
static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
static void * malloc(size_t size)
static void free(void *ptr)
static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE
static const size_t MAX_SUBQUEUE_SIZE
static const size_t EXPLICIT_INITIAL_INDEX_SIZE
static const size_t IMPLICIT_INITIAL_INDEX_SIZE
static const size_t BLOCK_SIZE
static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD
std::uint32_t itemsConsumedFromCurrent
ConsumerToken(ConsumerToken const &) MOODYCAMEL_DELETE_FUNCTION
ConsumerToken(ConcurrentQueue< T, Traits > &q)
ConsumerToken & operator=(ConsumerToken &&other) MOODYCAMEL_NOEXCEPT
friend class ConcurrentQueueTests
details::ConcurrentQueueProducerTypelessBase * desiredProducer
std::uint32_t lastKnownGlobalOffset
ConsumerToken & operator=(ConsumerToken const &) MOODYCAMEL_DELETE_FUNCTION
details::ConcurrentQueueProducerTypelessBase * currentProducer
ConsumerToken(BlockingConcurrentQueue< T, Traits > &q)
ConsumerToken(ConsumerToken &&other) MOODYCAMEL_NOEXCEPT
std::uint32_t initialOffset
void swap(ConsumerToken &other) MOODYCAMEL_NOEXCEPT
bool enqueue_bulk(It itemFirst, size_t count)
bool enqueue(U &&element)
size_t pr_blockIndexSlotsUsed
BlockIndexEntry * pr_blockIndexEntries
ExplicitProducer(ConcurrentQueue *parent)
size_t dequeue_bulk(It &itemFirst, size_t max)
std::atomic< BlockIndexHeader * > blockIndex
size_t pr_blockIndexFront
bool new_block_index(size_t numberOfFilledSlotsToExpose)
std::atomic< N * > freeListNext
std::atomic< std::uint32_t > freeListRefs
void add_knowing_refcount_is_zero(N *node)
FreeList(FreeList &&other)
void swap(FreeList &other)
std::atomic< N * > freeListHead
static const std::uint32_t SHOULD_BE_ON_FREELIST
FreeList & operator=(FreeList const &) MOODYCAMEL_DELETE_FUNCTION
FreeList(FreeList const &) MOODYCAMEL_DELETE_FUNCTION
static const std::uint32_t REFS_MASK
ImplicitProducerHash * prev
ImplicitProducerKVP * entries
std::atomic< details::thread_id_t > key
ImplicitProducerKVP(ImplicitProducerKVP &&other) MOODYCAMEL_NOEXCEPT
ImplicitProducerKVP & operator=(ImplicitProducerKVP &&other) MOODYCAMEL_NOEXCEPT
void swap(ImplicitProducerKVP &other) MOODYCAMEL_NOEXCEPT
std::atomic< Block * > value
std::atomic< index_t > key
std::atomic< BlockIndexHeader * > blockIndex
static const index_t INVALID_BLOCK_BASE
size_t get_block_index_index_for_index(index_t index, BlockIndexHeader *&localBlockIndex) const
bool enqueue_bulk(It itemFirst, size_t count)
bool insert_block_index_entry(BlockIndexEntry *&idxEntry, index_t blockStartIndex)
size_t dequeue_bulk(It &itemFirst, size_t max)
BlockIndexEntry * get_block_index_entry_for_index(index_t index) const
void rewind_block_index_tail()
bool enqueue(U &&element)
ImplicitProducer(ConcurrentQueue *parent)
size_t nextBlockIndexCapacity
ProducerBase * next_prod() const
std::atomic< index_t > headIndex
size_t size_approx() const
std::atomic< index_t > dequeueOvercommit
std::atomic< index_t > dequeueOptimisticCount
size_t dequeue_bulk(It &itemFirst, size_t max)
std::atomic< index_t > tailIndex
ProducerBase(ConcurrentQueue *parent_, bool isExplicit_)
ProducerToken(BlockingConcurrentQueue< T, Traits > &queue)
friend class ConcurrentQueueTests
ProducerToken & operator=(ProducerToken const &) MOODYCAMEL_DELETE_FUNCTION
ProducerToken(ConcurrentQueue< T, Traits > &queue)
details::ConcurrentQueueProducerTypelessBase * producer
ProducerToken(ProducerToken const &) MOODYCAMEL_DELETE_FUNCTION
void swap(ProducerToken &other) MOODYCAMEL_NOEXCEPT
ProducerToken(ProducerToken &&other) MOODYCAMEL_NOEXCEPT
ProducerToken & operator=(ProducerToken &&other) MOODYCAMEL_NOEXCEPT
std::atomic< bool > inactive
ConcurrentQueueProducerTypelessBase * next
ConcurrentQueueProducerTypelessBase()
static std::uint64_t hash(std::uint64_t h)
static std::uint32_t hash(std::uint32_t h)
static auto eval(U &&x) -> decltype(std::forward< U >(x))
static T const & eval(T const &x)
thread_id_t thread_id_numeric_size_t
thread_id_t thread_id_hash_t
static thread_id_hash_t prehash(thread_id_t const &x)