lf_heter_queue.h

//   Copyright Giuseppe Campana (giu.campana@gmail.com) 2016-2018.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#pragma once
#include <density/default_allocator.h>
#include <density/density_common.h>
#include <density/raw_atomic.h>
#include <density/runtime_type.h>
#include <limits>
#include <type_traits>

#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4324) // structure was padded due to alignment specifier
#endif

#include <density/detail/lf_queue_base.h>
#include <density/detail/lf_queue_head_multiple.h>
#include <density/detail/lf_queue_head_single.h>
#include <density/detail/lf_queue_tail_multiple_relaxed.h>
#include <density/detail/lf_queue_tail_multiple_seq_cst.h>
#include <density/detail/lf_queue_tail_single.h>

namespace density
{
    template <
      typename RUNTIME_TYPE                        = runtime_type<>,
      typename ALLOCATOR_TYPE                      = default_allocator,
      concurrency_cardinality PROD_CARDINALITY     = concurrency_multiple,
      concurrency_cardinality CONSUMER_CARDINALITY = concurrency_multiple,
      consistency_model       CONSISTENCY_MODEL    = consistency_sequential>
    class lf_heter_queue
        : private detail::LFQueue_Head<
            RUNTIME_TYPE,
            ALLOCATOR_TYPE,
            CONSUMER_CARDINALITY,
            detail::LFQueue_Tail<RUNTIME_TYPE, ALLOCATOR_TYPE, PROD_CARDINALITY, CONSISTENCY_MODEL>>
    {
      private:
        using Base = detail::LFQueue_Head<
          RUNTIME_TYPE,
          ALLOCATOR_TYPE,
          CONSUMER_CARDINALITY,
          detail::LFQueue_Tail<RUNTIME_TYPE, ALLOCATOR_TYPE, PROD_CARDINALITY, CONSISTENCY_MODEL>>;
        using Base::try_inplace_allocate;
        using typename Base::Allocation;
        using typename Base::Consume;
        using typename Base::ControlBlock;

        enum class PrivateType
        {
        };

      public:
        constexpr static size_t min_alignment = Base::min_alignment;

        using runtime_type    = RUNTIME_TYPE;
        using value_type      = std::pair<const runtime_type &, void * const>;
        using allocator_type  = ALLOCATOR_TYPE;
        using pointer         = value_type *;
        using const_pointer   = const value_type *;
        using reference       = value_type;
        using const_reference = const value_type &;
        using size_type       = std::size_t;
        using difference_type = std::ptrdiff_t;

        static constexpr bool concurrent_puts = PROD_CARDINALITY == concurrency_multiple;

        static constexpr bool concurrent_consumes = CONSUMER_CARDINALITY == concurrency_multiple;

        static constexpr bool concurrent_put_consumes = true;

        static constexpr bool is_seq_cst = CONSISTENCY_MODEL == consistency_sequential;

        static_assert(
          is_power_of_2(ALLOCATOR_TYPE::page_alignment) &&
            ALLOCATOR_TYPE::page_alignment >= ALLOCATOR_TYPE::page_size &&
            (ALLOCATOR_TYPE::page_alignment % min_alignment) == 0,
          "The alignment of the pages must be a power of 2, greater or equal to the size of the "
          "pages, and a multiple of min_alignment");

        static_assert(
          ALLOCATOR_TYPE::page_size > (min_alignment + alignof(ControlBlock)) * 4,
          "Invalid page size");

        constexpr lf_heter_queue() noexcept = default;

        constexpr explicit lf_heter_queue(const ALLOCATOR_TYPE & i_source_allocator) noexcept
            : Base(i_source_allocator)
        {
        }

        constexpr explicit lf_heter_queue(ALLOCATOR_TYPE && i_source_allocator) noexcept
            : Base(std::move(i_source_allocator))
        {
            static_assert(std::is_nothrow_move_constructible<ALLOCATOR_TYPE>::value, "");
        }

        lf_heter_queue(lf_heter_queue && i_source) noexcept = default;

        lf_heter_queue & operator=(lf_heter_queue && i_source) noexcept
        {
            swap(static_cast<Base &>(*this), static_cast<Base &>(i_source));
            return *this;
        }

        allocator_type get_allocator() noexcept { return *this; }

        allocator_type & get_allocator_ref() noexcept { return *this; }

        const allocator_type & get_allocator_ref() const noexcept { return *this; }

        friend void swap(lf_heter_queue & i_first, lf_heter_queue & i_second) noexcept
        {
            swap(static_cast<Base &>(i_first), static_cast<Base &>(i_second));
        }

        ~lf_heter_queue()
        {
            clear();

            Consume consume;
            consume.begin_iteration(this);
            if (!consume.empty())
            {
                consume.clean_dead_elements();
            }
        }

        bool empty() const noexcept { return Consume().is_queue_empty(this); }

        void clear() noexcept
        {
            consume_operation consume;
            while (try_start_consume(consume))
            {
                consume.commit();
            }
        }

        template <typename ELEMENT_COMPLETE_TYPE = void> class put_transaction
        {
            static_assert(
              std::is_same<
                ELEMENT_COMPLETE_TYPE,
                typename std::decay<ELEMENT_COMPLETE_TYPE>::type>::value,
              "");

          public:
            put_transaction() noexcept = default;

            put_transaction(const put_transaction &) = delete;

            put_transaction & operator=(const put_transaction &) = delete;

            template <
              typename OTHERTYPE,
              typename = typename std::enable_if<
                std::is_same<OTHERTYPE, ELEMENT_COMPLETE_TYPE>::value ||
                std::is_void<ELEMENT_COMPLETE_TYPE>::value>::type>
            put_transaction(put_transaction<OTHERTYPE> && i_source) noexcept
                : m_put(i_source.m_put), m_queue(i_source.m_queue)
            {
                i_source.m_put.m_user_storage = nullptr;
            }

            template <
              typename OTHERTYPE,
              typename = typename std::enable_if<
                std::is_same<OTHERTYPE, ELEMENT_COMPLETE_TYPE>::value ||
                std::is_void<ELEMENT_COMPLETE_TYPE>::value>::type>
            put_transaction & operator=(put_transaction<OTHERTYPE> && i_source) noexcept
            {
                using std::swap;
                swap(m_put, i_source.m_put);
                swap(m_queue, i_source.m_queue);
                return *this;
            }

            friend void swap(put_transaction & i_first, put_transaction & i_second) noexcept
            {
                using std::swap;
                swap(i_first.m_put, i_second.m_put);
                swap(i_first.m_queue, i_second.m_queue);
            }

            void * raw_allocate(size_t i_size, size_t i_alignment)
            {
                DENSITY_ASSERT(!empty());
                auto push_data =
                  m_queue->template try_inplace_allocate_impl<detail::LfQueue_Throwing>(
                    detail::LfQueue_Dead, false, i_size, i_alignment);
                return push_data.m_user_storage;
            }

            template <typename INPUT_ITERATOR>
            typename std::iterator_traits<INPUT_ITERATOR>::value_type *
              raw_allocate_copy(INPUT_ITERATOR i_begin, INPUT_ITERATOR i_end)
            {
                using ValueType = typename std::iterator_traits<INPUT_ITERATOR>::value_type;
                static_assert(
                  std::is_trivially_destructible<ValueType>::value,
                  "raw_allocate_copy provides a raw memory in-place allocation that does not "
                  "invoke "
                  "destructors when deallocating");

                auto const count_s = std::distance(i_begin, i_end);
                auto const count   = static_cast<size_t>(count_s);
                DENSITY_ASSUME(static_cast<decltype(count_s)>(count) == count_s);

                auto const elements = static_cast<ValueType *>(
                  raw_allocate(sizeof(ValueType) * count, alignof(ValueType)));
                for (auto curr = elements; i_begin != i_end; ++i_begin, ++curr)
                    new (curr) ValueType(*i_begin);
                return elements;
            }

            template <typename INPUT_RANGE>
            auto raw_allocate_copy(const INPUT_RANGE & i_source_range)
              -> decltype(std::declval<put_transaction>().raw_allocate_copy(
                std::begin(i_source_range), std::end(i_source_range)))
            {
                return raw_allocate_copy(std::begin(i_source_range), std::end(i_source_range));
            }

            void * try_raw_allocate(
              progress_guarantee i_progress_guarantee, size_t i_size, size_t i_alignment) noexcept
            {
                DENSITY_ASSERT(!empty());
                auto push_data = m_queue->try_inplace_allocate(
                  i_progress_guarantee, detail::LfQueue_Dead, false, i_size, i_alignment);
                return push_data.m_user_storage;
            }

            template <typename INPUT_ITERATOR>
            typename std::iterator_traits<INPUT_ITERATOR>::value_type * try_raw_allocate_copy(
              progress_guarantee i_progress_guarantee,
              INPUT_ITERATOR     i_begin,
              INPUT_ITERATOR
                i_end) noexcept(std::
                                  is_nothrow_copy_constructible<typename std::iterator_traits<
                                    INPUT_ITERATOR>::value_type>::value)
            {
                using ValueType = typename std::iterator_traits<INPUT_ITERATOR>::value_type;
                static_assert(
                  std::is_trivially_destructible<ValueType>::value,
                  "raw_allocate_copy provides a raw memory in-place allocation that does not "
                  "invoke "
                  "destructors when deallocating");

                auto const count_s = std::distance(i_begin, i_end);
                auto const count   = static_cast<size_t>(count_s);
                DENSITY_ASSUME(static_cast<decltype(count_s)>(count) == count_s);

                auto const elements = static_cast<ValueType *>(try_raw_allocate(
                  i_progress_guarantee, sizeof(ValueType) * count, alignof(ValueType)));
                if (elements != nullptr)
                {
                    for (auto curr = elements; i_begin != i_end; ++i_begin, ++curr)
                        new (curr) ValueType(*i_begin);
                }
                return elements;
            }

            template <typename INPUT_RANGE>
            auto try_raw_allocate_copy(
              progress_guarantee  i_progress_guarantee,
              const INPUT_RANGE & i_source_range) noexcept(noexcept(std::declval<put_transaction>()
                                                                      .try_raw_allocate_copy(
                                                                        i_progress_guarantee,
                                                                        std::begin(i_source_range),
                                                                        std::end(i_source_range))))
              -> decltype(std::declval<put_transaction>().try_raw_allocate_copy(
                i_progress_guarantee, std::begin(i_source_range), std::end(i_source_range)))
            {
                return try_raw_allocate_copy(
                  i_progress_guarantee, std::begin(i_source_range), std::end(i_source_range));
            }

            void commit() noexcept
            {
                DENSITY_ASSERT(!empty());
                Base::commit_put_impl(m_put);
                m_put.m_user_storage = nullptr;
            }

            void cancel() noexcept
            {
                DENSITY_ASSERT(!empty());
                Base::cancel_put_impl(m_put);
                m_put.m_user_storage = nullptr;
            }

            bool empty() const noexcept { return m_put.m_user_storage == nullptr; }

            explicit operator bool() const noexcept { return m_put.m_user_storage != nullptr; }

            lf_heter_queue * queue() const noexcept
            {
                return m_put.m_user_storage != nullptr ? m_queue : nullptr;
            }

            void * element_ptr() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return m_put.m_user_storage;
            }

#ifndef DOXYGEN_DOC_GENERATION
            template <
              typename EL                                               = ELEMENT_COMPLETE_TYPE,
              typename std::enable_if<!std::is_void<EL>::value>::type * = nullptr>
            EL &
#else
            ELEMENT_COMPLETE_TYPE &
#endif
              element() const noexcept
            {
                return *static_cast<ELEMENT_COMPLETE_TYPE *>(element_ptr());
            }

            const RUNTIME_TYPE & complete_type() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return *Base::type_after_control(m_put.m_control_block);
            }

            ~put_transaction()
            {
                if (m_put.m_user_storage != nullptr)
                {
                    Base::cancel_put_impl(m_put);
                }
            }

            put_transaction(
              PrivateType, lf_heter_queue * i_queue, const Allocation & i_put) noexcept
                : m_put(i_put), m_queue(i_queue)
            {
            }

          private:
            Allocation       m_put;
            lf_heter_queue * m_queue;
            template <typename OTHERTYPE> friend class put_transaction;
        };

        class consume_operation
        {
          public:
            consume_operation() noexcept = default;

            consume_operation(const consume_operation &) = delete;

            consume_operation & operator=(const consume_operation &) = delete;

            consume_operation(consume_operation && i_source) noexcept = default;

            consume_operation & operator=(consume_operation && i_source) noexcept = default;

            ~consume_operation()
            {
                if (!m_consume_data.empty())
                {
                    m_consume_data.cancel_consume_impl();
                }
            }

            friend void swap(consume_operation & i_first, consume_operation & i_second) noexcept
            {
                i_first.m_consume_data.swap(i_second.m_consume_data);
            }

            bool empty() const noexcept { return m_consume_data.empty(); }

            explicit operator bool() const noexcept { return !m_consume_data.empty(); }

            lf_heter_queue * queue() const noexcept
            {
                return static_cast<lf_heter_queue *>(m_consume_data.m_queue);
            }

            void commit() noexcept
            {
                DENSITY_ASSERT(!empty());

                auto const & type    = complete_type();
                auto const   element = element_ptr();
                type.destroy(element);

                type.RUNTIME_TYPE::~RUNTIME_TYPE();

                m_consume_data.commit_consume_impl();
            }

            void commit_nodestroy() noexcept
            {
                DENSITY_ASSERT(!empty());

                bool destroy_type = !std::is_trivially_destructible<RUNTIME_TYPE>::value;
                if (destroy_type)
                {
                    auto const & type = complete_type();
                    type.RUNTIME_TYPE::~RUNTIME_TYPE();
                }

                m_consume_data.commit_consume_impl();
            }

            void cancel() noexcept
            {
                DENSITY_ASSERT(!empty());
                m_consume_data.cancel_consume_impl();
            }

            const RUNTIME_TYPE & complete_type() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return *Base::type_after_control(m_consume_data.m_control);
            }

            void * unaligned_element_ptr() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return Base::get_unaligned_element(
                  m_consume_data.m_control, m_consume_data.external());
            }

            void * element_ptr() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return Base::get_element(m_consume_data.m_control, m_consume_data.external());
            }

            template <typename COMPLETE_ELEMENT_TYPE>
            COMPLETE_ELEMENT_TYPE & element() const noexcept
            {
                DENSITY_ASSERT(!empty() && complete_type().template is<COMPLETE_ELEMENT_TYPE>());
                return *static_cast<COMPLETE_ELEMENT_TYPE *>(
                  Base::get_element(m_consume_data.m_control, m_consume_data.external()));
            }

            consume_operation(PrivateType, lf_heter_queue * i_queue) noexcept
            {
                m_consume_data.start_consume_impl(i_queue);
            }

            bool start_consume_impl(PrivateType, lf_heter_queue * i_queue)
            {
                if (!m_consume_data.empty())
                {
                    m_consume_data.cancel_consume_impl();
                }

                m_consume_data.start_consume_impl(i_queue);

                return !m_consume_data.empty();
            }

          private:
            Consume m_consume_data;
        };

        template <typename ELEMENT_TYPE> void push(ELEMENT_TYPE && i_source)
        {
            return emplace<typename std::decay<ELEMENT_TYPE>::type>(
              std::forward<ELEMENT_TYPE>(i_source));
        }

        template <typename ELEMENT_TYPE, typename... CONSTRUCTION_PARAMS>
        void emplace(CONSTRUCTION_PARAMS &&... i_construction_params)
        {
            start_emplace<ELEMENT_TYPE>(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...)
              .commit();
        }

        void dyn_push(const runtime_type & i_type) { start_dyn_push(i_type).commit(); }

        void dyn_push_copy(const runtime_type & i_type, const void * i_source)
        {
            start_dyn_push_copy(i_type, i_source).commit();
        }

        void dyn_push_move(const runtime_type & i_type, void * i_source)
        {
            start_dyn_push_move(i_type, i_source).commit();
        }

        template <typename ELEMENT_TYPE>
        put_transaction<typename std::decay<ELEMENT_TYPE>::type>
          start_push(ELEMENT_TYPE && i_source)
        {
            return start_emplace<typename std::decay<ELEMENT_TYPE>::type>(
              std::forward<ELEMENT_TYPE>(i_source));
        }

        template <typename ELEMENT_TYPE, typename... CONSTRUCTION_PARAMS>
        put_transaction<ELEMENT_TYPE> start_emplace(CONSTRUCTION_PARAMS &&... i_construction_params)
        {
            auto push_data = Base::template try_inplace_allocate_impl<
              detail::LfQueue_Throwing,
              detail::LfQueue_Busy,
              true,
              detail::size_of<ELEMENT_TYPE>::value,
              alignof(ELEMENT_TYPE)>();

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(runtime_type::template make<ELEMENT_TYPE>());

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                new (push_data.m_user_storage)
                  ELEMENT_TYPE(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();

                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return put_transaction<ELEMENT_TYPE>(PrivateType(), this, push_data);
        }

        put_transaction<> start_dyn_push(const runtime_type & i_type)
        {
            auto push_data = Base::template try_inplace_allocate_impl<detail::LfQueue_Throwing>(
              detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.default_construct(push_data.m_user_storage);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();

                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return put_transaction<void>(PrivateType(), this, push_data);
        }


        put_transaction<> start_dyn_push_copy(const runtime_type & i_type, const void * i_source)
        {
            auto push_data = Base::template try_inplace_allocate_impl<detail::LfQueue_Throwing>(
              detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.copy_construct(push_data.m_user_storage, i_source);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return put_transaction<void>(PrivateType(), this, push_data);
        }

        put_transaction<> start_dyn_push_move(const runtime_type & i_type, void * i_source)
        {
            auto push_data = Base::template try_inplace_allocate_impl<detail::LfQueue_Throwing>(
              detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());
            ;
            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.move_construct(push_data.m_user_storage, i_source);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return put_transaction<void>(PrivateType(), this, push_data);
        }

        template <typename ELEMENT_TYPE>
        bool try_push(progress_guarantee i_progress_guarantee, ELEMENT_TYPE && i_source) noexcept(
          noexcept(std::declval<lf_heter_queue>()
                     .template try_emplace<typename std::decay<ELEMENT_TYPE>::type>(
                       i_progress_guarantee, std::forward<ELEMENT_TYPE>(i_source))))
        {
            return try_emplace<typename std::decay<ELEMENT_TYPE>::type>(
              i_progress_guarantee, std::forward<ELEMENT_TYPE>(i_source));
        }

        template <typename ELEMENT_TYPE, typename... CONSTRUCTION_PARAMS>
        bool
          try_emplace(progress_guarantee i_progress_guarantee, CONSTRUCTION_PARAMS &&... i_construction_params) noexcept(
            noexcept(std::declval<lf_heter_queue>().template try_start_emplace<ELEMENT_TYPE>(
              i_progress_guarantee, std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...)))
        {
            auto tranasction = try_start_emplace<ELEMENT_TYPE>(
              i_progress_guarantee, std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...);
            if (!tranasction)
                return false;
            tranasction.commit();
            return true;
        }

        bool try_dyn_push(progress_guarantee i_progress_guarantee, const runtime_type & i_type)
        {
            auto tranasction = try_start_dyn_push(i_progress_guarantee, i_type);
            if (!tranasction)
                return false;
            tranasction.commit();
            return true;
        }

        bool try_dyn_push_copy(
          progress_guarantee   i_progress_guarantee,
          const runtime_type & i_type,
          const void *         i_source)
        {
            auto tranasction = try_start_dyn_push_copy(i_progress_guarantee, i_type, i_source);
            if (!tranasction)
                return false;
            tranasction.commit();
            return true;
        }

        bool try_dyn_push_move(
          progress_guarantee i_progress_guarantee, const runtime_type & i_type, void * i_source)
        {
            auto tranasction = try_start_dyn_push_move(i_progress_guarantee, i_type, i_source);
            if (!tranasction)
                return false;
            tranasction.commit();
            return true;
        }

        template <typename ELEMENT_TYPE>
        put_transaction<typename std::decay<ELEMENT_TYPE>::type> try_start_push(
          progress_guarantee i_progress_guarantee,
          ELEMENT_TYPE &&    i_source) noexcept(noexcept(std::declval<lf_heter_queue>()
                                                        .template try_start_emplace<
                                                          typename std::decay<ELEMENT_TYPE>::type>(
                                                          i_progress_guarantee,
                                                          std::forward<ELEMENT_TYPE>(i_source))))
        {
            return try_start_emplace<typename std::decay<ELEMENT_TYPE>::type>(
              i_progress_guarantee, std::forward<ELEMENT_TYPE>(i_source));
        }

        template <typename ELEMENT_TYPE, typename... CONSTRUCTION_PARAMS>
        put_transaction<ELEMENT_TYPE>
          try_start_emplace(progress_guarantee i_progress_guarantee, CONSTRUCTION_PARAMS &&... i_construction_params) noexcept(
            noexcept(ELEMENT_TYPE(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...)) &&
            noexcept(runtime_type(runtime_type::template make<ELEMENT_TYPE>())))
        {
            auto push_data = Base::template try_inplace_allocate<
              detail::LfQueue_Busy,
              true,
              detail::size_of<ELEMENT_TYPE>::value,
              alignof(ELEMENT_TYPE)>(i_progress_guarantee);
            if (push_data.m_user_storage == nullptr)
            {
                return put_transaction<ELEMENT_TYPE>();
            }

            constexpr bool is_noexcept =
              std::is_nothrow_constructible<ELEMENT_TYPE, CONSTRUCTION_PARAMS...>::value &&
              noexcept(runtime_type(runtime_type::template make<ELEMENT_TYPE>()));

            runtime_type * type = nullptr;

            if (is_noexcept)
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(runtime_type::template make<ELEMENT_TYPE>());

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                new (push_data.m_user_storage)
                  ELEMENT_TYPE(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...);
            }
            else
            {
                try
                {
                    auto const type_storage = Base::type_after_control(push_data.m_control_block);
                    DENSITY_ASSUME(type_storage != nullptr);
                    type =
                      new (type_storage) runtime_type(runtime_type::template make<ELEMENT_TYPE>());

                    DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                    new (push_data.m_user_storage)
                      ELEMENT_TYPE(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...);
                }
                catch (...)
                {
                    if (type != nullptr)
                        type->RUNTIME_TYPE::~RUNTIME_TYPE();

                    Base::cancel_put_nodestroy_impl(push_data);
                    DENSITY_INTERNAL_RETHROW_FROM_NOEXCEPT
                }
            }

            return put_transaction<ELEMENT_TYPE>(PrivateType(), this, push_data);
        }

        put_transaction<>
          try_start_dyn_push(progress_guarantee i_progress_guarantee, const runtime_type & i_type)
        {
            auto push_data = try_inplace_allocate(
              i_progress_guarantee, detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());
            if (push_data.m_user_storage == nullptr)
            {
                return put_transaction<>();
            }

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.default_construct(push_data.m_user_storage);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();

                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return put_transaction<void>(PrivateType(), this, push_data);
        }


        put_transaction<> try_start_dyn_push_copy(
          progress_guarantee   i_progress_guarantee,
          const runtime_type & i_type,
          const void *         i_source)
        {
            auto push_data = try_inplace_allocate(
              i_progress_guarantee, detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());
            if (push_data.m_user_storage == nullptr)
            {
                return put_transaction<>();
            }

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.copy_construct(push_data.m_user_storage, i_source);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return put_transaction<void>(PrivateType(), this, push_data);
        }

        put_transaction<> try_start_dyn_push_move(
          progress_guarantee i_progress_guarantee, const runtime_type & i_type, void * i_source)
        {
            auto push_data = try_inplace_allocate(
              i_progress_guarantee, detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());
            if (push_data.m_user_storage == nullptr)
            {
                return put_transaction<>();
            }

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.move_construct(push_data.m_user_storage, i_source);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return put_transaction<void>(PrivateType(), this, push_data);
        }

        bool try_pop() noexcept
        {
            if (auto operation = try_start_consume())
            {
                operation.commit();
                return true;
            }
            return false;
        }

        consume_operation try_start_consume() noexcept
        {
            return consume_operation(PrivateType(), this);
        }

        bool try_start_consume(consume_operation & i_consume) noexcept
        {
            return i_consume.start_consume_impl(PrivateType(), this);
        }


        template <typename ELEMENT_COMPLETE_TYPE = void> class reentrant_put_transaction
        {
            static_assert(
              std::is_same<
                ELEMENT_COMPLETE_TYPE,
                typename std::decay<ELEMENT_COMPLETE_TYPE>::type>::value,
              "");

          public:
            reentrant_put_transaction() noexcept = default;

            reentrant_put_transaction(const reentrant_put_transaction &) = delete;

            reentrant_put_transaction & operator=(const reentrant_put_transaction &) = delete;

            template <
              typename OTHERTYPE,
              typename = typename std::enable_if<
                std::is_same<OTHERTYPE, ELEMENT_COMPLETE_TYPE>::value ||
                std::is_void<ELEMENT_COMPLETE_TYPE>::value>::type>
            reentrant_put_transaction(reentrant_put_transaction<OTHERTYPE> && i_source) noexcept
                : m_put(i_source.m_put), m_queue(i_source.m_queue)
            {
                i_source.m_put.m_user_storage = nullptr;
            }

            template <
              typename OTHERTYPE,
              typename = typename std::enable_if<
                std::is_same<OTHERTYPE, ELEMENT_COMPLETE_TYPE>::value ||
                std::is_void<ELEMENT_COMPLETE_TYPE>::value>::type>
            reentrant_put_transaction &
              operator=(reentrant_put_transaction<OTHERTYPE> && i_source) noexcept
            {
                using std::swap;
                swap(m_put, i_source.m_put);
                swap(m_queue, i_source.m_queue);
                return *this;
            }

            friend void swap(
              reentrant_put_transaction & i_first, reentrant_put_transaction & i_second) noexcept
            {
                using std::swap;
                swap(i_first.m_put, i_second.m_put);
                swap(i_first.m_queue, i_second.m_queue);
            }

            void * raw_allocate(size_t i_size, size_t i_alignment)
            {
                DENSITY_ASSERT(!empty());
                auto push_data =
                  m_queue->template try_inplace_allocate_impl<detail::LfQueue_Throwing>(
                    detail::LfQueue_Dead, false, i_size, i_alignment);
                return push_data.m_user_storage;
            }

            template <typename INPUT_ITERATOR>
            typename std::iterator_traits<INPUT_ITERATOR>::value_type *
              raw_allocate_copy(INPUT_ITERATOR i_begin, INPUT_ITERATOR i_end)
            {
                using ValueType = typename std::iterator_traits<INPUT_ITERATOR>::value_type;
                static_assert(
                  std::is_trivially_destructible<ValueType>::value,
                  "raw_allocate_copy provides a raw memory in-place allocation that does not "
                  "invoke "
                  "destructors when deallocating");

                auto const count_s = std::distance(i_begin, i_end);
                auto const count   = static_cast<size_t>(count_s);
                DENSITY_ASSUME(static_cast<decltype(count_s)>(count) == count_s);

                auto const elements = static_cast<ValueType *>(
                  raw_allocate(sizeof(ValueType) * count, alignof(ValueType)));
                for (auto curr = elements; i_begin != i_end; ++i_begin, ++curr)
                    new (curr) ValueType(*i_begin);
                return elements;
            }

            template <typename INPUT_RANGE>
            auto raw_allocate_copy(const INPUT_RANGE & i_source_range)
              -> decltype(std::declval<reentrant_put_transaction>().raw_allocate_copy(
                std::begin(i_source_range), std::end(i_source_range)))
            {
                return raw_allocate_copy(std::begin(i_source_range), std::end(i_source_range));
            }

            void * try_raw_allocate(
              progress_guarantee i_progress_guarantee, size_t i_size, size_t i_alignment) noexcept
            {
                DENSITY_ASSERT(!empty());
                auto push_data = m_queue->try_inplace_allocate(
                  i_progress_guarantee, detail::LfQueue_Dead, false, i_size, i_alignment);
                return push_data.m_user_storage;
            }

            template <typename INPUT_ITERATOR>
            typename std::iterator_traits<INPUT_ITERATOR>::value_type * try_raw_allocate_copy(
              progress_guarantee i_progress_guarantee,
              INPUT_ITERATOR     i_begin,
              INPUT_ITERATOR
                i_end) noexcept(std::
                                  is_nothrow_copy_constructible<typename std::iterator_traits<
                                    INPUT_ITERATOR>::value_type>::value)
            {
                using ValueType = typename std::iterator_traits<INPUT_ITERATOR>::value_type;
                static_assert(
                  std::is_trivially_destructible<ValueType>::value,
                  "raw_allocate_copy provides a raw memory in-place allocation that does not "
                  "invoke "
                  "destructors when deallocating");

                auto const count_s = std::distance(i_begin, i_end);
                auto const count   = static_cast<size_t>(count_s);
                DENSITY_ASSUME(static_cast<decltype(count_s)>(count) == count_s);

                auto const elements = static_cast<ValueType *>(try_raw_allocate(
                  i_progress_guarantee, sizeof(ValueType) * count, alignof(ValueType)));
                if (elements != nullptr)
                {
                    for (auto curr = elements; i_begin != i_end; ++i_begin, ++curr)
                        new (curr) ValueType(*i_begin);
                }
                return elements;
            }

            template <typename INPUT_RANGE>
            auto try_raw_allocate_copy(
              progress_guarantee i_progress_guarantee,
              const INPUT_RANGE &
                i_source_range) noexcept(noexcept(std::declval<reentrant_put_transaction>()
                                                    .try_raw_allocate_copy(
                                                      i_progress_guarantee,
                                                      std::begin(i_source_range),
                                                      std::end(i_source_range))))
              -> decltype(std::declval<reentrant_put_transaction>().try_raw_allocate_copy(
                i_progress_guarantee, std::begin(i_source_range), std::end(i_source_range)))
            {
                return try_raw_allocate_copy(
                  i_progress_guarantee, std::begin(i_source_range), std::end(i_source_range));
            }

            void commit() noexcept
            {
                DENSITY_ASSERT(!empty());
                Base::commit_put_impl(m_put);
                m_put.m_user_storage = nullptr;
            }

            void cancel() noexcept
            {
                DENSITY_ASSERT(!empty());
                Base::cancel_put_impl(m_put);
                m_put.m_user_storage = nullptr;
            }

            bool empty() const noexcept { return m_put.m_user_storage == nullptr; }

            explicit operator bool() const noexcept { return m_put.m_user_storage != nullptr; }

            lf_heter_queue * queue() const noexcept
            {
                return m_put.m_user_storage != nullptr ? m_queue : nullptr;
            }

            void * element_ptr() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return m_put.m_user_storage;
            }

#ifndef DOXYGEN_DOC_GENERATION
            template <
              typename EL                                               = ELEMENT_COMPLETE_TYPE,
              typename std::enable_if<!std::is_void<EL>::value>::type * = nullptr>
            EL &
#else
            ELEMENT_COMPLETE_TYPE &
#endif
              element() const noexcept
            {
                return *static_cast<ELEMENT_COMPLETE_TYPE *>(element_ptr());
            }

            const RUNTIME_TYPE & complete_type() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return *Base::type_after_control(m_put.m_control_block);
            }

            ~reentrant_put_transaction()
            {
                if (m_put.m_user_storage != nullptr)
                {
                    Base::cancel_put_impl(m_put);
                }
            }

            reentrant_put_transaction(
              PrivateType, lf_heter_queue * i_queue, const Allocation & i_put) noexcept
                : m_put(i_put), m_queue(i_queue)
            {
            }

          private:
            Allocation       m_put;
            lf_heter_queue * m_queue = nullptr;
            template <typename OTHERTYPE> friend class reentrant_put_transaction;
        };


        class reentrant_consume_operation
        {
          public:
            reentrant_consume_operation() noexcept = default;

            reentrant_consume_operation(const reentrant_consume_operation &) = delete;

            reentrant_consume_operation & operator=(const reentrant_consume_operation &) = delete;

            reentrant_consume_operation(reentrant_consume_operation && i_source) noexcept = default;

            reentrant_consume_operation &
              operator=(reentrant_consume_operation && i_source) noexcept = default;

            ~reentrant_consume_operation()
            {
                if (!m_consume_data.empty())
                {
                    m_consume_data.cancel_consume_impl();
                }
            }

            friend void swap(
              reentrant_consume_operation & i_first,
              reentrant_consume_operation & i_second) noexcept
            {
                i_first.m_consume_data.swap(i_second.m_consume_data);
            }

            bool empty() const noexcept { return m_consume_data.empty(); }

            explicit operator bool() const noexcept { return !m_consume_data.empty(); }

            lf_heter_queue * queue() const noexcept
            {
                return static_cast<lf_heter_queue *>(m_consume_data.m_queue);
            }

            void commit() noexcept
            {
                DENSITY_ASSERT(!empty());

                auto const & type    = complete_type();
                auto const   element = element_ptr();
                type.destroy(element);

                type.RUNTIME_TYPE::~RUNTIME_TYPE();

                m_consume_data.commit_consume_impl();
            }

            void commit_nodestroy() noexcept
            {
                DENSITY_ASSERT(!empty());

                bool destroy_type = !std::is_trivially_destructible<RUNTIME_TYPE>::value;
                if (destroy_type)
                {
                    auto const & type = complete_type();
                    type.RUNTIME_TYPE::~RUNTIME_TYPE();
                }

                m_consume_data.commit_consume_impl();
            }

            void cancel() noexcept
            {
                DENSITY_ASSERT(!empty());
                m_consume_data.cancel_consume_impl();
            }

            const RUNTIME_TYPE & complete_type() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return *Base::type_after_control(m_consume_data.m_control);
            }

            void * unaligned_element_ptr() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return Base::get_unaligned_element(
                  m_consume_data.m_control, m_consume_data.external());
            }

            void * element_ptr() const noexcept
            {
                DENSITY_ASSERT(!empty());
                return Base::get_element(m_consume_data.m_control, m_consume_data.external());
            }

            template <typename COMPLETE_ELEMENT_TYPE>
            COMPLETE_ELEMENT_TYPE & element() const noexcept
            {
                DENSITY_ASSERT(!empty() && complete_type().template is<COMPLETE_ELEMENT_TYPE>());
                return *static_cast<COMPLETE_ELEMENT_TYPE *>(
                  Base::get_element(m_consume_data.m_control, m_consume_data.external()));
            }

            reentrant_consume_operation(PrivateType, lf_heter_queue * i_queue) noexcept
            {
                m_consume_data.start_consume_impl(i_queue);
            }

            bool start_consume_impl(PrivateType, lf_heter_queue * i_queue)
            {
                if (!m_consume_data.empty())
                {
                    m_consume_data.cancel_consume_impl();
                }

                m_consume_data.start_consume_impl(i_queue);

                return !m_consume_data.empty();
            }

          private:
            Consume m_consume_data;
        };

        template <typename ELEMENT_TYPE> void reentrant_push(ELEMENT_TYPE && i_source)
        {
            return reentrant_emplace<typename std::decay<ELEMENT_TYPE>::type>(
              std::forward<ELEMENT_TYPE>(i_source));
        }

        template <typename ELEMENT_TYPE, typename... CONSTRUCTION_PARAMS>
        void reentrant_emplace(CONSTRUCTION_PARAMS &&... i_construction_params)
        {
            start_reentrant_emplace<ELEMENT_TYPE>(
              std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...)
              .commit();
        }

        void reentrant_dyn_push(const runtime_type & i_type)
        {
            start_reentrant_dyn_push(i_type).commit();
        }

        void reentrant_dyn_push_copy(const runtime_type & i_type, const void * i_source)
        {
            start_reentrant_dyn_push_copy(i_type, i_source).commit();
        }

        void reentrant_dyn_push_move(const runtime_type & i_type, void * i_source)
        {
            start_reentrant_dyn_push_move(i_type, i_source).commit();
        }

        template <typename ELEMENT_TYPE>
        reentrant_put_transaction<typename std::decay<ELEMENT_TYPE>::type>
          start_reentrant_push(ELEMENT_TYPE && i_source)
        {
            return start_reentrant_emplace<typename std::decay<ELEMENT_TYPE>::type>(
              std::forward<ELEMENT_TYPE>(i_source));
        }

        template <typename ELEMENT_TYPE, typename... CONSTRUCTION_PARAMS>
        reentrant_put_transaction<ELEMENT_TYPE>
          start_reentrant_emplace(CONSTRUCTION_PARAMS &&... i_construction_params)
        {
            auto push_data = Base::template try_inplace_allocate_impl<
              detail::LfQueue_Throwing,
              detail::LfQueue_Busy,
              true,
              detail::size_of<ELEMENT_TYPE>::value,
              alignof(ELEMENT_TYPE)>();

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(runtime_type::template make<ELEMENT_TYPE>());

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                new (push_data.m_user_storage)
                  ELEMENT_TYPE(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return reentrant_put_transaction<ELEMENT_TYPE>(PrivateType(), this, push_data);
        }

        reentrant_put_transaction<> start_reentrant_dyn_push(const runtime_type & i_type)
        {
            auto push_data = Base::template try_inplace_allocate_impl<detail::LfQueue_Throwing>(
              detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.default_construct(push_data.m_user_storage);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return reentrant_put_transaction<void>(PrivateType(), this, push_data);
        }

        reentrant_put_transaction<>
          start_reentrant_dyn_push_copy(const runtime_type & i_type, const void * i_source)
        {
            auto push_data = Base::template try_inplace_allocate_impl<detail::LfQueue_Throwing>(
              detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.copy_construct(push_data.m_user_storage, i_source);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return reentrant_put_transaction<void>(PrivateType(), this, push_data);
        }

        reentrant_put_transaction<>
          start_reentrant_dyn_push_move(const runtime_type & i_type, void * i_source)
        {
            auto push_data = Base::template try_inplace_allocate_impl<detail::LfQueue_Throwing>(
              detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.move_construct(push_data.m_user_storage, i_source);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return reentrant_put_transaction<void>(PrivateType(), this, push_data);
        }

        template <typename ELEMENT_TYPE>
        bool try_reentrant_push(
          progress_guarantee i_progress_guarantee,
          ELEMENT_TYPE &&    i_source) noexcept(noexcept(std::declval<lf_heter_queue>()
                                                        .template try_reentrant_emplace<
                                                          typename std::decay<ELEMENT_TYPE>::type>(
                                                          i_progress_guarantee,
                                                          std::forward<ELEMENT_TYPE>(i_source))))
        {
            return try_reentrant_emplace<typename std::decay<ELEMENT_TYPE>::type>(
              i_progress_guarantee, std::forward<ELEMENT_TYPE>(i_source));
        }

        template <typename ELEMENT_TYPE, typename... CONSTRUCTION_PARAMS>
        bool
          try_reentrant_emplace(progress_guarantee i_progress_guarantee, CONSTRUCTION_PARAMS &&... i_construction_params) noexcept(
            noexcept(
              std::declval<lf_heter_queue>().template try_start_reentrant_emplace<ELEMENT_TYPE>(
                i_progress_guarantee, std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...)))
        {
            auto tranasction = try_start_reentrant_emplace<ELEMENT_TYPE>(
              i_progress_guarantee, std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...);
            if (!tranasction)
                return false;
            tranasction.commit();
            return true;
        }

        bool try_reentrant_dyn_push(
          progress_guarantee i_progress_guarantee, const runtime_type & i_type)
        {
            auto tranasction = try_start_reentrant_dyn_push(i_progress_guarantee, i_type);
            if (!tranasction)
                return false;
            tranasction.commit();
            return true;
        }

        bool try_reentrant_dyn_push_copy(
          progress_guarantee   i_progress_guarantee,
          const runtime_type & i_type,
          const void *         i_source)
        {
            auto tranasction =
              try_start_reentrant_dyn_push_copy(i_progress_guarantee, i_type, i_source);
            if (!tranasction)
                return false;
            tranasction.commit();
            return true;
        }

        bool try_reentrant_dyn_push_move(
          progress_guarantee i_progress_guarantee, const runtime_type & i_type, void * i_source)
        {
            auto tranasction =
              try_start_reentrant_dyn_push_move(i_progress_guarantee, i_type, i_source);
            if (!tranasction)
                return false;
            tranasction.commit();
            return true;
        }

        template <typename ELEMENT_TYPE>
        reentrant_put_transaction<typename std::decay<ELEMENT_TYPE>::type> try_start_reentrant_push(
          progress_guarantee i_progress_guarantee,
          ELEMENT_TYPE &&    i_source) noexcept(noexcept(std::declval<lf_heter_queue>()
                                                        .template try_start_reentrant_emplace<
                                                          typename std::decay<ELEMENT_TYPE>::type>(
                                                          i_progress_guarantee,
                                                          std::forward<ELEMENT_TYPE>(i_source))))
        {
            return try_start_reentrant_emplace<typename std::decay<ELEMENT_TYPE>::type>(
              i_progress_guarantee, std::forward<ELEMENT_TYPE>(i_source));
        }

        template <typename ELEMENT_TYPE, typename... CONSTRUCTION_PARAMS>
        reentrant_put_transaction<ELEMENT_TYPE>
          try_start_reentrant_emplace(progress_guarantee i_progress_guarantee, CONSTRUCTION_PARAMS &&... i_construction_params) noexcept(
            noexcept(ELEMENT_TYPE(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...)) &&
            noexcept(runtime_type(runtime_type::template make<ELEMENT_TYPE>())))
        {
            auto push_data = Base::template try_inplace_allocate<
              detail::LfQueue_Busy,
              true,
              detail::size_of<ELEMENT_TYPE>::value,
              alignof(ELEMENT_TYPE)>(i_progress_guarantee);
            if (push_data.m_user_storage == nullptr)
            {
                return reentrant_put_transaction<ELEMENT_TYPE>();
            }

            constexpr bool is_noexcept =
              std::is_nothrow_constructible<ELEMENT_TYPE, CONSTRUCTION_PARAMS...>::value &&
              noexcept(runtime_type(runtime_type::template make<ELEMENT_TYPE>()));

            runtime_type * type = nullptr;

            if (is_noexcept)
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(runtime_type::template make<ELEMENT_TYPE>());

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                new (push_data.m_user_storage)
                  ELEMENT_TYPE(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...);
            }
            else
            {
                try
                {
                    auto const type_storage = Base::type_after_control(push_data.m_control_block);
                    DENSITY_ASSUME(type_storage != nullptr);
                    type =
                      new (type_storage) runtime_type(runtime_type::template make<ELEMENT_TYPE>());

                    DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                    new (push_data.m_user_storage)
                      ELEMENT_TYPE(std::forward<CONSTRUCTION_PARAMS>(i_construction_params)...);
                }
                catch (...)
                {
                    if (type != nullptr)
                        type->RUNTIME_TYPE::~RUNTIME_TYPE();

                    Base::cancel_put_nodestroy_impl(push_data);
                    DENSITY_INTERNAL_RETHROW_FROM_NOEXCEPT
                }
            }

            return reentrant_put_transaction<ELEMENT_TYPE>(PrivateType(), this, push_data);
        }

        reentrant_put_transaction<> try_start_reentrant_dyn_push(
          progress_guarantee i_progress_guarantee, const runtime_type & i_type)
        {
            auto push_data = try_inplace_allocate(
              i_progress_guarantee, detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());
            if (push_data.m_user_storage == nullptr)
            {
                return reentrant_put_transaction<>();
            };

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.default_construct(push_data.m_user_storage);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();

                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return reentrant_put_transaction<void>(PrivateType(), this, push_data);
        }

        reentrant_put_transaction<> try_start_reentrant_dyn_push_copy(
          progress_guarantee   i_progress_guarantee,
          const runtime_type & i_type,
          const void *         i_source)
        {
            auto push_data = try_inplace_allocate(
              i_progress_guarantee, detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());
            if (push_data.m_user_storage == nullptr)
            {
                return reentrant_put_transaction<>();
            }

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.copy_construct(push_data.m_user_storage, i_source);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return reentrant_put_transaction<void>(PrivateType(), this, push_data);
        }

        reentrant_put_transaction<> try_start_reentrant_dyn_push_move(
          progress_guarantee i_progress_guarantee, const runtime_type & i_type, void * i_source)
        {
            auto push_data = try_inplace_allocate(
              i_progress_guarantee, detail::LfQueue_Busy, true, i_type.size(), i_type.alignment());
            if (push_data.m_user_storage == nullptr)
            {
                return reentrant_put_transaction<>();
            }

            runtime_type * type = nullptr;
            try
            {
                auto const type_storage = Base::type_after_control(push_data.m_control_block);
                DENSITY_ASSUME(type_storage != nullptr);
                type = new (type_storage) runtime_type(i_type);

                DENSITY_ASSUME(push_data.m_user_storage != nullptr);
                i_type.move_construct(push_data.m_user_storage, i_source);
            }
            catch (...)
            {
                if (type != nullptr)
                    type->RUNTIME_TYPE::~RUNTIME_TYPE();
                Base::cancel_put_nodestroy_impl(push_data);
                throw;
            }

            return reentrant_put_transaction<void>(PrivateType(), this, push_data);
        }

        bool try_reentrant_pop() noexcept
        {
            if (auto operation = try_start_reentrant_consume())
            {
                operation.commit();
                return true;
            }
            return false;
        }


        reentrant_consume_operation try_start_reentrant_consume() noexcept
        {
            return reentrant_consume_operation(PrivateType(), this);
        }

        bool try_start_reentrant_consume(reentrant_consume_operation & i_consume) noexcept
        {
            return i_consume.start_consume_impl(PrivateType(), this);
        }
    };

} // namespace density

#ifdef _MSC_VER
#pragma warning(pop)
#endif