From 5a74bfa054e652238d40d01fd05c8c08f01dc172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Sep 2025 10:11:56 +0300 Subject: [PATCH 1/5] MDEV-35049 fixup: Fix SUX_LOCK_GENERIC xtrabackup_backup_func(): Invoke btr_search_sys_create(), because innodb_shutdown() assumes that it will have been called. srv_boot(): Invoke btr_search_sys_create(). This fixes assertion failures in the test innodb.temporary_table. btr_sea::create(): Do not invoke enable(). buf_pool_t::create(): Instead of invoking btr_sea::create(), invoke btr_sea::enable() when needed. --- extra/mariabackup/xtrabackup.cc | 1 + storage/innobase/btr/btr0sea.cc | 2 -- storage/innobase/buf/buf0buf.cc | 5 ++++- storage/innobase/srv/srv0srv.cc | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 5229f707f97c5..090bd99ac4adb 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -5510,6 +5510,7 @@ static bool xtrabackup_backup_func() srv_thread_pool_init(); /* Reset the system variables in the recovery module. */ trx_pool_init(); + btr_search_sys_create(); recv_sys.create(); xb_filters_init(); diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 7604d61d4eec9..1472334e2e15d 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -118,8 +118,6 @@ void btr_sea::create() noexcept { for (partition &part : parts) part.init(); - if (enabled) - enable(); } void btr_sea::alloc(ulint hash_size) noexcept diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index fadd50dc898df..0df5ac488454c 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1506,7 +1506,10 @@ bool buf_pool_t::create() noexcept last_activity_count= srv_get_activity_count(); buf_LRU_old_ratio_update(100 * 3 / 8, false); - btr_search_sys_create(); +#ifdef BTR_CUR_HASH_ADAPT + if (btr_search.enabled) + btr_search.enable(); +#endif #ifdef __linux__ if (srv_operation == SRV_OPERATION_NORMAL) diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 82d5d040b525c..dabc7f4b492e1 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -604,6 +604,7 @@ void srv_boot() buf_dblwr.init(); srv_thread_pool_init(); trx_pool_init(); + btr_search_sys_create(); srv_init(); } From 83a48d2bc84781a289173edb1928e2df29d8519f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Sep 2025 10:12:10 +0300 Subject: [PATCH 2/5] MDEV-29445 fixup for innodb_adaptive_hash_index buf_pool_t::clear_hash_index(): Also clear index->search_info.ref_count --- storage/innobase/btr/btr0sea.cc | 11 +++++++++++ storage/innobase/buf/buf0buf.cc | 2 ++ 2 files changed, 13 insertions(+) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 1472334e2e15d..be3190770c5ff 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1014,6 +1014,7 @@ btr_search_guess_on_hash( if (!btr_search.enabled) { + ut_ad(!index->any_ahi_pages()); ahi_release_and_fail: part.latch.rd_unlock(); fail: @@ -1379,6 +1380,7 @@ static void btr_search_build_page_hash_index(dict_index_t *index, (block_index != index || block->ahi_left_bytes_fields != left_bytes_fields); const bool enabled= btr_search.enabled; + ut_ad(enabled || !index->any_ahi_pages()); part.latch.rd_unlock(); @@ -1462,7 +1464,10 @@ static void btr_search_build_page_hash_index(dict_index_t *index, if (!block->index) { if (!btr_search.enabled) + { + ut_ad(!index->any_ahi_pages()); goto exit_func; + } ut_ad(!block->n_pointers); index->search_info.ref_count++; } @@ -1611,7 +1616,10 @@ void btr_search_update_hash_on_delete(btr_cur_t *cursor) noexcept } } else + { + ut_ad(btr_search.enabled || !index->any_ahi_pages()); part.latch.wr_unlock(); + } } void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept @@ -1723,7 +1731,10 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept locked= true; part.latch.wr_lock(SRW_LOCK_CALL); if (!block->index) + { + ut_ad(btr_search.enabled || !index->any_ahi_pages()); goto unlock_exit; + } ha_insert_for_fold(part, ins_fold, block, ins_rec); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 0df5ac488454c..7aead847db8fd 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -3703,6 +3703,8 @@ ATTRIBUTE_COLD void buf_pool_t::clear_hash_index() noexcept # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ if (index->freed()) garbage.insert(index); + else + index->search_info.ref_count= 0; block->index= nullptr; } From 429c5b18a2da1b79dfa38b1d41b57954248edbc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Sep 2025 10:14:29 +0300 Subject: [PATCH 3/5] MDEV-35049 fixup: Avoid useless rebuild if cmp=0 btr_search_info_update_hash(): Avoid useless rebuild if cmp==0. Consider index(col) on a VARCHAR column for which we have the values col='prefix1' and col='prefix2'. In WHERE col LIKE 'prefix%', page_cur_search_with_match_bytes() would return 0 as well as 0 matched fields and bytes on each side. It turns out that page_cur_search_with_match_bytes() can only return nonzero matched bytes when a binary comparison can be used, such as with VARBINARY or INT columns. Co-developed by: Alessandro Vetere --- storage/innobase/btr/btr0sea.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index be3190770c5ff..15e3228d4dd66 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -562,7 +562,15 @@ static uint32_t btr_search_info_update_hash(const btr_cur_t &cursor) noexcept static_assert(buf_block_t::LEFT_SIDE == 1U << 31, ""); left_bytes_fields= (cmp >= 0) << 31; - if (left_bytes_fields) + if (cmp == 0) + /* Reset to the default case (a single index field). + Without this special handling, we could end up setting totally + useless parameters buf_block_t::LEFT_SIDE | 1 << 16 below + (rebuilding the adaptive hash index on a 1-byte prefix) + for example when page_cur_search_with_match_bytes() finds matches + of LIKE 'a%' in the first index field. */ + left_bytes_fields|= 1; + else if (left_bytes_fields) { if (cursor.up_match >= n_uniq) left_bytes_fields|= n_uniq; From 9654b816f3458fbc22bb0b45841c8c78c3915ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Sep 2025 10:14:59 +0300 Subject: [PATCH 4/5] MDEV-37482: Contention on btr_sea::partition::latch To reduce contention between insert, erase and search, let us mimic commit b08448de64f2af9c849154647bdd61d1725f8928 (MDEV-20612). That is, btr_sea::partition::insert() and btr_sea::partition::erase() will use a combination of a shared btr_sea::partition::latch and a tiny page_hash_latch that is pushed down to the btr_sea::hash_table::array. An exclusive btr_sea::partition::latch will be used in the final part of btr_search_drop_page_hash_index(), where we must guarantee that all entries will be removed, as well as in operations that affect an entire adaptive hash index partition. btr_sea::hash_chain: Chain of ahi_node hash buckets. btr_sea::hash_table: A hash table that includes page_hash_latch interleaved with hash_chain. page_hash_latch::try_lock(): Attempt to acquire an exclusive latch without waiting. btr_search_guess_on_hash(): Acquire also the page_hash_latch in order to prevent a concurrent modification of the hash bucket chain that our lookup is traversing. ha_insert_for_fold(): Remove. Invoke btr_sea::partition::insert() directly. btr_sea::partition::erase(): Add template for indicating whether an exclusive or a shared btr_sea::partition::latch is being held. If the ex=false operation fails to free the memory, btr_search_update_hash_on_delete() will retry with ex=true. btr_sea::partition::cleanup_after_erase(): Add an overload for the case where instead of holding an exclusive latch, we hold a shared latch along with a page_hash_latch. When not holding an exclusve latch, we may fail to free the memory, and the caller has to retry with an exclusive latch. btr_sea::partition::cleanup_after_erase_start(), btr_sea::partition::cleanup_after_erase_finish(): Split from cleanup_after_erase() to reduce the amount of code duplication. btr_sea::partition::block_mutex: Protect only the linked list of blocks. The spare block will exclusively be updated via Atomic_relaxed::exchange(). btr_sea::partition::rollback_insert(): Free the spare block in the unlikely event that the adaptive hash index has been disabled after our invocation of btr_sea::partition::prepare_insert(). ha_remove_all_nodes_to_page(): Merged to the only caller btr_search_drop_page_hash_index(). ssux_lock_impl::wr_rd_downgrade(): Downgrade an X latch to S. srw_lock_debug::wr_rd_downgrade(), srw_lock_impl::wr_rd_downgrade(): Downgrade from exclusive to shared. This operation is unavailable if _WIN32 or SUX_LOCK_GENERIC is defined. btr_search_build_page_hash_index(): Downgrade from exclusive to shared part.latch before starting to insert records into the adaptive hash index. In multi-batch operation, we preserve the last fr[] value in order to ensure the correct operation when buf_block_t::LEFT_SIDE is not set. --- storage/innobase/btr/btr0sea.cc | 723 ++++++++++++++-------- storage/innobase/include/btr0sea.h | 160 ++++- storage/innobase/include/buf0buf.h | 7 +- storage/innobase/include/buf0types.h | 4 + storage/innobase/include/srw_lock.h | 44 +- storage/innobase/srv/srv0srv.cc | 8 +- storage/innobase/sync/srw_lock.cc | 12 + storage/innobase/unittest/innodb_ahi-t.cc | 15 +- 8 files changed, 684 insertions(+), 289 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 15e3228d4dd66..4d6cc92154b06 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -64,6 +64,24 @@ struct ahi_node { #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ }; +template +inline ahi_node *btr_sea::hash_chain::find(UnaryPred u) const noexcept +{ + ahi_node *node= first; + while (node && !u(node)) + node= node->next; + return node; +} + +template +inline ahi_node **btr_sea::hash_chain::search(UnaryPred u) noexcept +{ + ahi_node **prev= &first; + while (!u(*prev)) + prev= &(*prev)->next; + return prev; +} + inline void btr_sea::partition::init() noexcept { latch.SRW_LOCK_INIT(btr_search_latch_key); @@ -73,13 +91,9 @@ inline void btr_sea::partition::init() noexcept inline void btr_sea::partition::clear() noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_write_locked()); - ut_ad(blocks_mutex.is_locked()); -#endif - if (buf_block_t *b= spare) + ut_ad(latch.have_wr()); + if (buf_block_t *b= spare.exchange(nullptr)) { - spare= nullptr; MEM_MAKE_ADDRESSABLE(b->page.frame, srv_page_size); buf_pool.free_block(b); } @@ -100,15 +114,23 @@ inline void btr_sea::partition::free() noexcept if (table.array) { ut_d(latch.wr_lock(SRW_LOCK_CALL)); - ut_d(blocks_mutex.wr_lock()); clear(); - ut_d(blocks_mutex.wr_unlock()); ut_d(latch.wr_unlock()); } latch.destroy(); blocks_mutex.destroy(); } +inline void btr_sea::hash_table::create(ulint n) noexcept +{ + n_cells= ut_find_prime(n); + const size_t size= MY_ALIGN(pad(n_cells) * sizeof *array, + CPU_LEVEL1_DCACHE_LINESIZE); + void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); + memset_aligned(v, 0, size); + array= static_cast(v); +} + inline void btr_sea::partition::alloc(ulint hash_size) noexcept { table.create(hash_size); @@ -300,16 +322,25 @@ void btr_sea::partition::prepare_insert() noexcept if (!spare) { buf_block_t *block= buf_block_alloc(); - blocks_mutex.wr_lock(); - if (!spare && btr_search.enabled) - { - MEM_NOACCESS(block->page.frame, srv_page_size); - spare= block; - block= nullptr; - } - blocks_mutex.wr_unlock(); + MEM_NOACCESS(block->page.frame, srv_page_size); + block= spare.exchange(block); if (block) + { + MEM_MAKE_ADDRESSABLE(block->page.frame, srv_page_size); buf_pool.free_block(block); + } + } +} + +ATTRIBUTE_COLD ATTRIBUTE_NOINLINE +void btr_sea::partition::rollback_insert() noexcept +{ + ut_ad(latch.have_any()); + ut_ad(!btr_search.enabled); + if (buf_block_t *block= spare.exchange(nullptr)) + { + MEM_MAKE_ADDRESSABLE(block->page.frame, srv_page_size); + buf_pool.free_block(block); } } @@ -355,10 +386,7 @@ ATTRIBUTE_COLD bool btr_sea::disable() noexcept dict_sys.freeze(SRW_LOCK_CALL); for (ulong i= 0; i < n_parts; i++) - { parts[i].latch.wr_lock(SRW_LOCK_CALL); - parts[i].blocks_mutex.wr_lock(); - } const bool was_enabled{enabled}; @@ -377,10 +405,7 @@ ATTRIBUTE_COLD bool btr_sea::disable() noexcept dict_sys.unfreeze(); for (ulong i= 0; i < n_parts; i++) - { parts[i].latch.wr_unlock(); - parts[i].blocks_mutex.wr_unlock(); - } return was_enabled; } @@ -399,10 +424,7 @@ ATTRIBUTE_COLD void btr_sea::enable(bool resize) noexcept } for (ulong i= 0; i < n_parts; i++) - { parts[i].latch.wr_lock(SRW_LOCK_CALL); - parts[i].blocks_mutex.wr_lock(); - } if (!parts[0].table.array) { @@ -413,17 +435,107 @@ ATTRIBUTE_COLD void btr_sea::enable(bool resize) noexcept ut_ad(enabled); for (ulong i= 0; i < n_parts; i++) - { - parts[i].blocks_mutex.wr_unlock(); parts[i].latch.wr_unlock(); - } } #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG -# define ha_insert_for_fold(p,f,b,d) (p).insert(f,d,b) +/** Maximum number of records in a page */ +constexpr ulint MAX_N_POINTERS = UNIV_PAGE_SIZE_MAX / REC_N_NEW_EXTRA_BYTES; + +void btr_sea::partition::insert(uint32 fold, const rec_t *rec, + buf_block_t *block) noexcept #else -# define ha_insert_for_fold(p,f,b,d) (p).insert(f,d) +void btr_sea::partition::insert(uint32_t fold, const rec_t *rec) noexcept +# define insert(fold,rec,block) insert(fold,rec) +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ +{ + ut_ad(latch.have_rd()); +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + ut_a(block->page.frame == page_align(rec)); +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + ut_ad(btr_search.enabled); + + hash_chain &cell{table.cell_get(fold)}; + page_hash_latch &hash_lock{table.lock_get(cell)}; + hash_lock.lock(); + + ahi_node **prev= cell.search([fold](const ahi_node *node) + { return !node || node->fold == fold; }); + ahi_node *node= *prev; + + if (node) + { +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + { + buf_block_t *prev_block= node->block; + if (prev_block != block) + { + ut_a(prev_block->page.frame == page_align(node->rec)); + ut_a(prev_block->n_pointers-- < MAX_N_POINTERS); + ut_a(block->n_pointers++ < MAX_N_POINTERS); + node->block= block; + } + } +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + node->rec= rec; + unlock: + hash_lock.unlock(); + return; + } + + /* We have to allocate a new chain node */ + + { + blocks_mutex.wr_lock(); + buf_page_t *last= UT_LIST_GET_LAST(blocks); + if (last && last->free_offset < srv_page_size - sizeof *node) + { + node= reinterpret_cast(last->frame + last->free_offset); +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic push +# if __GNUC__ < 12 || defined WITH_UBSAN +# pragma GCC diagnostic ignored "-Wconversion" +# endif #endif + last->free_offset+= sizeof *node; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic pop +#endif + MEM_MAKE_ADDRESSABLE(node, sizeof *node); + } + else + { + last= reinterpret_cast(spare.exchange(nullptr)); + if (!last) + { + blocks_mutex.wr_unlock(); + goto unlock; + } + ut_ad(last->state() == buf_page_t::MEMORY); + ut_ad(!reinterpret_cast(last)->index); + ut_ad(!reinterpret_cast(last)->n_pointers); + UT_LIST_ADD_LAST(blocks, last); + last->free_offset= sizeof *node; + node= reinterpret_cast(last->frame); + MEM_UNDEFINED(last->frame, srv_page_size); + MEM_MAKE_ADDRESSABLE(node, sizeof *node); + MEM_NOACCESS(node + 1, srv_page_size - sizeof *node); + } + blocks_mutex.wr_unlock(); + } + +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + ut_a(block->n_pointers++ < MAX_N_POINTERS); + node->block= block; +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + node->rec= rec; + + node->fold= fold; + node->next= nullptr; + + *prev= node; + goto unlock; +} ATTRIBUTE_NOINLINE /** Update a hash node reference when it has been unsuccessfully used in a @@ -449,7 +561,7 @@ static void btr_search_update_hash_ref(const btr_cur_t &cursor, ut_ad(block->page.id().space() == index->table->space_id); btr_sea::partition &part= btr_search.get_part(index->id); part.prepare_insert(); - part.latch.wr_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); if (ut_d(const dict_index_t *block_index=) block->index) { @@ -485,16 +597,23 @@ static void btr_search_update_hash_ref(const btr_cur_t &cursor, } } - ha_insert_for_fold(part, fold, block, rec); + part.insert(fold, rec, block); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } -# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG else + { +# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG ut_a(!block->n_pointers); # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + if (!btr_search.enabled) + { + ut_ad(!index->any_ahi_pages()); + part.rollback_insert(); + } + } skip: - part.latch.wr_unlock(); + part.latch.rd_unlock(); } /** Updates the search info of an index about hash successes. @@ -652,214 +771,187 @@ static uint32_t btr_search_info_update_hash(const btr_cur_t &cursor) noexcept return ret; } -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG -/** Maximum number of records in a page */ -constexpr ulint MAX_N_POINTERS = UNIV_PAGE_SIZE_MAX / REC_N_NEW_EXTRA_BYTES; -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG -void btr_sea::partition::insert(uint32 fold, const rec_t *rec, - buf_block_t *block) noexcept -#else -void btr_sea::partition::insert(uint32_t fold, const rec_t *rec) noexcept -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ +inline ahi_node *btr_sea::partition::cleanup_after_erase_start() noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_write_locked()); -#endif -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - ut_a(block->page.frame == page_align(rec)); -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - ut_ad(btr_search.enabled); - - ahi_node **prev= table.cell_get(fold)-> - search(&ahi_node::next, [fold](const ahi_node *node) - { return !node || node->fold == fold; }); - ahi_node *node= *prev; - - if (node) - { -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - buf_block_t *prev_block= node->block; - if (prev_block != block) - { - ut_a(prev_block->page.frame == page_align(node->rec)); - ut_a(prev_block->n_pointers-- < MAX_N_POINTERS); - ut_a(block->n_pointers++ < MAX_N_POINTERS); - node->block= block; - } -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - node->rec= rec; - return; - } + blocks_mutex.wr_lock(); - /* We have to allocate a new chain node */ + buf_page_t *last= UT_LIST_GET_LAST(blocks); + return reinterpret_cast + (last->frame + last->free_offset - sizeof(ahi_node)); +} - { - blocks_mutex.wr_lock(); - buf_page_t *last= UT_LIST_GET_LAST(blocks); - if (last && last->free_offset < srv_page_size - sizeof *node) - { - node= reinterpret_cast(last->frame + last->free_offset); +buf_block_t *btr_sea::partition::cleanup_after_erase_finish() noexcept +{ + buf_block_t *freed= nullptr; + buf_page_t *last= UT_LIST_GET_LAST(blocks); #if defined __GNUC__ && !defined __clang__ # pragma GCC diagnostic push # if __GNUC__ < 12 || defined WITH_UBSAN # pragma GCC diagnostic ignored "-Wconversion" # endif #endif - last->free_offset+= sizeof *node; + /* We may be able to shrink or free the last block */ + if (!(last->free_offset-= uint16_t(sizeof(ahi_node)))) #if defined __GNUC__ && !defined __clang__ # pragma GCC diagnostic pop #endif - MEM_MAKE_ADDRESSABLE(node, sizeof *node); - } - else - { - last= &spare.load()->page; - if (!last) - { - blocks_mutex.wr_unlock(); - return; - } - spare= nullptr; - ut_ad(last->state() == buf_page_t::MEMORY); - ut_ad(!reinterpret_cast(last)->index); - ut_ad(!reinterpret_cast(last)->n_pointers); - UT_LIST_ADD_LAST(blocks, last); - last->free_offset= sizeof *node; - node= reinterpret_cast(last->frame); - MEM_UNDEFINED(last->frame, srv_page_size); - MEM_MAKE_ADDRESSABLE(node, sizeof *node); - MEM_NOACCESS(node + 1, srv_page_size - sizeof *node); - } - blocks_mutex.wr_unlock(); - } + { + UT_LIST_REMOVE(blocks, last); -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - ut_a(block->n_pointers++ < MAX_N_POINTERS); - node->block= block; -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - node->rec= rec; + freed= spare.exchange(reinterpret_cast(last)); - node->fold= fold; - node->next= nullptr; + if (freed) + MEM_MAKE_ADDRESSABLE(freed->page.frame, srv_page_size); + } + else + MEM_NOACCESS(last->frame + last->free_offset, sizeof(ahi_node)); - *prev= node; + blocks_mutex.wr_unlock(); + return freed; } -buf_block_t *btr_sea::partition::cleanup_after_erase(ahi_node *erase) noexcept +buf_block_t * +btr_sea::partition::cleanup_after_erase(ahi_node *erase) noexcept { + ut_ad(latch.have_wr()); ut_ad(btr_search.enabled); -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_write_locked()); -#endif -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - ut_a(erase->block->page.frame == page_align(erase->rec)); - ut_a(erase->block->n_pointers-- < MAX_N_POINTERS); -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - blocks_mutex.wr_lock(); - - buf_page_t *last= UT_LIST_GET_LAST(blocks); - const ahi_node *const top= reinterpret_cast - (last->frame + last->free_offset - sizeof *top); + const ahi_node *const top= cleanup_after_erase_start(); if (erase != top) { + hash_chain &top_cell{table.cell_get(top->fold)}; + ahi_node **prev= + top_cell.search([top](const ahi_node *n) { return n == top; }); /* Shrink the allocation by replacing the erased element with the top. */ *erase= *top; - ahi_node **prev= table.cell_get(top->fold)-> - search(&ahi_node::next, [top](const ahi_node *n) { return n == top; }); *prev= erase; } - buf_block_t *freed= nullptr; - -#if defined __GNUC__ && !defined __clang__ -# pragma GCC diagnostic push -# if __GNUC__ < 12 || defined WITH_UBSAN -# pragma GCC diagnostic ignored "-Wconversion" -# endif -#endif - /* We may be able to shrink or free the last block */ - if (!(last->free_offset-= uint16_t(sizeof *erase))) -#if defined __GNUC__ && !defined __clang__ -# pragma GCC diagnostic pop -#endif - { - if (spare) - { - freed= reinterpret_cast(last); - MEM_MAKE_ADDRESSABLE(last->frame, srv_page_size); - } - else - spare= reinterpret_cast(last); - UT_LIST_REMOVE(blocks, last); - } - else - MEM_NOACCESS(last->frame + last->free_offset, sizeof *erase); - - blocks_mutex.wr_unlock(); - return freed; + return cleanup_after_erase_finish(); } -__attribute__((nonnull)) -/** Delete all pointers to a page. -@param part hash table partition -@param fold CRC-32C value -@param page page of a record to be deleted */ -static void ha_remove_all_nodes_to_page(btr_sea::partition &part, - uint32_t fold, const page_t *page) +buf_block_t * +btr_sea::partition::cleanup_after_erase(ahi_node *erase, page_hash_latch *l) noexcept { - hash_cell_t *cell= part.table.cell_get(fold); - const uintptr_t page_size{srv_page_size}; + ut_ad(latch.have_rd()); + ut_ad(l->is_write_locked()); + ut_ad(btr_search.enabled); -rewind: - ahi_node **prev= - cell->search(&ahi_node::next, [page,page_size](const ahi_node *node) - { return !node || (uintptr_t(node->rec) ^ uintptr_t(page)) < page_size; }); + const ahi_node *const top= cleanup_after_erase_start(); - if (ahi_node *node= *prev) + if (erase != top) { - *prev= node->next; - node->next= nullptr; - if (buf_block_t *block= part.cleanup_after_erase(node)) - buf_pool.free_block(block); - /* The deletion may compact the heap of nodes and move other nodes! */ - goto rewind; + hash_chain &top_cell{table.cell_get(top->fold)}; + page_hash_latch &top_latch{table.lock_get(top_cell)}; + + if (l == &top_latch) + /* We are already holding top_latch. */ + l= nullptr; + else + { + /* A lock wait below would easily lead to a deadlock due to + lock order inversion. A concurrent cleanup_after_erase() + could hold top_lock while waiting for our blocks_mutex. */ + l= &top_latch; + if (UNIV_UNLIKELY(!l->try_lock())) + { + retry: + /* Tell the caller to retry with l=nullptr. */ + blocks_mutex.wr_unlock(); + return reinterpret_cast(-1); + } + } + + ahi_node **prev= + top_cell.search([top](const ahi_node *n) { return !n || n == top; }); + if (UNIV_UNLIKELY(!*prev)) + { + /* A concurrent thread, which would eventually end up waiting + for blocks_mutex in cleanup_after_erase(), may have detached top + from top_cell while we were not holding top_latch. */ + if (l) + l->unlock(); + goto retry; + } + + /* Shrink the allocation by replacing the erased element with the top. */ + *erase= *top; + *prev= erase; + if (l) + l->unlock(); } - /* Check that all nodes really got deleted */ - ut_ad(!cell->find(&ahi_node::next, [page](const ahi_node* node) - { return page_align(node->rec) == page; })); + return cleanup_after_erase_finish(); } -inline bool btr_sea::partition::erase(uint32_t fold, const rec_t *rec) noexcept -{ -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_write_locked()); +#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 15 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wnonnull" /* if (ex) hash_lock==nullptr */ #endif +template +btr_sea::partition::erase_status +btr_sea::partition::erase(hash_chain &cell, const rec_t *rec) noexcept +{ + ut_ad(ex ? latch.have_wr() : latch.have_rd()); ut_ad(btr_search.enabled); - ahi_node **prev= table.cell_get(fold)-> - search(&ahi_node::next, [rec](const ahi_node *node) - { return !node || node->rec == rec; }); + page_hash_latch *const hash_lock{ex ? nullptr : &table.lock_get(cell)}; + buf_block_t *block= nullptr; + if (!ex) hash_lock->lock(); + + ahi_node **prev= cell.search([rec](const ahi_node *node) + { return (!ex && !node) || node->rec == rec; }); + + ahi_node *node= *prev; - if (ahi_node *node= *prev) + if (ex || node) { +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + buf_block_t *const node_block{node->block}; +#endif *prev= node->next; - node->next= nullptr; - buf_block_t *block= cleanup_after_erase(node); + if (ex) + block= cleanup_after_erase(node); + else + block= cleanup_after_erase(node, hash_lock); + ut_ad(!ex || block != reinterpret_cast(-1)); + if (!ex && block == reinterpret_cast(-1)) + *prev= node; /* We could not free the memory: roll back */ +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + else + { + ut_a(node_block->page.frame == page_align(rec)); + ut_a(node_block->n_pointers-- < MAX_N_POINTERS); + } +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + } + + if (ex) latch.wr_unlock(); - if (block) - buf_pool.free_block(block); - return true; + else + { + hash_lock->unlock(); + latch.rd_unlock(); } - latch.wr_unlock(); - return false; + if (!node) + return NOT_ERASED; + + if (block && block != reinterpret_cast(-1)) + { + buf_pool.free_block(block); + block= nullptr; + } + + static_assert(uintptr_t(ERASE_RETRY) == uintptr_t(-1), ""); + static_assert(uintptr_t(ERASED) == 0, ""); + return erase_status(uintptr_t(block)); } +#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 15 +# pragma GCC diagnostic pop +#endif __attribute__((nonnull)) /** Looks for an element when we know the pointer to the data and @@ -869,7 +961,8 @@ updates the pointer to data if found. @param data pointer to the data @param new_data new pointer to the data @return whether the element was found */ -static bool ha_search_and_update_if_found(hash_table_t *table, uint32_t fold, +static bool ha_search_and_update_if_found(btr_sea::hash_table *table, + uint32_t fold, const rec_t *data, #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG /** block containing new_data */ @@ -882,9 +975,12 @@ static bool ha_search_and_update_if_found(hash_table_t *table, uint32_t fold, #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ ut_ad(btr_search.enabled); - if (ahi_node *node= table->cell_get(fold)-> - find(&ahi_node::next, [data](const ahi_node *node) - { return node->rec == data; })) + btr_sea::hash_chain &cell{table->cell_get(fold)}; + page_hash_latch &hash_lock{table->lock_get(cell)}; + hash_lock.lock(); + ahi_node *node= + cell.find([data](const ahi_node *node){ return node->rec == data; }); + if (node) { #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG if (node->block != new_block) @@ -895,10 +991,9 @@ static bool ha_search_and_update_if_found(hash_table_t *table, uint32_t fold, } #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ node->rec= new_data; - return true; } - - return false; + hash_lock.unlock(); + return node != nullptr; } #if !defined UNIV_AHI_DEBUG && !defined UNIV_DEBUG @@ -1017,13 +1112,15 @@ btr_search_guess_on_hash( const uint32_t fold= dtuple_fold(tuple, cursor); cursor->fold= fold; btr_sea::partition &part= btr_search.get_part(*index); - + page_hash_latch *hash_lock= nullptr; part.latch.rd_lock(SRW_LOCK_CALL); if (!btr_search.enabled) { ut_ad(!index->any_ahi_pages()); ahi_release_and_fail: + if (hash_lock) + hash_lock->unlock(); part.latch.rd_unlock(); fail: #ifdef UNIV_SEARCH_PERF_STAT @@ -1035,9 +1132,11 @@ btr_search_guess_on_hash( return false; } - const ahi_node *node= part.table.cell_get(fold)-> - find(&ahi_node::next, [fold](const ahi_node* node) - { return node->fold == fold; }); + btr_sea::hash_chain &cell{part.table.cell_get(fold)}; + hash_lock= &part.table.lock_get(cell); + hash_lock->lock(); + const ahi_node *node= + cell.find([fold](const ahi_node* node){ return node->fold == fold; }); if (!node) { @@ -1086,7 +1185,6 @@ btr_search_guess_on_hash( if (index != block_index && index_id == block_index->id) { ut_a(block_index->freed()); - cursor->flag= BTR_CUR_HASH_FAIL; goto block_and_ahi_release_and_fail; } @@ -1095,6 +1193,8 @@ btr_search_guess_on_hash( Because we are holding block->page.lock, the page cannot be modified or evicted (buf_page_t::can_relocate() will not hold) while we validate the guessed rec. */ + hash_lock->unlock(); + ut_d(hash_lock= reinterpret_cast(-1)); part.latch.rd_unlock(); if (mtr->trx) @@ -1171,16 +1271,16 @@ static void btr_search_drop_page_hash_index(buf_block_t *block, /* We must not dereference block->index here, because it could be freed if (!index->table->get_ref_count() && !dict_sys.frozen()). - Determine the ahi_slot based on the block contents. */ - const index_id_t index_id= btr_page_get_index_id(block->page.frame); - btr_sea::partition &part= btr_search.get_part(index_id); + It is safe to determine the index_id from the page frame. */ + const index_id_t index_id{btr_page_get_index_id(block->page.frame)}; + btr_sea::partition &part{btr_search.get_part(index_id)}; part.latch.rd_lock(SRW_LOCK_CALL); index= block->index; if (!index) { - unlock_and_return: + unfreeze_and_return: part.latch.rd_unlock(); return; } @@ -1201,9 +1301,9 @@ static void btr_search_drop_page_hash_index(buf_block_t *block, } else if (not_garbage != nullptr) { - ut_ad(!index || index == not_garbage || + ut_ad(index == not_garbage || not_garbage == reinterpret_cast(-1)); - goto unlock_and_return; + goto unfreeze_and_return; } assert_block_ahi_valid(block); @@ -1280,20 +1380,49 @@ static void btr_search_drop_page_hash_index(buf_block_t *block, /* Someone else has meanwhile dropped the hash index */ goto cleanup; ut_a(block->index == index); - } - if (block->ahi_left_bytes_fields != left_bytes_fields) - { - /* Someone else has meanwhile built a new hash index on the page, - with different parameters */ - part.latch.wr_unlock(); - goto retry; + if (UNIV_UNLIKELY(block->ahi_left_bytes_fields != left_bytes_fields)) + { + /* Someone else has meanwhile built a new hash index on the page, + with different parameters */ + part.latch.wr_unlock(); + goto retry; + } } + else + ut_ad(block->ahi_left_bytes_fields == left_bytes_fields); MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_REMOVED, n_folds); while (n_folds) - ha_remove_all_nodes_to_page(part, folds[--n_folds], page); + { + btr_sea::hash_chain &cell{part.table.cell_get(folds[--n_folds])}; + rewind: + ahi_node **prev= + cell.search([page](const ahi_node *n) + { return !n || (uintptr_t(n->rec) ^ uintptr_t(page)) < srv_page_size; }); + + if (ahi_node *node= *prev) + { +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + ut_a(node->block == block); + ut_a(block->n_pointers-- < MAX_N_POINTERS); +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + *prev= node->next; + + if (buf_block_t *b= part.cleanup_after_erase(node)) + /* NOTE: We are holding part.latch.wr_lock() while waiting for + buf_pool.mutex. This should be a rather rare occurrence, + because b->page.frame should host hundreds of ahi_node. */ + buf_pool.free_block(b); + /* The deletion may compact the heap of nodes and move other nodes! */ + goto rewind; + } + + /* Check that all nodes really got deleted */ + ut_ad(!cell.find([page](const ahi_node *node) + { return page_align(node->rec) == page; })); + } if (!rec); else if (page_is_comp(page)) @@ -1358,8 +1487,6 @@ void btr_search_drop_page_hash_when_freed(mtr_t *mtr, const page_id_t page_id) /** Build a hash index on a page with the given parameters. If the page already has a hash index with different parameters, the old hash index is removed. -If index is non-NULL, this function checks if n_fields and n_bytes are -sensible, and does not build a hash index if not. @param index B-tree index @param block latched B-tree leaf page @param part the adaptive search partition @@ -1469,35 +1596,71 @@ static void btr_search_build_page_hash_index(dict_index_t *index, part.prepare_insert(); part.latch.wr_lock(SRW_LOCK_CALL); - if (!block->index) + if (ut_d(dict_index_t *b_index=) block->index) { + ut_ad(b_index == index); + if (block->ahi_left_bytes_fields != left_bytes_fields) + { + /* Another thread already built a hash index. */ + unlock_and_exit: + part.latch.wr_unlock(); + return; + } + } + else + { + ut_ad(!block->n_pointers); + if (!btr_search.enabled) { ut_ad(!index->any_ahi_pages()); - goto exit_func; + part.rollback_insert(); + goto unlock_and_exit; } - ut_ad(!block->n_pointers); + + block->n_hash_helps= 0; + block->index= index; + block->ahi_left_bytes_fields= left_bytes_fields; index->search_info.ref_count++; } - else if (block->ahi_left_bytes_fields != left_bytes_fields) - goto exit_func; - block->n_hash_helps= 0; - block->index= index; - block->ahi_left_bytes_fields= left_bytes_fields; +# if defined _WIN32 || defined SUX_LOCK_GENERIC + part.latch.wr_unlock(); + part.latch.rd_lock(SRW_LOCK_CALL); + if (ut_d(dict_index_t *b_index=) block->index) + { + ut_ad(b_index == index); + if (block->ahi_left_bytes_fields != left_bytes_fields) + goto unfreeze_and_exit; + } + else + { + ut_ad(!block->n_pointers); + if (!btr_search.enabled) + { + ut_ad(!index->any_ahi_pages()); + part.rollback_insert(); + } + unfreeze_and_exit: + part.latch.rd_unlock(); + return; + } +# else + part.latch.wr_rd_downgrade(SRW_LOCK_CALL); +# endif MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_ADDED, n_cached); - while (n_cached) + for (size_t i= 0; i < n_cached; i++) { #if SIZEOF_SIZE_T <= 4 - const auto &f= fr[--n_cached]; + const auto &f= fr[i]; const rec_t *rec= reinterpret_cast(f.offset); #else - const auto f= fr[--n_cached]; + const auto f= fr[i]; const rec_t *rec= page + (uint32_t(uintptr_t(page)) ^ f.offset); #endif - ha_insert_for_fold(part, f.fold, block, rec); + part.insert(f.fold, rec, block); } if (!rec); @@ -1505,20 +1668,25 @@ static void btr_search_build_page_hash_index(dict_index_t *index, { if (rec != page + PAGE_NEW_SUPREMUM) { - part.latch.wr_unlock(); + part.latch.rd_unlock(); + ut_ad(n_cached == array_elements(fr)); + fr[0]= fr[array_elements(fr) - 1]; + n_cached= 1; goto next_not_redundant; } } else if (rec != page + PAGE_OLD_SUPREMUM) { - part.latch.wr_unlock(); + part.latch.rd_unlock(); + ut_ad(n_cached == array_elements(fr)); + fr[0]= fr[array_elements(fr) - 1]; + n_cached= 1; goto next_redundant; } MONITOR_INC(MONITOR_ADAPTIVE_HASH_PAGE_ADDED); -exit_func: assert_block_ahi_valid(block); - part.latch.wr_unlock(); + part.latch.rd_unlock(); } void btr_cur_t::search_info_update() const noexcept @@ -1607,14 +1775,34 @@ void btr_search_update_hash_on_delete(btr_cur_t *cursor) noexcept uint32_t fold= rec_fold(rec, *index, n_bytes_fields, page_is_comp(btr_cur_get_page(cursor))); btr_sea::partition &part= btr_search.get_part(*index); - part.latch.wr_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); assert_block_ahi_valid(block); if (ut_d(dict_index_t *block_index=) block->index) { ut_ad(btr_search.enabled); ut_ad(block_index == index); - if (part.erase(fold, rec)) + + btr_sea::partition::erase_status s= + part.erase(part.table.cell_get(fold), rec); + if (s == btr_sea::partition::ERASE_RETRY) + { + part.latch.wr_lock(SRW_LOCK_CALL); + btr_sea::hash_chain &cell{part.table.cell_get(fold)}; + + if (UNIV_LIKELY(cell.first != nullptr)) + { + s= part.erase(cell, rec); + ut_ad(s != btr_sea::partition::ERASE_RETRY); + } + else + { + ut_ad(!index->any_ahi_pages()); + part.latch.wr_unlock(); + } + } + + if (s == btr_sea::partition::ERASED) { MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_REMOVED); } @@ -1626,7 +1814,7 @@ void btr_search_update_hash_on_delete(btr_cur_t *cursor) noexcept else { ut_ad(btr_search.enabled || !index->any_ahi_pages()); - part.latch.wr_unlock(); + part.latch.rd_unlock(); } } @@ -1669,7 +1857,7 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept if (!reorg && cursor->flag == BTR_CUR_HASH && left_bytes_fields == cursor->n_bytes_fields) { - part.latch.wr_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); if (!block->index) goto unlock_exit; ut_ad(btr_search.enabled); @@ -1737,13 +1925,18 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept if (left_bytes_fields & buf_block_t::LEFT_SIDE) { locked= true; - part.latch.wr_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); if (!block->index) { - ut_ad(btr_search.enabled || !index->any_ahi_pages()); + rollback: + if (!btr_search.enabled) + { + ut_ad(!index->any_ahi_pages()); + part.rollback_insert(); + } goto unlock_exit; } - ha_insert_for_fold(part, ins_fold, block, ins_rec); + part.insert(ins_fold, ins_rec, block); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } } @@ -1752,13 +1945,13 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept if (!locked) { locked= true; - part.latch.wr_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); if (!block->index) - goto unlock_exit; + goto rollback; } if (left_bytes_fields & buf_block_t::LEFT_SIDE) fold= ins_fold, rec= ins_rec; - ha_insert_for_fold(part, fold, block, rec); + part.insert(fold, rec, block); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } @@ -1769,11 +1962,11 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept if (!locked) { locked= true; - part.latch.wr_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); if (!block->index) - goto unlock_exit; + goto rollback; } - ha_insert_for_fold(part, ins_fold, block, ins_rec); + part.insert(ins_fold, ins_rec, block); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } } @@ -1782,13 +1975,13 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept if (!locked) { locked= true; - part.latch.wr_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); if (!block->index) - goto unlock_exit; + goto rollback; } if (!(left_bytes_fields & ~buf_block_t::LEFT_SIDE)) next_fold= ins_fold, next_rec= ins_rec; - ha_insert_for_fold(part, next_fold, block, next_rec); + part.insert(next_fold, next_rec, block); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } @@ -1796,23 +1989,29 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept if (locked) unlock_exit: - part.latch.wr_unlock(); + part.latch.rd_unlock(); } # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG __attribute__((nonnull)) /** @return whether a range of the cells is valid */ -static bool ha_validate(const hash_table_t *table, +static bool ha_validate(const btr_sea::hash_table *table, ulint start_index, ulint end_index) { ut_a(start_index <= end_index); - ut_a(end_index < table->n_cells); + ut_ad(end_index < table->pad(table->n_cells)); bool ok= true; + /* Note: We are also iterating over storage that is reserved for + btr_sea::hash_table::lock_get(). Because we are holding + exclusive btr_sea::partition::latch, those elements (which are not + pointers) must be bitwise zero. The latches can only be acquired + (and set nonzero) when btr_sea::partition::latch is being held + by another thread. */ for (ulint i= start_index; i <= end_index; i++) { - for (auto node= static_cast(table->array[i].node); node; + for (auto node= static_cast(table->array[i].first); node; node= node->next) { if (table->calc_hash(node->fold) != i) { @@ -1868,8 +2067,12 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) btr_sea::partition& part = btr_search.parts[hash_table_id]; - cell_count = part.table.n_cells; - + cell_count = part.table.pad(part.table.n_cells); + /* Note: We are also iterating over storage that is reserved for + part.table.lock_get(). Because we are holding exclusive part.latch, + those elements (which are not pointers) must be bitwise zero. + The page_hash_latch can only be acquired (and set nonzero) when + part.latch is being held by another thread. */ for (i = 0; i < cell_count; i++) { /* We release search latches every once in a while to give other queries a chance to run. */ @@ -1901,7 +2104,7 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) } } - node = static_cast(part.table.array[i].node); + node = static_cast(part.table.array[i].first); for (; node != NULL; node = node->next) { const buf_block_t* block diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index c21d0ff7844c5..b69f1beaad554 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -119,14 +119,107 @@ struct btr_sea @param resize whether buf_pool_t::resize() is the caller */ ATTRIBUTE_COLD void enable(bool resize= false) noexcept; + /** Hash cell chain in hash_table */ + struct hash_chain + { + /** pointer to the first block */ + ahi_node *first; + + /** Find an element. + @param u unary predicate + @return the first matching element + @retval nullptr if not found */ + template + inline ahi_node *find(UnaryPred u) const noexcept; + + /** Search for a pointer to an element. + @param u unary predicate + @return pointer to the first matching element, + or to the last element in the chain */ + template + inline ahi_node **search(UnaryPred u) noexcept; + }; + + /** Hash table with singly-linked overflow lists. + Based on @see buf_pool_t::page_hash_table */ + struct hash_table + { + static_assert(CPU_LEVEL1_DCACHE_LINESIZE >= 64, "less than 64 bytes"); + static_assert(!(CPU_LEVEL1_DCACHE_LINESIZE & 63), + "not a multiple of 64 bytes"); + + /** Number of array[] elements per page_hash_latch. + Must be one less than a power of 2. */ +#if 0 + static constexpr size_t ELEMENTS_PER_LATCH= 64 / sizeof(void*) - 1; + + /** Extra padding. FIXME: Is this ever useful to be nonzero? + Long time ago, some testing on an ARMv8 implementation seemed + to suggest so, but this has not been validated recently. */ + static constexpr size_t EMPTY_SLOTS_PER_LATCH= + ((CPU_LEVEL1_DCACHE_LINESIZE / 64) - 1) * (64 / sizeof(void*)); +#else + static constexpr size_t ELEMENTS_PER_LATCH= + CPU_LEVEL1_DCACHE_LINESIZE / sizeof(void*) - 1; + static constexpr size_t EMPTY_SLOTS_PER_LATCH= 0; +#endif + + /** number of payload elements in array[] */ + Atomic_relaxed n_cells; + /** the hash table, with pad(n_cells) elements, aligned to L1 cache size */ + hash_chain *array; + + /** Create the hash table. + @param n the lower bound of n_cells */ + inline void create(ulint n) noexcept; + + /** Free the hash table. */ + void free() noexcept { aligned_free(array); array= nullptr; } + + /** @return the index of an array element */ + ulint calc_hash(ulint fold) const noexcept + { return calc_hash(fold, n_cells); } + /** @return raw array index converted to padded index */ + static ulint pad(ulint h) noexcept + { + ulint latches= h / ELEMENTS_PER_LATCH; + ulint empty_slots= latches * EMPTY_SLOTS_PER_LATCH; + return 1 + latches + empty_slots + h; + } + private: + /** @return the index of an array element */ + static ulint calc_hash(ulint fold, ulint n_cells) noexcept + { + return pad(fold % n_cells); + } + public: + /** @return the latch covering a hash table chain */ + static page_hash_latch &lock_get(hash_chain &chain) noexcept + { + static_assert(!((ELEMENTS_PER_LATCH + 1) & ELEMENTS_PER_LATCH), + "must be one less than a power of 2"); + const size_t addr= reinterpret_cast(&chain); + ut_ad(addr & (ELEMENTS_PER_LATCH * sizeof chain)); + return *reinterpret_cast + (addr & ~(ELEMENTS_PER_LATCH * sizeof chain)); + } + + /** Get a hash table slot. */ + hash_chain &cell_get(ulint fold) const + { return array[calc_hash(fold, n_cells)]; } + }; + /** Partition of the hash table */ struct partition { - /** latch protecting table */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_spin_lock latch; + /** latch protecting table: either an exclusive latch, or + a shared latch combined with lock_get() */ + alignas(CPU_LEVEL1_DCACHE_LINESIZE) + IF_DBUG(srw_lock_debug,srw_spin_lock) latch; /** map of CRC-32C of rec prefix to rec_t* in buf_page_t::frame */ - hash_table_t table; - /** latch protecting blocks, spare; may be acquired while holding latch */ + hash_table table; + /** protects blocks; acquired while holding latch + and possibly table.lock_get() */ srw_mutex blocks_mutex; /** allocated blocks */ UT_LIST_BASE_NODE_T(buf_page_t) blocks; @@ -141,15 +234,50 @@ struct btr_sea inline void free() noexcept; + /** @return the number of allocated buffer pool blocks */ + TPOOL_SUPPRESS_TSAN size_t get_blocks() const noexcept + { return UT_LIST_GET_LEN(blocks) + !!spare; } + /** Ensure that there is a spare block for a future insert() */ void prepare_insert() noexcept; - /** Clean up after erasing an AHI node - @param erase node being erased + /** Undo prepare_insert() in case !btr_search.enabled */ + void rollback_insert() noexcept; + + private: + /** Start cleanup_after_erase() + @return the last allocated element */ + inline ahi_node *cleanup_after_erase_start() noexcept; + /** Finish cleanup_after_erase(). + We reduce the allocated size in UT_LIST_GET_LAST(blocks)->free_offset. + If that size reaches 0, the last block will be removed from blocks, + and a block may have to be freed by our caller. + @return buffer block to be freed + @retval nullptr if no buffer block was freed */ + buf_block_t *cleanup_after_erase_finish() noexcept; + public: + __attribute__((nonnull)) + /** Clean up after erasing an AHI node, while the caller is + holding an exclusive latch. Unless "erase" is the last allocated + element, we will swap it with the last allocated element. + Finally, we return via cleanup_after_erase_finish(). + @param erase node being erased @return buffer block to be freed @retval nullptr if no buffer block was freed */ buf_block_t *cleanup_after_erase(ahi_node *erase) noexcept; + __attribute__((nonnull)) + /** Clean up after erasing an AHI node. This is similar to + cleanup_after_erase(ahi_node*), except that the operation may fail. + @param erase node being erased + @param l the latch held together with shared latch + @return buffer block to be freed + @retval nullptr if no buffer block was freed + @retval -1 if we fail to shrink the allocation and erasing + needs to be retried while holding an exclusive latch */ + buf_block_t *cleanup_after_erase(ahi_node *erase, page_hash_latch *l) + noexcept; + __attribute__((nonnull)) # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG /** Insert or replace an entry into the hash table. @@ -164,11 +292,23 @@ struct btr_sea void insert(uint32_t fold, const rec_t *rec) noexcept; # endif - /** Delete a pointer to a record if it exists. - @param fold CRC-32C of rec prefix + /** erase() return value */ + enum erase_status{ + /** must retry with exclusive latch */ + ERASE_RETRY= -1, + /** the pointer to the record was erased */ + ERASED= 0, + /** nothing was erased */ + NOT_ERASED= 1 + }; + + /** Delete a pointer to a record if it exists, and release the latch. + @tparam ex true=holding exclusive latch, false=shared latch + @param cell hash table cell that may contain the CRC-32C of rec prefix @param rec B-tree leaf page record - @return whether a record existed and was removed */ - inline bool erase(uint32_t fold, const rec_t *rec) noexcept; + @return status */ + template + erase_status erase(hash_chain &cell, const rec_t *rec) noexcept; }; /** innodb_adaptive_hash_index_parts */ diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index cf4d93d2aa96a..3cef20f3b14f5 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -473,7 +473,8 @@ class buf_page_t protected by buf_pool.page_hash.lock_get() */ buf_page_t *hash; /** for state()==MEMORY that are part of recv_sys.pages and - protected by recv_sys.mutex */ + protected by recv_sys.mutex, or part of btr_sea::partition::table + and protected by btr_sea::partition::blocks_mutex */ struct { /** number of recv_sys.pages entries stored in the block */ uint16_t used_records; @@ -868,7 +869,7 @@ struct buf_block_t{ Atomic_relaxed n_hash_helps; # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG /** number of pointers from the btr_sea::partition::table; - !n_pointers == !index */ + !index implies n_pointers == 0 */ Atomic_counter n_pointers; # define assert_block_ahi_empty(block) ut_a(!(block)->n_pointers) # define assert_block_ahi_valid(b) ut_a((b)->index || !(b)->n_pointers) @@ -878,7 +879,7 @@ struct buf_block_t{ # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ /** index for which the adaptive hash index has been created, or nullptr if the page does not exist in the index. - Protected by btr_sea::partition::latch. */ + May be modified while holding exclusive btr_sea::partition::latch. */ Atomic_relaxed index; /* @} */ #else /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/include/buf0types.h b/storage/innobase/include/buf0types.h index 0f00d416ae3e5..cf5ab38df5f9f 100644 --- a/storage/innobase/include/buf0types.h +++ b/storage/innobase/include/buf0types.h @@ -192,6 +192,8 @@ class page_hash_latch : private rw_lock inline void lock_shared() noexcept; /** Acquire an exclusive lock */ inline void lock() noexcept; + /** @return whether an exclusive lock was acquired without waiting */ + bool try_lock() noexcept { return write_trylock(); } /** @return whether an exclusive lock is being held by any thread */ bool is_write_locked() const noexcept { return rw_lock::is_write_locked(); } @@ -215,6 +217,7 @@ class page_hash_latch void lock_shared() noexcept { lk.rd_lock(); } void unlock_shared() noexcept { lk.rd_unlock(); } void lock() noexcept { lk.wr_lock(); } + bool try_lock() noexcept { return lk.wr_lock_try(); } void unlock() noexcept { lk.wr_unlock(); } bool is_write_locked() const noexcept { return lk.is_write_locked(); } bool is_locked() const noexcept { return lk.is_locked(); } @@ -229,6 +232,7 @@ class page_hash_latch void lock_shared() noexcept { lock(); } void unlock_shared() noexcept { unlock(); } void lock() noexcept { lk.wr_lock(); } + bool try_lock() noexcept { return lk.wr_lock_try(); } void unlock() noexcept { lk.wr_unlock(); } bool is_locked() const noexcept { return lk.is_locked(); } bool is_write_locked() const noexcept { return is_locked(); } diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h index 79a3e7c42a7b0..1181d646c0798 100644 --- a/storage/innobase/include/srw_lock.h +++ b/storage/innobase/include/srw_lock.h @@ -34,6 +34,16 @@ this program; if not, write to the Free Software Foundation, Inc., # define SUX_LOCK_GENERIC /* Use dummy implementation for debugging purposes */ #endif +#ifndef UNIV_PFS_RWLOCK +# define SRW_LOCK_INIT(key) init() +# define SRW_LOCK_ARGS(file, line) /* nothing */ +# define SRW_LOCK_CALL /* nothing */ +#else +# define SRW_LOCK_INIT(key) init(key) +# define SRW_LOCK_ARGS(file, line) file, line +# define SRW_LOCK_CALL __FILE__, __LINE__ +#endif + /** An exclusive-only variant of srw_lock */ template class pthread_mutex_wrapper final @@ -310,6 +320,7 @@ class ssux_lock_impl ut_ad(lk < WRITER); u_unlock(); } + void wr_rd_downgrade() noexcept { wr_u_downgrade(); u_rd_downgrade(); } void rd_unlock() noexcept { @@ -415,16 +426,9 @@ typedef ssux_lock_impl srw_spin_lock_low; #endif #ifndef UNIV_PFS_RWLOCK -# define SRW_LOCK_INIT(key) init() -# define SRW_LOCK_ARGS(file, line) /* nothing */ -# define SRW_LOCK_CALL /* nothing */ typedef srw_lock_low srw_lock; typedef srw_spin_lock_low srw_spin_lock; #else -# define SRW_LOCK_INIT(key) init(key) -# define SRW_LOCK_ARGS(file, line) file, line -# define SRW_LOCK_CALL __FILE__, __LINE__ - /** Slim shared-update-exclusive lock with PERFORMANCE_SCHEMA instrumentation */ class ssux_lock { @@ -556,6 +560,23 @@ class srw_lock_impl PSI_RWLOCK_CALL(unlock_rwlock)(pfs_psi); lock.wr_unlock(); } +# if defined _WIN32 || defined SUX_LOCK_GENERIC +# else + void wr_rd_downgrade(const char *file, unsigned line) noexcept + { + if (psi_likely(pfs_psi != nullptr)) + { + PSI_RWLOCK_CALL(unlock_rwlock)(pfs_psi); + PSI_rwlock_locker_state state; + if (PSI_rwlock_locker *locker= + PSI_RWLOCK_CALL(start_rwlock_rdwait) + (&state, pfs_psi, PSI_RWLOCK_READLOCK, file, line)) + PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0); + } + + lock.wr_rd_downgrade(); + } +#endif bool rd_lock_try() noexcept { return lock.rd_lock_try(); } bool wr_lock_try() noexcept { return lock.wr_lock_try(); } void lock_shared() noexcept { return rd_lock(SRW_LOCK_CALL); } @@ -595,13 +616,13 @@ class srw_lock_debug : private srw_lock void SRW_LOCK_INIT(mysql_pfs_key_t key) noexcept; void destroy() noexcept; -#ifndef SUX_LOCK_GENERIC +# ifndef SUX_LOCK_GENERIC /** @return whether any lock may be held by any thread */ bool is_locked_or_waiting() const noexcept { return srw_lock::is_locked_or_waiting(); } /** @return whether an exclusive lock may be held by any thread */ bool is_write_locked() const noexcept { return srw_lock::is_write_locked(); } -#endif +# endif /** Acquire an exclusive lock */ void wr_lock(SRW_LOCK_ARGS(const char *file, unsigned line)) noexcept; @@ -609,6 +630,11 @@ class srw_lock_debug : private srw_lock bool wr_lock_try() noexcept; /** Release after wr_lock() */ void wr_unlock() noexcept; +# if defined _WIN32 || defined SUX_LOCK_GENERIC +# else + /** Downgrade wr_lock() to rd_lock() */ + void wr_rd_downgrade(SRW_LOCK_ARGS(const char*,unsigned)) noexcept; +# endif /** Acquire a shared lock */ void rd_lock(SRW_LOCK_ARGS(const char *file, unsigned line)) noexcept; /** @return whether a shared lock was acquired */ diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index dabc7f4b492e1..6009d7e7c5a46 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -748,7 +748,8 @@ srv_printf_innodb_monitor( part.blocks_mutex.wr_lock(); fprintf(file, "Hash table size " ULINTPF ", node heap has " ULINTPF " buffer(s)\n", - part.table.n_cells, part.blocks.count + !!part.spare); + size_t{part.table.n_cells}, + part.blocks.count + !!part.spare); part.blocks_mutex.wr_unlock(); } @@ -830,10 +831,7 @@ srv_export_innodb_status(void) ulint mem_adaptive_hash = 0; for (ulong i = 0; i < btr_search.n_parts; i++) { - btr_sea::partition& part= btr_search.parts[i]; - part.blocks_mutex.wr_lock(); - mem_adaptive_hash += part.blocks.count + !!part.spare; - part.blocks_mutex.wr_unlock(); + mem_adaptive_hash += btr_search.parts[i].get_blocks(); } mem_adaptive_hash <<= srv_page_size_shift; btr_search.parts[0].latch.rd_lock(SRW_LOCK_CALL); diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc index 28b586de5df92..4e41f87a7030d 100644 --- a/storage/innobase/sync/srw_lock.cc +++ b/storage/innobase/sync/srw_lock.cc @@ -669,6 +669,18 @@ void srw_lock_debug::wr_unlock() noexcept srw_lock::wr_unlock(); } +# if defined _WIN32 || defined SUX_LOCK_GENERIC +# else +void srw_lock_debug::wr_rd_downgrade +(SRW_LOCK_ARGS(const char *file, unsigned line)) noexcept +{ + ut_ad(have_wr()); + writer.store(0, std::memory_order_relaxed); + readers_register(); + srw_lock::wr_rd_downgrade(SRW_LOCK_ARGS(file, line)); +} +# endif + void srw_lock_debug::readers_register() noexcept { readers_lock.wr_lock(); diff --git a/storage/innobase/unittest/innodb_ahi-t.cc b/storage/innobase/unittest/innodb_ahi-t.cc index 7743c849f705c..cb433099fd41e 100644 --- a/storage/innobase/unittest/innodb_ahi-t.cc +++ b/storage/innobase/unittest/innodb_ahi-t.cc @@ -1,7 +1,11 @@ #include "tap.h" -#define SUX_LOCK_GENERIC -#define NO_ELISION +#ifndef SUX_LOCK_GENERIC +# define SUX_LOCK_GENERIC +#endif +#ifndef NO_ELISION +# define NO_ELISION +#endif #define thd_kill_level(thd) 0 #define srv0mon_h #define MONITOR_INC(x) @@ -47,9 +51,15 @@ std::ostream &operator<<(std::ostream &out, const page_id_t) { return out; } #ifdef UNIV_DEBUG byte data_error; +void srw_lock_debug::SRW_LOCK_INIT(mysql_pfs_key_t) noexcept {} +void srw_lock_debug::destroy() noexcept {} bool srw_lock_debug::have_wr() const noexcept { return false; } +bool srw_lock_debug::have_rd() const noexcept { return false; } +bool srw_lock_debug::have_any() const noexcept { return false; } void srw_lock_debug::rd_unlock() noexcept {} void srw_lock_debug::rd_lock(SRW_LOCK_ARGS(const char*,unsigned)) noexcept {} +void srw_lock_debug::wr_lock(SRW_LOCK_ARGS(const char*,unsigned)) noexcept {} +void srw_lock_debug::wr_unlock() noexcept {} #endif void page_hash_latch::read_lock_wait() noexcept {} @@ -60,6 +70,7 @@ template<> void srw_lock_::rd_wait() noexcept {} template<> void srw_lock_::wr_wait() noexcept {} template void ssux_lock_impl::wake() noexcept {} template void srw_mutex_impl::wake() noexcept {} +void page_hash_latch::write_lock_wait() noexcept {} #ifdef UNIV_PFS_MEMORY PSI_memory_key ut_new_get_key_by_file(uint32_t){ return PSI_NOT_INSTRUMENTED; } From 3cc9ac0b3074b75b735843f0ff980cbeb07eedb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Sep 2025 10:15:09 +0300 Subject: [PATCH 5/5] MDEV-37482: Introduce innodb_adaptive_hash_index_cells SET GLOBAL innodb_adaptive_hash_index_cells may be executed while the server is running. This parameter will be effectively multiplied by innodb_adaptive_hash_index_parts, because each partition will contain its own hash table. Previously, the number of hash table cells in the InnoDB adaptive hash index depended on the initial innodb_buffer_pool_size and was insufficient for some workloads, leading to excessively long hash bucket chains. If innodb_adaptive_hash_index_cells is at its minimum and default value 16381 at startup, it will be derived from the innodb_buffer_pool_size, for backward compatibility. --- .../suite/innodb/r/insert_into_empty.result | 3 + .../suite/innodb/t/insert_into_empty.test | 3 + .../sys_vars/r/sysvars_innodb,32bit.rdiff | 79 +++++++++++-------- .../suite/sys_vars/r/sysvars_innodb.result | 12 +++ storage/innobase/btr/btr0sea.cc | 61 ++++++++++---- storage/innobase/handler/ha_innodb.cc | 65 ++++++++++----- storage/innobase/include/btr0sea.h | 26 ++++-- 7 files changed, 174 insertions(+), 75 deletions(-) diff --git a/mysql-test/suite/innodb/r/insert_into_empty.result b/mysql-test/suite/innodb/r/insert_into_empty.result index df831c74758e0..24f2751cea5f1 100644 --- a/mysql-test/suite/innodb/r/insert_into_empty.result +++ b/mysql-test/suite/innodb/r/insert_into_empty.result @@ -19,6 +19,8 @@ DROP TEMPORARY TABLE t; # SET @save_ahi = @@global.innodb_adaptive_hash_index; SET GLOBAL innodb_adaptive_hash_index = 1; +SET @save_ahi_cells = @@global.innodb_adaptive_hash_index_cells; +SET GLOBAL innodb_adaptive_hash_index_cells=1048576; CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; SET @old_bulk_op= (SELECT variable_value FROM information_schema.global_status @@ -43,6 +45,7 @@ INSERT INTO t1 SELECT * FROM seq_1_to_500; ROLLBACK; DROP TABLE t1; SET GLOBAL innodb_adaptive_hash_index = @save_ahi; +SET GLOBAL innodb_adaptive_hash_index_cells = @save_ahi_cells; # # MDEV-24951 Assertion m.first->second.valid(trx->undo_no) failed # in trx_undo_report_row_operation diff --git a/mysql-test/suite/innodb/t/insert_into_empty.test b/mysql-test/suite/innodb/t/insert_into_empty.test index 38349146e6b47..557161fdd9cad 100644 --- a/mysql-test/suite/innodb/t/insert_into_empty.test +++ b/mysql-test/suite/innodb/t/insert_into_empty.test @@ -25,6 +25,8 @@ DROP TEMPORARY TABLE t; --echo # SET @save_ahi = @@global.innodb_adaptive_hash_index; SET GLOBAL innodb_adaptive_hash_index = 1; +SET @save_ahi_cells = @@global.innodb_adaptive_hash_index_cells; +SET GLOBAL innodb_adaptive_hash_index_cells=1048576; CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; SET @old_bulk_op= @@ -49,6 +51,7 @@ INSERT INTO t1 SELECT * FROM seq_1_to_500; ROLLBACK; DROP TABLE t1; SET GLOBAL innodb_adaptive_hash_index = @save_ahi; +SET GLOBAL innodb_adaptive_hash_index_cells = @save_ahi_cells; --echo # --echo # MDEV-24951 Assertion m.first->second.valid(trx->undo_no) failed diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff index 82a595ee00345..a2b062a586036 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff @@ -1,6 +1,15 @@ --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result -@@ -47,7 +47,7 @@ +@@ -51,7 +51,7 @@ + VARIABLE_TYPE INT UNSIGNED + VARIABLE_COMMENT Number of adaptive hash table cells in each partition; 16381 at start defaults to being derived from innodb_buffer_pool_size + NUMERIC_MIN_VALUE 16381 +-NUMERIC_MAX_VALUE 4294967295 ++NUMERIC_MAX_VALUE 1073741823 + NUMERIC_BLOCK_SIZE 0 + ENUM_VALUE_LIST NULL + READ_ONLY NO +@@ -60,7 +60,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 8 VARIABLE_SCOPE GLOBAL @@ -9,7 +18,7 @@ VARIABLE_COMMENT Number of InnoDB Adaptive Hash Index Partitions (default 8) NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 512 -@@ -83,7 +83,7 @@ +@@ -96,7 +96,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -18,7 +27,7 @@ VARIABLE_COMMENT The AUTOINC lock modes supported by InnoDB: 0 => Old style AUTOINC locking (for backward compatibility); 1 => New style AUTOINC locking; 2 => No AUTOINC locking (unsafe for SBR) NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -95,10 +95,10 @@ +@@ -108,10 +108,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -31,7 +40,7 @@ NUMERIC_BLOCK_SIZE 1048576 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -131,7 +131,7 @@ +@@ -144,7 +144,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 25 VARIABLE_SCOPE GLOBAL @@ -40,7 +49,7 @@ VARIABLE_COMMENT Dump only the hottest N% of each buffer pool, defaults to 25 NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 100 -@@ -203,10 +203,10 @@ +@@ -216,10 +216,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 134217728 VARIABLE_SCOPE GLOBAL @@ -53,7 +62,7 @@ NUMERIC_BLOCK_SIZE 1048576 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -215,11 +215,11 @@ +@@ -228,11 +228,11 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -68,7 +77,7 @@ ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED -@@ -227,11 +227,11 @@ +@@ -240,11 +240,11 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -83,7 +92,7 @@ ENUM_VALUE_LIST NULL READ_ONLY YES COMMAND_LINE_ARGUMENT REQUIRED -@@ -239,7 +239,7 @@ +@@ -252,7 +252,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -92,7 +101,7 @@ VARIABLE_COMMENT A number that tells how often buffer pool dump status in percentages should be printed. E.g. 10 means that buffer pool dump status is printed when every 10% of number of buffer pool pages are dumped. Default is 0 (only start and end status is printed) NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -311,7 +311,7 @@ +@@ -324,7 +324,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 5 VARIABLE_SCOPE GLOBAL @@ -101,7 +110,7 @@ VARIABLE_COMMENT If the compression failure rate of a table is greater than this number more padding is added to the pages to reduce the failures. A value of zero implies no padding NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -335,7 +335,7 @@ +@@ -348,7 +348,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 50 VARIABLE_SCOPE GLOBAL @@ -110,7 +119,7 @@ VARIABLE_COMMENT Percentage of empty space on a data page that can be reserved to make the page compressible NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 75 -@@ -575,7 +575,7 @@ +@@ -588,7 +588,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 600 VARIABLE_SCOPE GLOBAL @@ -119,7 +128,7 @@ VARIABLE_COMMENT Maximum number of seconds that semaphore times out in InnoDB NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 4294967295 -@@ -623,7 +623,7 @@ +@@ -636,7 +636,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 30 VARIABLE_SCOPE GLOBAL @@ -128,7 +137,7 @@ VARIABLE_COMMENT Number of iterations over which the background flushing is averaged NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1000 -@@ -647,7 +647,7 @@ +@@ -660,7 +660,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -137,7 +146,7 @@ VARIABLE_COMMENT Controls the durability/speed trade-off for commits. Set to 0 (write and flush redo log to disk only once per second), 1 (flush to disk at each commit), 2 (write to log at commit but flush to disk only once per second) or 3 (flush to disk at prepare and at commit, slower and usually redundant). 1 and 3 guarantees that after a crash, committed transactions will not be lost and will be consistent with the binlog and other transactional engines. 2 can get inconsistent and lose transactions if there is a power failure or kernel crash but not if mysqld crashes. 0 has no guarantees in case of crash. 0 and 2 can be faster than 1 or 3 NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 3 -@@ -671,7 +671,7 @@ +@@ -684,7 +684,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -146,7 +155,7 @@ VARIABLE_COMMENT Set to 0 (don't flush neighbors from buffer pool), 1 (flush contiguous neighbors from buffer pool) or 2 (flush neighbors from buffer pool), when flushing a block NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -707,7 +707,7 @@ +@@ -720,7 +720,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -155,7 +164,7 @@ VARIABLE_COMMENT Helps to save your data in case the disk image of the database becomes corrupt. Value 5 can return bogus data, and 6 can permanently corrupt data NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 6 -@@ -731,10 +731,10 @@ +@@ -744,10 +744,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 8000000 VARIABLE_SCOPE GLOBAL @@ -168,7 +177,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -767,7 +767,7 @@ +@@ -780,7 +780,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 84 VARIABLE_SCOPE GLOBAL @@ -177,7 +186,7 @@ VARIABLE_COMMENT InnoDB Fulltext search maximum token size in characters NUMERIC_MIN_VALUE 10 NUMERIC_MAX_VALUE 84 -@@ -779,7 +779,7 @@ +@@ -792,7 +792,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 3 VARIABLE_SCOPE GLOBAL @@ -186,7 +195,7 @@ VARIABLE_COMMENT InnoDB Fulltext search minimum token size in characters NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16 -@@ -791,7 +791,7 @@ +@@ -804,7 +804,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 2000 VARIABLE_SCOPE GLOBAL @@ -195,7 +204,7 @@ VARIABLE_COMMENT InnoDB Fulltext search number of words to optimize for each optimize table call NUMERIC_MIN_VALUE 1000 NUMERIC_MAX_VALUE 10000 -@@ -803,10 +803,10 @@ +@@ -816,10 +816,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 2000000000 VARIABLE_SCOPE GLOBAL @@ -208,7 +217,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -827,7 +827,7 @@ +@@ -840,7 +840,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 2 VARIABLE_SCOPE GLOBAL @@ -217,7 +226,7 @@ VARIABLE_COMMENT InnoDB Fulltext search parallel sort degree, will round up to nearest power of 2 number NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 16 -@@ -839,10 +839,10 @@ +@@ -852,10 +852,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 640000000 VARIABLE_SCOPE GLOBAL @@ -230,7 +239,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -887,7 +887,7 @@ +@@ -900,7 +900,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 200 VARIABLE_SCOPE GLOBAL @@ -239,7 +248,7 @@ VARIABLE_COMMENT Number of IOPs the server can do. Tunes the background IO rate NUMERIC_MIN_VALUE 100 NUMERIC_MAX_VALUE 4294967295 -@@ -899,7 +899,7 @@ +@@ -912,7 +912,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 4294967295 VARIABLE_SCOPE GLOBAL @@ -248,7 +257,7 @@ VARIABLE_COMMENT Limit to which innodb_io_capacity can be inflated NUMERIC_MIN_VALUE 100 NUMERIC_MAX_VALUE 4294967295 -@@ -1031,10 +1031,10 @@ +@@ -1044,10 +1044,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 32 VARIABLE_SCOPE GLOBAL @@ -261,7 +270,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1043,10 +1043,10 @@ +@@ -1056,10 +1056,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 1536 VARIABLE_SCOPE GLOBAL @@ -274,7 +283,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1079,10 +1079,10 @@ +@@ -1092,10 +1092,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -287,7 +296,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1091,7 +1091,7 @@ +@@ -1104,7 +1104,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -296,7 +305,7 @@ VARIABLE_COMMENT Maximum delay of user threads in micro-seconds NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 10000000 -@@ -1223,10 +1223,10 @@ +@@ -1236,10 +1236,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -309,7 +318,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -1247,7 +1247,7 @@ +@@ -1260,7 +1260,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 16384 VARIABLE_SCOPE GLOBAL @@ -318,7 +327,7 @@ VARIABLE_COMMENT Page size to use for all InnoDB tablespaces NUMERIC_MIN_VALUE 4096 NUMERIC_MAX_VALUE 65536 -@@ -1283,7 +1283,7 @@ +@@ -1296,7 +1296,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 127 VARIABLE_SCOPE GLOBAL @@ -327,7 +336,7 @@ VARIABLE_COMMENT Number of UNDO log pages to purge in one batch from the history list NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 5000 -@@ -1295,7 +1295,7 @@ +@@ -1308,7 +1308,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -336,7 +345,7 @@ VARIABLE_COMMENT Unused NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -1331,7 +1331,7 @@ +@@ -1344,7 +1344,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 56 VARIABLE_SCOPE GLOBAL @@ -345,7 +354,7 @@ VARIABLE_COMMENT Number of pages that must be accessed sequentially for InnoDB to trigger a readahead NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 64 -@@ -1415,7 +1415,7 @@ +@@ -1428,7 +1428,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 1048576 VARIABLE_SCOPE GLOBAL @@ -354,7 +363,7 @@ VARIABLE_COMMENT Memory buffer size for index creation NUMERIC_MIN_VALUE 65536 NUMERIC_MAX_VALUE 67108864 -@@ -1583,10 +1583,10 @@ +@@ -1596,10 +1596,10 @@ SESSION_VALUE NULL DEFAULT_VALUE 30 VARIABLE_SCOPE GLOBAL diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 2ff85735db99a..f9becde110983 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -44,6 +44,18 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME INNODB_ADAPTIVE_HASH_INDEX_CELLS +SESSION_VALUE NULL +DEFAULT_VALUE 16381 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE INT UNSIGNED +VARIABLE_COMMENT Number of adaptive hash table cells in each partition; 16381 at start defaults to being derived from innodb_buffer_pool_size +NUMERIC_MIN_VALUE 16381 +NUMERIC_MAX_VALUE 4294967295 +NUMERIC_BLOCK_SIZE 0 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_ADAPTIVE_HASH_INDEX_PARTS SESSION_VALUE NULL DEFAULT_VALUE 8 diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 4d6cc92154b06..7d0921112fb53 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -121,19 +121,21 @@ inline void btr_sea::partition::free() noexcept blocks_mutex.destroy(); } -inline void btr_sea::hash_table::create(ulint n) noexcept +inline bool btr_sea::hash_table::create(ulint n) noexcept { - n_cells= ut_find_prime(n); + n_cells= n; const size_t size= MY_ALIGN(pad(n_cells) * sizeof *array, CPU_LEVEL1_DCACHE_LINESIZE); void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); - memset_aligned(v, 0, size); + if (v) + memset_aligned(v, 0, size); array= static_cast(v); + return array != nullptr; } -inline void btr_sea::partition::alloc(ulint hash_size) noexcept +inline bool btr_sea::partition::alloc(ulint hash_size) noexcept { - table.create(hash_size); + return table.create(hash_size); } void btr_sea::create() noexcept @@ -142,11 +144,16 @@ void btr_sea::create() noexcept part.init(); } -void btr_sea::alloc(ulint hash_size) noexcept +bool btr_sea::alloc(size_t hash_size) noexcept { - hash_size/= n_parts; for (ulong i= 0; i < n_parts; ++i) - parts[i].alloc(hash_size); + if (!parts[i].alloc(hash_size)) + { + while (i--) + parts[i].clear(); + return false; + } + return true; } inline void btr_sea::clear() noexcept @@ -381,7 +388,7 @@ ATTRIBUTE_COLD void btr_search_lazy_free(dict_index_t *index) noexcept table->autoinc_mutex.wr_unlock(); } -ATTRIBUTE_COLD bool btr_sea::disable() noexcept +ATTRIBUTE_COLD bool btr_sea::disable_and_lock() noexcept { dict_sys.freeze(SRW_LOCK_CALL); @@ -404,9 +411,19 @@ ATTRIBUTE_COLD bool btr_sea::disable() noexcept else dict_sys.unfreeze(); + return was_enabled; +} + +ATTRIBUTE_COLD void btr_sea::unlock() noexcept +{ for (ulong i= 0; i < n_parts; i++) parts[i].latch.wr_unlock(); +} +ATTRIBUTE_COLD bool btr_sea::disable() noexcept +{ + const bool was_enabled{disable_and_lock()}; + unlock(); return was_enabled; } @@ -427,15 +444,27 @@ ATTRIBUTE_COLD void btr_sea::enable(bool resize) noexcept parts[i].latch.wr_lock(SRW_LOCK_CALL); if (!parts[0].table.array) - { - enabled= true; - alloc(buf_pool.curr_pool_size() / sizeof(void *) / 64); - } + enabled= alloc(n_cells); + else + ut_ad(enabled); - ut_ad(enabled); + unlock(); +} - for (ulong i= 0; i < n_parts; i++) - parts[i].latch.wr_unlock(); +ATTRIBUTE_COLD void btr_sea::resize(uint n_cells) noexcept +{ + const bool was_enabled{disable_and_lock()}; + + clear(); + ut_ad(!parts[0].table.array); + + if (was_enabled) + enabled= alloc(n_cells); + + if (!was_enabled || enabled) + this->n_cells= n_cells; + + unlock(); } #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index fe4d357b12c85..0b7e5adf3d3b1 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3706,6 +3706,38 @@ static MYSQL_SYSVAR_UINT(log_write_ahead_size, log_sys.write_size, "Redo log write size to avoid read-on-write; must be a power of two", nullptr, nullptr, 512, 512, 4096, 1); + +#ifdef BTR_CUR_HASH_ADAPT +static void innodb_adaptive_hash_index_update(THD*, st_mysql_sys_var*, void*, + const void *save) noexcept +{ + /* Prevent a possible deadlock with innobase_fts_load_stopword() */ + mysql_mutex_unlock(&LOCK_global_system_variables); + if (*static_cast(save)) + btr_search.enable(); + else + btr_search.disable(); + mysql_mutex_lock(&LOCK_global_system_variables); +} + +static void innodb_adaptive_hash_index_cells_update(THD*, st_mysql_sys_var*, + void*, const void *save) + noexcept +{ + /* Prevent a possible deadlock with innobase_fts_load_stopword() */ + mysql_mutex_unlock(&LOCK_global_system_variables); + btr_search.resize(*static_cast(save)); + mysql_mutex_lock(&LOCK_global_system_variables); +} + +static MYSQL_SYSVAR_UINT(adaptive_hash_index_cells, btr_search.n_cells, + PLUGIN_VAR_RQCMDARG, + "Number of adaptive hash table cells in each partition;" + " 16381 at start defaults to being derived from innodb_buffer_pool_size", + nullptr, innodb_adaptive_hash_index_cells_update, 16381, 16381, + sizeof(size_t) < 8 ? (1U << 30) - 1 : UINT_MAX, 0); +#endif /* BTR_CUR_HASH_ADAPT */ + /****************************************************************//** Gives the file extension of an InnoDB single-table tablespace. */ static const char* ha_innobase_exts[] = { @@ -3822,6 +3854,19 @@ static int innodb_init_params() DBUG_RETURN(HA_ERR_INITIALIZATION); } + if (btr_search.n_cells <= + MYSQL_SYSVAR_NAME(adaptive_hash_index_cells).min_val) + { + size_t n{innodb_buffer_pool_size / 512 / btr_search.n_parts}; + if (4 < sizeof n) + n= std::min(size_t{UINT_MAX}, ut_find_prime(n)); + else + n= std::min(size_t{(1U << 30) - 1}, + ut_find_prime(std::min(size_t{(1U << 30) - 1}, n))); + btr_search.n_cells= + std::max(MYSQL_SYSVAR_NAME(adaptive_hash_index_cells).min_val, uint(n)); + } + if (compression_algorithm_is_not_loaded(innodb_compression_algorithm, ME_ERROR_LOG)) DBUG_RETURN(HA_ERR_INITIALIZATION); @@ -17634,25 +17679,6 @@ static int innodb_ft_aux_table_validate(THD *thd, st_mysql_sys_var*, } } -#ifdef BTR_CUR_HASH_ADAPT -/****************************************************************//** -Update the system variable innodb_adaptive_hash_index using the "saved" -value. This function is registered as a callback with MySQL. */ -static -void -innodb_adaptive_hash_index_update(THD*, st_mysql_sys_var*, void*, - const void* save) -{ - mysql_mutex_unlock(&LOCK_global_system_variables); - if (*(my_bool*) save) { - btr_search.enable(); - } else { - btr_search.disable(); - } - mysql_mutex_lock(&LOCK_global_system_variables); -} -#endif /* BTR_CUR_HASH_ADAPT */ - /****************************************************************//** Update the system variable innodb_cmp_per_index using the "saved" value. This function is registered as a callback with MySQL. */ @@ -19997,6 +20023,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(stats_traditional), #ifdef BTR_CUR_HASH_ADAPT MYSQL_SYSVAR(adaptive_hash_index), + MYSQL_SYSVAR(adaptive_hash_index_cells), MYSQL_SYSVAR(adaptive_hash_index_parts), #endif /* BTR_CUR_HASH_ADAPT */ MYSQL_SYSVAR(stats_method), diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index b69f1beaad554..685d9a4feb2b1 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -111,6 +111,14 @@ struct btr_sea while a thread is holding a partition::latch, then also this must hold. */ Atomic_relaxed enabled; +private: + /** Disable the adaptive hash search system and empty the index. + @return whether the adaptive hash index was enabled */ + ATTRIBUTE_COLD bool disable_and_lock() noexcept; + + /** Unlock the adaptive hash search system. */ + ATTRIBUTE_COLD void unlock() noexcept; +public: /** Disable the adaptive hash search system and empty the index. @return whether the adaptive hash index was enabled */ ATTRIBUTE_COLD bool disable() noexcept; @@ -165,13 +173,14 @@ struct btr_sea #endif /** number of payload elements in array[] */ - Atomic_relaxed n_cells; + Atomic_relaxed n_cells; /** the hash table, with pad(n_cells) elements, aligned to L1 cache size */ hash_chain *array; /** Create the hash table. - @param n the lower bound of n_cells */ - inline void create(ulint n) noexcept; + @param n the lower bound of n_cells + @return whether the creation succeeded */ + inline bool create(ulint n) noexcept; /** Free the hash table. */ void free() noexcept { aligned_free(array); array= nullptr; } @@ -228,7 +237,8 @@ struct btr_sea inline void init() noexcept; - inline void alloc(ulint hash_size) noexcept; + /** @return whether the allocation succeeded */ + inline bool alloc(size_t hash_size) noexcept; inline void clear() noexcept; @@ -311,6 +321,8 @@ struct btr_sea erase_status erase(hash_chain &cell, const rec_t *rec) noexcept; }; + /** number of hash table entries, to be multiplied by n_parts */ + uint n_cells; /** innodb_adaptive_hash_index_parts */ ulong n_parts; /** Partitions of the adaptive hash index */ @@ -326,7 +338,11 @@ struct btr_sea /** Create and initialize at startup */ void create() noexcept; - void alloc(ulint hash_size) noexcept; + /** @return whether the allocation succeeded */ + bool alloc(size_t hash_size) noexcept; + + /** Change the number of cells */ + void resize(uint n_cells) noexcept; /** Clear when disabling the adaptive hash index */ inline void clear() noexcept;