From 2ac7f6cca6655450790f353fd98a0aa837346409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:00:59 +0200 Subject: [PATCH 01/10] MDEV-35049: btr_search_check_free_space_in_heap() is a bottleneck Let us use implement a simple fixed-size allocator for the adaptive hash index, insted of complicating mem_heap_t or mem_block_info_t. MEM_HEAP_BTR_SEARCH: Remove. mem_block_info_t::free_block(), mem_heap_free_block_free(): Remove. mem_heap_free_top(), mem_heap_get_top(): Remove. btr_sea::partition::spare: Replaces mem_block_info_t::free_block. This keeps one spare block per adaptive hash index partition, to process an insert. We must not wait for buf_pool.mutex while holding any btr_sea::partition::latch. That is why we cache one block for future allocations. This is protected by a new btr_sea::partition::blocks_mutex in order to relieve pressure on btr_sea::partition::latch. btr_sea::partition::prepare_insert(): Replaces btr_search_check_free_space_in_heap(). btr_sea::partition::erase(): Replaces ha_search_and_delete_if_found(). btr_sea::partition::cleanup_after_erase(): Replaces the most part of ha_delete_hash_node(). Unlike the previous implementation, we will retain a spare block for prepare_insert(). This should reduce some contention on buf_pool.mutex. btr_search.n_parts: Replaces btr_ahi_parts. btr_search.enabled: Replaces btr_search_enabled. This must hold whenever buf_block_t::index is set while a thread is holding a btr_sea::partition::latch. dict_index_t::search_info: Remove pointer indirection, and use Atomic_relaxed or Atomic_counter for most fields. btr_search_guess_on_hash(): Let the caller ensure that latch_mode is BTR_MODIFY_LEAF or BTR_SEARCH_LEAF. Release btr_sea::partition::latch before buffer-fixing the block. The page latch that we already acquired is preventing buffer pool eviction. We must validate both block->index and block->page.state while holding part.latch in order to avoid race conditions with buffer page relocation or buf_pool_t::resize(). btr_search_check_guess(): Remove the constant parameter can_only_compare_to_cursor_rec=false. ahi_node: Replaces ha_node_t. This has been tested by running the regression test suite with the adaptive hash index enabled: ./mtr --mysqld=--loose-innodb-adaptive-hash-index=ON Reviewed by: Vladislav Lesin --- extra/mariabackup/xtrabackup.cc | 6 +- storage/innobase/CMakeLists.txt | 3 - storage/innobase/btr/btr0btr.cc | 14 +- storage/innobase/btr/btr0cur.cc | 58 +- storage/innobase/btr/btr0sea.cc | 1203 +++++++++++---------- storage/innobase/buf/buf0buf.cc | 8 +- storage/innobase/buf/buf0lru.cc | 11 +- storage/innobase/dict/dict0crea.cc | 2 +- storage/innobase/dict/dict0dict.cc | 15 +- storage/innobase/gis/gis0sea.cc | 5 +- storage/innobase/handler/ha_innodb.cc | 13 +- storage/innobase/handler/handler0alter.cc | 6 +- storage/innobase/ibuf/ibuf0ibuf.cc | 3 - storage/innobase/include/btr0cur.h | 7 +- storage/innobase/include/btr0sea.h | 396 ++----- storage/innobase/include/btr0sea.inl | 117 -- storage/innobase/include/btr0types.h | 11 - storage/innobase/include/buf0buf.h | 6 +- storage/innobase/include/dict0mem.h | 66 +- storage/innobase/include/ha0ha.h | 60 - storage/innobase/include/ha0ha.inl | 154 --- storage/innobase/include/mem0mem.h | 50 +- storage/innobase/include/mem0mem.inl | 93 +- storage/innobase/mem/mem0mem.cc | 52 +- storage/innobase/mtr/mtr0mtr.cc | 3 +- storage/innobase/row/row0import.cc | 7 - storage/innobase/row/row0ins.cc | 11 +- storage/innobase/row/row0merge.cc | 2 +- storage/innobase/row/row0mysql.cc | 2 +- storage/innobase/row/row0sel.cc | 2 +- storage/innobase/srv/srv0srv.cc | 32 +- storage/innobase/srv/srv0start.cc | 2 +- 32 files changed, 892 insertions(+), 1528 deletions(-) delete mode 100644 storage/innobase/include/btr0sea.inl delete mode 100644 storage/innobase/include/ha0ha.h delete mode 100644 storage/innobase/include/ha0ha.inl diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index c1d7085237f3c..a22f80b639c97 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1635,8 +1635,8 @@ struct my_option xb_server_options[] = #ifdef BTR_CUR_HASH_ADAPT {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX, "Enable InnoDB adaptive hash index (disabled by default).", - &btr_search_enabled, - &btr_search_enabled, + &btr_search.enabled, + &btr_search.enabled, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, #endif /* BTR_CUR_HASH_ADAPT */ {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT, @@ -2173,7 +2173,7 @@ static bool innodb_init_param() srv_page_size = 0; srv_page_size_shift = 0; #ifdef BTR_CUR_HASH_ADAPT - btr_ahi_parts = 1; + btr_search.n_parts = 1; #endif /* BTR_CUR_HASH_ADAPT */ if (innobase_page_size != (1LL << 14)) { diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 37b2703e171f9..ba1fee9d4072d 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -199,7 +199,6 @@ SET(INNOBASE_SOURCES include/btr0pcur.h include/btr0pcur.inl include/btr0sea.h - include/btr0sea.inl include/btr0types.h include/buf0buddy.h include/buf0buf.h @@ -266,8 +265,6 @@ SET(INNOBASE_SOURCES include/gis0rtree.inl include/gis0type.h include/ha_prototypes.h - include/ha0ha.h - include/ha0ha.inl include/ha0storage.h include/ha0storage.inl include/handler0alter.h diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index f91290925d63f..7befde5fa7c18 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -284,7 +284,7 @@ btr_root_block_get( #ifndef BTR_CUR_ADAPT static constexpr buf_block_t *guess= nullptr; #else - buf_block_t *&guess= btr_search_get_info(index)->root_guess; + buf_block_t *&guess= index->search_info.root_guess; guess= #endif block= @@ -1221,7 +1221,7 @@ dberr_t dict_index_t::clear(que_thr_t *thr) #ifndef BTR_CUR_ADAPT static constexpr buf_block_t *guess= nullptr; #else - buf_block_t *&guess= btr_search_get_info(this)->root_guess; + buf_block_t *&guess= search_info.root_guess; guess= #endif root_block= buf_page_get_gen({table->space_id, page}, @@ -1231,14 +1231,12 @@ dberr_t dict_index_t::clear(que_thr_t *thr) { btr_free_but_not_root(root_block, mtr.get_log_mode() #ifdef BTR_CUR_HASH_ADAPT - ,n_ahi_pages() != 0 + ,any_ahi_pages() #endif ); - + btr_search_drop_page_hash_index(root_block, false); #ifdef BTR_CUR_HASH_ADAPT - if (root_block->index) - btr_search_drop_page_hash_index(root_block, false); - ut_ad(n_ahi_pages() == 0); + ut_ad(!any_ahi_pages()); #endif mtr.memset(root_block, PAGE_HEADER + PAGE_BTR_SEG_LEAF, FSEG_HEADER_SIZE, 0); @@ -1283,7 +1281,7 @@ void btr_drop_temporary_table(const dict_table_t &table) #ifndef BTR_CUR_ADAPT static constexpr buf_block_t *guess= nullptr; #else - buf_block_t *guess= index->search_info->root_guess; + buf_block_t *guess= index->search_info.root_guess; #endif if (buf_block_t *block= buf_page_get_low({SRV_TMP_SPACE_ID, index->page}, 0, RW_X_LATCH, guess, BUF_GET, diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 6a57f2dc30755..4d11052ba6d0f 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1164,17 +1164,19 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, #ifndef BTR_CUR_ADAPT guess= nullptr; #else - btr_search_t *info= btr_search_get_info(index()); + auto info= &index()->search_info; guess= info->root_guess; # ifdef BTR_CUR_HASH_ADAPT # ifdef UNIV_SEARCH_PERF_STAT info->n_searches++; # endif - /* We do a dirty read of btr_search_enabled below, - and btr_search_guess_on_hash() will have to check it again. */ - if (!btr_search_enabled); - else if (btr_search_guess_on_hash(index(), info, tuple, mode, + if (latch_mode > BTR_MODIFY_LEAF) + /* The adaptive hash index cannot be useful for these searches. */; + /* We do a dirty read of btr_search.enabled below, + and btr_search_guess_on_hash() will have to check it again. */ + else if (!btr_search.enabled); + else if (btr_search_guess_on_hash(index(), tuple, mode, latch_mode, this, mtr)) { /* Search using the hash index succeeded */ @@ -1507,7 +1509,7 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, reached_latched_leaf: #ifdef BTR_CUR_HASH_ADAPT - if (btr_search_enabled && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG)) + if (btr_search.enabled && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG)) { if (page_cur_search_with_match_bytes(tuple, mode, &up_match, &up_bytes, @@ -1532,18 +1534,18 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, goto need_opposite_intention; #ifdef BTR_CUR_HASH_ADAPT - /* We do a dirty read of btr_search_enabled here. We will - properly check btr_search_enabled again in + /* We do a dirty read of btr_search.enabled here. We will recheck in btr_search_build_page_hash_index() before building a page hash index, while holding search latch. */ - if (!btr_search_enabled); + if (!btr_search.enabled); else if (tuple->info_bits & REC_INFO_MIN_REC_FLAG) /* This may be a search tuple for btr_pcur_t::restore_position(). */ ut_ad(tuple->is_metadata() || (tuple->is_metadata(tuple->info_bits ^ REC_STATUS_INSTANT))); else if (index()->table->is_temporary()); - else if (!rec_is_metadata(page_cur.rec, *index())) - btr_search_info_update(index(), this); + else if (!rec_is_metadata(page_cur.rec, *index()) && + index()->search_info.hash_analysis_useful()) + search_info_update(); #endif /* BTR_CUR_HASH_ADAPT */ goto func_exit; @@ -1779,18 +1781,18 @@ dberr_t btr_cur_t::pessimistic_search_leaf(const dtuple_t *tuple, ut_ad(low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); #ifdef BTR_CUR_HASH_ADAPT - /* We do a dirty read of btr_search_enabled here. We will - properly check btr_search_enabled again in + /* We do a dirty read of btr_search.enabled here. We will recheck in btr_search_build_page_hash_index() before building a page hash index, while holding search latch. */ - if (!btr_search_enabled); + if (!btr_search.enabled); else if (tuple->info_bits & REC_INFO_MIN_REC_FLAG) /* This may be a search tuple for btr_pcur_t::restore_position(). */ ut_ad(tuple->is_metadata() || (tuple->is_metadata(tuple->info_bits ^ REC_STATUS_INSTANT))); else if (index()->table->is_temporary()); - else if (!rec_is_metadata(page_cur.rec, *index())) - btr_search_info_update(index(), this); + else if (!rec_is_metadata(page_cur.rec, *index()) && + index()->search_info.hash_analysis_useful()) + search_info_update(); #endif /* BTR_CUR_HASH_ADAPT */ err= DB_SUCCESS; } @@ -1893,8 +1895,7 @@ dberr_t btr_cur_search_to_nth_level(ulint level, #ifndef BTR_CUR_ADAPT buf_block_t *block= nullptr; #else - btr_search_t *info= btr_search_get_info(index); - buf_block_t *block= info->root_guess; + buf_block_t *block= index->search_info.root_guess; #endif /* BTR_CUR_ADAPT */ ut_ad(mtr->memo_contains_flagged(&index->lock, @@ -2651,12 +2652,10 @@ btr_cur_optimistic_insert( ut_ad(flags == BTR_NO_LOCKING_FLAG); } else if (index->table->is_temporary()) { } else { - srw_spin_lock* ahi_latch = btr_search_sys.get_latch(*index); if (!reorg && cursor->flag == BTR_CUR_HASH) { - btr_search_update_hash_node_on_insert( - cursor, ahi_latch); + btr_search_update_hash_node_on_insert(cursor); } else { - btr_search_update_hash_on_insert(cursor, ahi_latch); + btr_search_update_hash_on_insert(cursor); } } #endif /* BTR_CUR_HASH_ADAPT */ @@ -2849,8 +2848,7 @@ btr_cur_pessimistic_insert( ut_ad(!(flags & BTR_CREATE_FLAG)); } else if (index->table->is_temporary()) { } else { - btr_search_update_hash_on_insert( - cursor, btr_search_sys.get_latch(*index)); + btr_search_update_hash_on_insert(cursor); } #endif /* BTR_CUR_HASH_ADAPT */ if (inherit && !(flags & BTR_NO_LOCKING_FLAG)) { @@ -3434,9 +3432,9 @@ btr_cur_update_in_place( #ifdef BTR_CUR_HASH_ADAPT { - srw_spin_lock* ahi_latch = block->index - ? btr_search_sys.get_latch(*index) : NULL; - if (ahi_latch) { + auto part = block->index + ? &btr_search.get_part(*index) : nullptr; + if (part) { /* TO DO: Can we skip this if none of the fields index->search_info->curr_n_fields are being updated? */ @@ -3454,7 +3452,7 @@ btr_cur_update_in_place( btr_search_update_hash_on_delete(cursor); } - ahi_latch->wr_lock(SRW_LOCK_CALL); + part->latch.wr_lock(SRW_LOCK_CALL); } assert_block_ahi_valid(block); @@ -3464,8 +3462,8 @@ btr_cur_update_in_place( mtr); #ifdef BTR_CUR_HASH_ADAPT - if (ahi_latch) { - ahi_latch->wr_unlock(); + if (part) { + part->latch.wr_unlock(); } } #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index c74b7fdfc3fba..e2cc2739832b9 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -34,13 +34,7 @@ Created 2/17/1996 Heikki Tuuri #include "btr0pcur.h" #include "btr0btr.h" #include "srv0mon.h" - -/** Is search system enabled. -Search system is protected by array of latches. */ -char btr_search_enabled; - -/** Number of adaptive hash index partition. */ -ulong btr_ahi_parts; +#include "log.h" #ifdef UNIV_SEARCH_PERF_STAT /** Number of successful adaptive hash index lookups */ @@ -54,7 +48,97 @@ mysql_pfs_key_t btr_search_latch_key; #endif /* UNIV_PFS_RWLOCK */ /** The adaptive hash index */ -btr_search_sys_t btr_search_sys; +btr_sea btr_search; + +struct ahi_node { + /** rec_fold(rec) */ + ulint fold; + /** pointer to next record in the hash bucket chain, or nullptr */ + ahi_node *next; + /** B-tree index leaf page record */ + const rec_t *rec; +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + /** block containing rec, or nullptr */ + buf_block_t *block; +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ +}; + +inline void btr_sea::partition::init() noexcept +{ + latch.SRW_LOCK_INIT(btr_search_latch_key); + blocks_mutex.init(); + UT_LIST_INIT(blocks, &buf_page_t::list); +} + +inline void btr_sea::partition::clear() noexcept +{ +#ifndef SUX_LOCK_GENERIC + ut_ad(latch.is_write_locked()); + ut_ad(blocks_mutex.is_locked()); +#endif + if (buf_block_t *b= spare) + { + spare= nullptr; + MEM_MAKE_ADDRESSABLE(b->page.frame, srv_page_size); + buf_pool.free_block(b); + } + table.free(); + + while (buf_page_t *b= UT_LIST_GET_FIRST(blocks)) + { + UT_LIST_REMOVE(blocks, b); + ut_ad(b->free_offset); + b->hash= nullptr; + MEM_MAKE_ADDRESSABLE(b->frame, srv_page_size); + buf_pool.free_block(reinterpret_cast(b)); + } +} + +inline void btr_sea::partition::free() noexcept +{ + if (table.array) + { + ut_d(latch.wr_lock(SRW_LOCK_CALL)); + ut_d(blocks_mutex.wr_lock()); + clear(); + ut_d(blocks_mutex.wr_unlock()); + ut_d(latch.wr_unlock()); + } + latch.destroy(); + blocks_mutex.destroy(); +} + +inline void btr_sea::partition::alloc(ulint hash_size) noexcept +{ + table.create(hash_size); +} + +void btr_sea::create() noexcept +{ + for (partition &part : parts) + part.init(); + if (enabled) + enable(); +} + +void btr_sea::alloc(ulint hash_size) noexcept +{ + hash_size/= n_parts; + for (ulong i= 0; i < n_parts; ++i) + parts[i].alloc(hash_size); +} + +inline void btr_sea::clear() noexcept +{ + for (ulong i= 0; i < n_parts; ++i) + parts[i].clear(); +} + +void btr_sea::free() noexcept +{ + for (partition &part : parts) + part.free(); +} /** If the number of records on the page divided by this parameter would have been successfully accessed using a hash index, the index @@ -156,42 +240,35 @@ btr_search_get_n_fields( return(btr_search_get_n_fields(cursor->n_fields, cursor->n_bytes)); } -/** This function should be called before reserving any btr search mutex, if -the intended operation might add nodes to the search system hash table. -Because of the latching order, once we have reserved the btr search system -latch, we cannot allocate a free frame from the buffer pool. Checks that -there is a free buffer frame allocated for hash table heap in the btr search -system. If not, allocates a free frames for the heap. This check makes it -probable that, when have reserved the btr search system latch and we need to -allocate a new node to the hash table, it will succeed. However, the check -will not guarantee success. -@param[in] index index handler */ -static void btr_search_check_free_space_in_heap(const dict_index_t *index) +void btr_sea::partition::prepare_insert() noexcept { - /* Note that we peek the value of heap->free_block without reserving - the latch: this is ok, because we will not guarantee that there will - be enough free space in the hash table. */ - - buf_block_t *block= buf_block_alloc(); - auto part= btr_search_sys.get_part(*index); - - part->latch.wr_lock(SRW_LOCK_CALL); - - if (!btr_search_enabled || part->heap->free_block) - buf_block_free(block); - else - part->heap->free_block= block; - - part->latch.wr_unlock(); + /* spare may be consumed by insert() or clear() */ + if (!spare) + { + buf_block_t *block= buf_block_alloc(); + blocks_mutex.wr_lock(); + if (!spare && btr_search.enabled) + { + MEM_NOACCESS(block->page.frame, srv_page_size); + spare= block; + block= nullptr; + } + blocks_mutex.wr_unlock(); + if (block) + buf_pool.free_block(block); + } } -/** Set index->ref_count = 0 on all indexes of a table. -@param[in,out] table table handler */ -static void btr_search_disable_ref_count(dict_table_t *table) +/** Clear the AHI reference count on all tables. +@param tables list of tables */ +static void +btr_search_disable(const UT_LIST_BASE_NODE_T(dict_table_t)& tables) noexcept { - for (dict_index_t *index= dict_table_get_first_index(table); index; - index= dict_table_get_next_index(index)) - index->search_info->ref_count= 0; + for (dict_table_t *table= UT_LIST_GET_FIRST(dict_sys.table_LRU); table; + table = UT_LIST_GET_NEXT(table_LRU, table)) + for (dict_index_t *index= dict_table_get_first_index(table); index; + index= dict_table_get_next_index(index)) + index->search_info.ref_count= 0; } /** Lazily free detached metadata when removing the last reference. */ @@ -220,82 +297,79 @@ ATTRIBUTE_COLD static void btr_search_lazy_free(dict_index_t *index) } /** Disable the adaptive hash search system and empty the index. */ -void btr_search_disable() +void btr_sea::disable() noexcept { - dict_table_t* table; + dict_sys.freeze(SRW_LOCK_CALL); - dict_sys.freeze(SRW_LOCK_CALL); - - btr_search_x_lock_all(); - - if (!btr_search_enabled) { - dict_sys.unfreeze(); - btr_search_x_unlock_all(); - return; - } - - btr_search_enabled = false; - - /* Clear the index->search_info->ref_count of every index in - the data dictionary cache. */ - for (table = UT_LIST_GET_FIRST(dict_sys.table_LRU); table; - table = UT_LIST_GET_NEXT(table_LRU, table)) { - - btr_search_disable_ref_count(table); - } - - for (table = UT_LIST_GET_FIRST(dict_sys.table_non_LRU); table; - table = UT_LIST_GET_NEXT(table_LRU, table)) { - - btr_search_disable_ref_count(table); - } - - dict_sys.unfreeze(); - - /* Set all block->index = NULL. */ - buf_pool.clear_hash_index(); + for (ulong i= 0; i < n_parts; i++) + { + parts[i].latch.wr_lock(SRW_LOCK_CALL); + parts[i].blocks_mutex.wr_lock(); + } - /* Clear the adaptive hash index. */ - btr_search_sys.clear(); + if (enabled) + { + enabled= false; + btr_search_disable(dict_sys.table_LRU); + btr_search_disable(dict_sys.table_non_LRU); + dict_sys.unfreeze(); + + /* Set all block->index= nullptr. */ + buf_pool.clear_hash_index(); + clear(); + } + else + dict_sys.unfreeze(); - btr_search_x_unlock_all(); + for (ulong i= 0; i < n_parts; i++) + { + parts[i].latch.wr_unlock(); + parts[i].blocks_mutex.wr_unlock(); + } } /** Enable the adaptive hash search system. @param resize whether buf_pool_t::resize() is the caller */ -void btr_search_enable(bool resize) +void btr_sea::enable(bool resize) noexcept { - if (!resize) { - mysql_mutex_lock(&buf_pool.mutex); - bool changed = srv_buf_pool_old_size != srv_buf_pool_size; - mysql_mutex_unlock(&buf_pool.mutex); - if (changed) { - return; - } - } + if (!resize) + { + mysql_mutex_lock(&buf_pool.mutex); + bool changed= srv_buf_pool_old_size != srv_buf_pool_size; + mysql_mutex_unlock(&buf_pool.mutex); + if (changed) + return; + } - btr_search_x_lock_all(); - ulint hash_size = buf_pool_get_curr_size() / sizeof(void *) / 64; + for (ulong i= 0; i < n_parts; i++) + { + parts[i].latch.wr_lock(SRW_LOCK_CALL); + parts[i].blocks_mutex.wr_lock(); + } - if (btr_search_sys.parts[0].heap) { - ut_ad(btr_search_enabled); - btr_search_x_unlock_all(); - return; - } + if (!parts[0].table.array) + { + enabled= true; + alloc(buf_pool_get_curr_size() / sizeof(void *) / 64); + } - btr_search_sys.alloc(hash_size); + ut_ad(enabled); - btr_search_enabled = true; - btr_search_x_unlock_all(); + for (ulong i= 0; i < n_parts; i++) + { + parts[i].blocks_mutex.wr_unlock(); + parts[i].latch.wr_unlock(); + } } /** Updates the search info of an index about hash successes. NOTE that info is NOT protected by any semaphore, to save CPU time! Do not assume its fields are consistent. -@param[in,out] info search info @param[in] cursor cursor which was just positioned */ -static void btr_search_info_update_hash(btr_search_t *info, btr_cur_t *cursor) +static void btr_search_info_update_hash(const btr_cur_t *cursor) { + ut_ad(cursor->flag != BTR_CUR_HASH); + dict_index_t* index = cursor->index(); int cmp; @@ -307,8 +381,10 @@ static void btr_search_info_update_hash(btr_search_t *info, btr_cur_t *cursor) } uint16_t n_unique = dict_index_get_n_unique_in_tree(index); + dict_index_t::ahi* info = &index->search_info; + uint8_t n_hash_potential = info->n_hash_potential; - if (info->n_hash_potential == 0) { + if (n_hash_potential == 0) { goto set_new_recomm; } @@ -318,8 +394,9 @@ static void btr_search_info_update_hash(btr_search_t *info, btr_cur_t *cursor) if (info->n_fields >= n_unique && cursor->up_match >= n_unique) { increment_potential: - info->n_hash_potential++; - + if (n_hash_potential < BTR_SEARCH_BUILD_LIMIT) { + info->n_hash_potential = ++n_hash_potential; + } return; } @@ -344,7 +421,7 @@ static void btr_search_info_update_hash(btr_search_t *info, btr_cur_t *cursor) for a while to avoid unnecessary CPU time usage when there is no chance for success */ - info->hash_analysis = 0; + info->hash_analysis_reset(); cmp = ut_pair_cmp(cursor->up_match, cursor->up_bytes, cursor->low_match, cursor->low_bytes); @@ -356,8 +433,6 @@ static void btr_search_info_update_hash(btr_search_t *info, btr_cur_t *cursor) info->n_fields = 1; info->n_bytes = 0; } else if (cmp > 0) { - info->n_hash_potential = 1; - if (cursor->up_match >= n_unique) { info->n_fields = n_unique; @@ -401,13 +476,12 @@ semaphore, to save CPU time! Do not assume the fields are consistent. @param[in,out] block buffer block */ static bool -btr_search_update_block_hash_info(btr_search_t* info, buf_block_t* block) +btr_search_update_block_hash_info(dict_index_t::ahi* info, buf_block_t* block) { ut_ad(block->page.lock.have_x() || block->page.lock.have_s()); info->last_hash_succ = FALSE; ut_ad(block->page.frame); - ut_ad(info->magic_n == BTR_SEARCH_MAGIC_N); if ((block->n_hash_helps > 0) && (info->n_hash_potential > 0) @@ -459,164 +533,207 @@ btr_search_update_block_hash_info(btr_search_t* info, buf_block_t* block) constexpr ulint MAX_N_POINTERS = UNIV_PAGE_SIZE_MAX / REC_N_NEW_EXTRA_BYTES; #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ -__attribute__((nonnull)) -/** -Insert an entry into the hash table. If an entry with the same fold number -is found, its node is updated to point to the new data, and no new node -is inserted. -@param table hash table -@param heap memory heap -@param fold folded value of the record -@param block buffer block containing the record -@param data the record -@retval true on success -@retval false if no more memory could be allocated */ -static bool ha_insert_for_fold(hash_table_t *table, mem_heap_t* heap, - ulint fold, #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - buf_block_t *block, /*!< buffer block of data */ +void btr_sea::partition::insert(ulint fold, const rec_t *rec, + buf_block_t *block) noexcept +#else +void btr_sea::partition::insert(ulint fold, const rec_t *rec) noexcept #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - const rec_t *data) { +#ifndef SUX_LOCK_GENERIC + ut_ad(latch.is_write_locked()); +#endif #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - ut_a(block->page.frame == page_align(data)); + ut_a(block->page.frame == page_align(rec)); #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - ut_ad(btr_search_enabled); + ut_ad(btr_search.enabled); - hash_cell_t *cell= &table->array[table->calc_hash(fold)]; + ahi_node **prev= table.cell_get(fold)-> + search(&ahi_node::next, [fold](const ahi_node *node) + { return !node || node->fold == fold; }); + ahi_node *node= *prev; - for (ha_node_t *prev= static_cast(cell->node); prev; - prev= prev->next) + if (node) { - if (prev->fold == fold) - { #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - buf_block_t *prev_block= prev->block; - ut_a(prev_block->page.frame == page_align(prev->data)); + buf_block_t *prev_block= node->block; + if (prev_block != block) + { + ut_a(prev_block->page.frame == page_align(node->rec)); ut_a(prev_block->n_pointers-- < MAX_N_POINTERS); ut_a(block->n_pointers++ < MAX_N_POINTERS); - - prev->block= block; -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - prev->data= data; - return true; + node->block= block; } +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + node->rec= rec; + return; } /* We have to allocate a new chain node */ - ha_node_t *node= static_cast(mem_heap_alloc(heap, sizeof *node)); - - if (!node) - return false; - ha_node_set_data(node, block, data); + { + blocks_mutex.wr_lock(); + buf_page_t *last= UT_LIST_GET_LAST(blocks); + if (last && last->free_offset < srv_page_size - sizeof *node) + { + node= reinterpret_cast(last->frame + last->free_offset); +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic push +# if __GNUC__ < 12 || defined WITH_UBSAN +# pragma GCC diagnostic ignored "-Wconversion" +# endif +#endif + last->free_offset+= sizeof *node; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic pop +#endif + MEM_MAKE_ADDRESSABLE(node, sizeof *node); + } + else + { + last= &spare.load()->page; + if (!last) + { + blocks_mutex.wr_unlock(); + return; + } + spare= nullptr; + ut_ad(last->state() == buf_page_t::MEMORY); + ut_ad(!reinterpret_cast(last)->index); + ut_ad(!reinterpret_cast(last)->n_pointers); + UT_LIST_ADD_LAST(blocks, last); + last->free_offset= sizeof *node; + node= reinterpret_cast(last->frame); + MEM_UNDEFINED(last->frame, srv_page_size); + MEM_MAKE_ADDRESSABLE(node, sizeof *node); + MEM_NOACCESS(node + 1, srv_page_size - sizeof *node); + } + blocks_mutex.wr_unlock(); + } #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG ut_a(block->n_pointers++ < MAX_N_POINTERS); + node->block= block; #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + node->rec= rec; node->fold= fold; node->next= nullptr; - ha_node_t *prev= static_cast(cell->node); - if (!prev) - cell->node= node; - else - { - while (prev->next) - prev= prev->next; - prev->next= node; - } - return true; + *prev= node; } -__attribute__((nonnull)) -/** Delete a record. -@param table hash table -@param heap memory heap -@param del_node record to be deleted */ -static void ha_delete_hash_node(hash_table_t *table, mem_heap_t *heap, - ha_node_t *del_node) +buf_block_t *btr_sea::partition::cleanup_after_erase(ahi_node *erase) noexcept { - ut_ad(btr_search_enabled); + ut_ad(btr_search.enabled); +#ifndef SUX_LOCK_GENERIC + ut_ad(latch.is_write_locked()); +#endif #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - ut_a(del_node->block->page.frame == page_align(del_node->data)); - ut_a(del_node->block->n_pointers-- < MAX_N_POINTERS); + ut_a(erase->block->page.frame == page_align(erase->rec)); + ut_a(erase->block->n_pointers-- < MAX_N_POINTERS); #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - table->cell_get(del_node->fold)->remove(*del_node, &ha_node_t::next); + blocks_mutex.wr_lock(); - ha_node_t *top= static_cast(mem_heap_get_top(heap, sizeof *top)); + buf_page_t *last= UT_LIST_GET_LAST(blocks); + const ahi_node *const top= reinterpret_cast + (last->frame + last->free_offset - sizeof *top); - if (del_node != top) + if (erase != top) { - /* Compact the heap of nodes by moving the top in the place of del_node. */ - *del_node= *top; - hash_cell_t *cell= &table->array[table->calc_hash(top->fold)]; - - /* Look for the pointer to the top node, to update it */ - if (cell->node == top) - /* The top node is the first in the chain */ - cell->node= del_node; - else - { - /* We have to look for the predecessor */ - ha_node_t *node= static_cast(cell->node); + /* Shrink the allocation by replacing the erased element with the top. */ + *erase= *top; + ahi_node **prev= table.cell_get(top->fold)-> + search(&ahi_node::next, [top](const ahi_node *n) { return n == top; }); + *prev= erase; + } - while (top != node->next) node= node->next; + buf_block_t *freed= nullptr; - /* Now we have the predecessor node */ - node->next= del_node; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic push +# if __GNUC__ < 12 || defined WITH_UBSAN +# pragma GCC diagnostic ignored "-Wconversion" +# endif +#endif + /* We may be able to shrink or free the last block */ + if (!(last->free_offset-= uint16_t(sizeof *erase))) +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic pop +#endif + { + if (spare) + { + freed= reinterpret_cast(last); + MEM_MAKE_ADDRESSABLE(last->frame, srv_page_size); } + else + spare= reinterpret_cast(last); + UT_LIST_REMOVE(blocks, last); } + else + MEM_NOACCESS(last->frame + last->free_offset, sizeof *erase); - /* Free the occupied space */ - mem_heap_free_top(heap, sizeof *top); + blocks_mutex.wr_unlock(); + return freed; } __attribute__((nonnull)) /** Delete all pointers to a page. -@param table hash table -@param heap memory heap -@param page record to be deleted */ -static void ha_remove_all_nodes_to_page(hash_table_t *table, mem_heap_t *heap, +@param part hash table partition +@param fold fold value +@param page page of a record to be deleted */ +static void ha_remove_all_nodes_to_page(btr_sea::partition &part, ulint fold, const page_t *page) + noexcept { - for (ha_node_t *node= ha_chain_get_first(table, fold); node; ) + hash_cell_t *cell= part.table.cell_get(fold); + const uintptr_t page_size{srv_page_size}; + +rewind: + ahi_node **prev= + cell->search(&ahi_node::next, [page,page_size](const ahi_node *node) + { return !node || (uintptr_t(node->rec) ^ uintptr_t(page)) < page_size; }); + + if (ahi_node *node= *prev) { - if (page_align(ha_node_get_data(node)) == page) - { - ha_delete_hash_node(table, heap, node); - /* The deletion may compact the heap of nodes and move other nodes! */ - node= ha_chain_get_first(table, fold); - } - else - node= ha_chain_get_next(node); + *prev= node->next; + node->next= nullptr; + if (buf_block_t *block= part.cleanup_after_erase(node)) + buf_pool.free_block(block); + /* The deletion may compact the heap of nodes and move other nodes! */ + goto rewind; } -#ifdef UNIV_DEBUG + /* Check that all nodes really got deleted */ - for (ha_node_t *node= ha_chain_get_first(table, fold); node; - node= ha_chain_get_next(node)) - ut_ad(page_align(ha_node_get_data(node)) != page); -#endif /* UNIV_DEBUG */ + ut_ad(!cell->find(&ahi_node::next, [page](const ahi_node* node) + { return page_align(node->rec) == page; })); } -/** Delete a record if found. -@param table hash table -@param heap memory heap for the hash bucket chain -@param fold folded value of the searched data -@param data pointer to the record -@return whether the record was found */ -static bool ha_search_and_delete_if_found(hash_table_t *table, - mem_heap_t *heap, - ulint fold, const rec_t *data) +inline bool btr_sea::partition::erase(ulint fold, const rec_t *rec) noexcept { - if (ha_node_t *node= ha_search_with_data(table, fold, data)) +#ifndef SUX_LOCK_GENERIC + ut_ad(latch.is_write_locked()); +#endif + ut_ad(btr_search.enabled); + + ahi_node **prev= table.cell_get(fold)-> + search(&ahi_node::next, [rec](const ahi_node *node) + { return !node || node->rec == rec; }); + + if (ahi_node *node= *prev) { - ha_delete_hash_node(table, heap, node); + *prev= node->next; + node->next= nullptr; + buf_block_t *block= cleanup_after_erase(node); + latch.wr_unlock(); + if (block) + buf_pool.free_block(block); return true; } + latch.wr_unlock(); return false; } @@ -640,18 +757,22 @@ static bool ha_search_and_update_if_found(hash_table_t *table, ulint fold, ut_a(new_block->page.frame == page_align(new_data)); #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - if (!btr_search_enabled) + if (!btr_search.enabled) return false; - if (ha_node_t *node= ha_search_with_data(table, fold, data)) + if (ahi_node *node= table->cell_get(fold)-> + find(&ahi_node::next, [data](const ahi_node *node) + { return node->rec == data; })) { #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - ut_a(node->block->n_pointers-- < MAX_N_POINTERS); - ut_a(new_block->n_pointers++ < MAX_N_POINTERS); - node->block= new_block; + if (node->block != new_block) + { + ut_a(node->block->n_pointers-- < MAX_N_POINTERS); + ut_a(new_block->n_pointers++ < MAX_N_POINTERS); + node->block= new_block; + } #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - node->data= new_data; - + node->rec= new_data; return true; } @@ -659,8 +780,9 @@ static bool ha_search_and_update_if_found(hash_table_t *table, ulint fold, } #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG +# define ha_insert_for_fold(p,f,b,d) (p).insert(f,d,b) #else -# define ha_insert_for_fold(t,h,f,b,d) ha_insert_for_fold(t,h,f,d) +# define ha_insert_for_fold(p,f,b,d) (p).insert(f,d) # define ha_search_and_update_if_found(table,fold,data,new_block,new_data) \ ha_search_and_update_if_found(table,fold,data,new_data) #endif @@ -672,18 +794,12 @@ what happens at page boundaries, and therefore there can be misleading hash nodes. Also, collisions in the fold value can lead to misleading references. This function lazily fixes these imperfections in the hash index. -@param[in] info search info -@param[in] block buffer block where cursor positioned @param[in] cursor cursor */ -static -void -btr_search_update_hash_ref( - const btr_search_t* info, - buf_block_t* block, - const btr_cur_t* cursor) +static void btr_search_update_hash_ref(const btr_cur_t* cursor) noexcept { ut_ad(cursor->flag == BTR_CUR_HASH_FAIL); + buf_block_t* const block = cursor->page_cur.block; ut_ad(block->page.lock.have_x() || block->page.lock.have_s()); ut_ad(btr_cur_get_page(cursor) == block->page.frame); ut_ad(page_is_leaf(block->page.frame)); @@ -691,7 +807,7 @@ btr_search_update_hash_ref( dict_index_t* index = block->index; - if (!index || !info->n_hash_potential) { + if (!index || !index->search_info.n_hash_potential) { return; } @@ -704,15 +820,15 @@ btr_search_update_hash_ref( ut_ad(block->page.id().space() == index->table->space_id); ut_ad(index == cursor->index()); ut_ad(!dict_index_is_ibuf(index)); - auto part = btr_search_sys.get_part(*index); - part->latch.wr_lock(SRW_LOCK_CALL); + btr_sea::partition& part = btr_search.get_part(*index); + part.latch.wr_lock(SRW_LOCK_CALL); ut_ad(!block->index || block->index == index); if (block->index - && (block->curr_n_fields == info->n_fields) - && (block->curr_n_bytes == info->n_bytes) - && (block->curr_left_side == info->left_side) - && btr_search_enabled) { + && (block->curr_n_fields == index->search_info.n_fields) + && (block->curr_n_bytes == index->search_info.n_bytes) + && (block->curr_left_side == index->search_info.left_side) + && btr_search.enabled) { mem_heap_t* heap = NULL; rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; rec_offs_init(offsets_); @@ -734,26 +850,19 @@ btr_search_update_hash_ref( mem_heap_free(heap); } - ha_insert_for_fold(&part->table, part->heap, fold, block, rec); + ha_insert_for_fold(part, fold, block, rec); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } func_exit: - part->latch.wr_unlock(); + part.latch.wr_unlock(); } /** Checks if a guessed position for a tree cursor is right. Note that if mode is PAGE_CUR_LE, which is used in inserts, and the function returns TRUE, then cursor->up_match and cursor->low_match both have sensible values. @param[in,out] cursor guess cursor position -@param[in] can_only_compare_to_cursor_rec - if we do not have a latch on the page of cursor, - but a latch corresponding search system, then - ONLY the columns of the record UNDER the cursor - are protected, not the next or previous record - in the chain: we cannot look at the next or - previous record to check our guess! @param[in] tuple data tuple @param[in] mode PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, PAGE_CUR_GE @return whether a match was found */ @@ -761,7 +870,6 @@ static bool btr_search_check_guess( btr_cur_t* cursor, - bool can_only_compare_to_cursor_rec, const dtuple_t* tuple, ulint mode) { @@ -829,12 +937,6 @@ btr_search_check_guess( } } - if (can_only_compare_to_cursor_rec) { - /* Since we could not determine if our guess is right just by - looking at the record under the cursor, return FALSE */ - goto exit_func; - } - match = 0; if ((mode == PAGE_CUR_G) || (mode == PAGE_CUR_GE)) { @@ -920,28 +1022,11 @@ btr_search_check_guess( return(success); } -static -void -btr_search_failure(btr_search_t* info, btr_cur_t* cursor) -{ - cursor->flag = BTR_CUR_HASH_FAIL; - -#ifdef UNIV_SEARCH_PERF_STAT - ++info->n_hash_fail; - - if (info->n_hash_succ > 0) { - --info->n_hash_succ; - } -#endif /* UNIV_SEARCH_PERF_STAT */ - - info->last_hash_succ = FALSE; -} - /** Clear the adaptive hash index on all pages in the buffer pool. */ inline void buf_pool_t::clear_hash_index() noexcept { ut_ad(!resizing); - ut_ad(!btr_search_enabled); + ut_ad(!btr_search.enabled); std::set garbage; @@ -951,7 +1036,6 @@ inline void buf_pool_t::clear_hash_index() noexcept block != end; block++) { dict_index_t *index= block->index; - assert_block_ahi_valid(block); /* We can clear block->index and block->n_pointers when holding all AHI latches exclusively; see the comments in buf0buf.h */ @@ -976,9 +1060,11 @@ inline void buf_pool_t::clear_hash_index() noexcept # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG block->n_pointers= 0; # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - if (index->freed()) - garbage.insert(index); block->index= nullptr; + if (index->freed()) + garbage.emplace(index); + else + index->search_info.ref_count= 0; } } @@ -1024,7 +1110,6 @@ of the index. Note that if mode is PAGE_CUR_LE, which is used in inserts, and the function returns TRUE, then cursor->up_match and cursor->low_match both have sensible values. @param[in,out] index index -@param[in,out] info index search info @param[in] tuple logical record @param[in] mode PAGE_CUR_L, .... @param[in] latch_mode BTR_SEARCH_LEAF, ... @@ -1035,162 +1120,162 @@ TRANSACTIONAL_TARGET bool btr_search_guess_on_hash( dict_index_t* index, - btr_search_t* info, const dtuple_t* tuple, ulint mode, ulint latch_mode, btr_cur_t* cursor, - mtr_t* mtr) + mtr_t* mtr) noexcept { - ulint fold; - index_id_t index_id; - - ut_ad(mtr->is_active()); - ut_ad(index->is_btree() || index->is_ibuf()); - - /* Note that, for efficiency, the struct info may not be protected by - any latch here! */ + ut_ad(mtr->is_active()); + ut_ad(index->is_btree() || index->is_ibuf()); + ut_ad(latch_mode == BTR_SEARCH_LEAF || latch_mode == BTR_MODIFY_LEAF); - if (latch_mode > BTR_MODIFY_LEAF - || !info->last_hash_succ || !info->n_hash_potential - || (tuple->info_bits & REC_INFO_MIN_REC_FLAG)) { - return false; - } + if ((tuple->info_bits & REC_INFO_MIN_REC_FLAG) || + !index->search_info.last_hash_succ || + !index->search_info.n_hash_potential) + return false; - ut_ad(index->is_btree()); - ut_ad(!index->table->is_temporary()); + ut_ad(index->is_btree()); + ut_ad(!index->table->is_temporary()); - ut_ad(latch_mode == BTR_SEARCH_LEAF || latch_mode == BTR_MODIFY_LEAF); - compile_time_assert(ulint{BTR_SEARCH_LEAF} == ulint{RW_S_LATCH}); - compile_time_assert(ulint{BTR_MODIFY_LEAF} == ulint{RW_X_LATCH}); + static_assert(ulint{BTR_SEARCH_LEAF} == ulint{RW_S_LATCH}, ""); + static_assert(ulint{BTR_MODIFY_LEAF} == ulint{RW_X_LATCH}, ""); - cursor->n_fields = info->n_fields; - cursor->n_bytes = info->n_bytes; + cursor->n_fields = index->search_info.n_fields; + cursor->n_bytes = index->search_info.n_bytes; - if (dtuple_get_n_fields(tuple) < btr_search_get_n_fields(cursor)) { - return false; - } + if (dtuple_get_n_fields(tuple) < btr_search_get_n_fields(cursor)) + return false; - index_id = index->id; + const index_id_t index_id= index->id; #ifdef UNIV_SEARCH_PERF_STAT - info->n_hash_succ++; + index->search_info.n_hash_succ++; #endif - fold = dtuple_fold(tuple, cursor->n_fields, cursor->n_bytes, index_id); - - cursor->fold = fold; - cursor->flag = BTR_CUR_HASH; - - auto part = btr_search_sys.get_part(*index); - const rec_t* rec; + ulint fold= dtuple_fold(tuple, cursor->n_fields, cursor->n_bytes, index_id); + cursor->fold= fold; + cursor->flag= BTR_CUR_HASH; + btr_sea::partition &part= btr_search.get_part(*index); - part->latch.rd_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); - if (!btr_search_enabled) { - goto ahi_release_and_fail; - } - - rec = static_cast( - ha_search_and_get_data(&part->table, fold)); - - if (!rec) { -ahi_release_and_fail: - part->latch.rd_unlock(); -fail: - btr_search_failure(info, cursor); - return false; - } - - buf_block_t* block = buf_pool.block_from_ahi(rec); - - buf_pool_t::hash_chain& chain = buf_pool.page_hash.cell_get( - block->page.id().fold()); - bool got_latch; - { - transactional_shared_lock_guard g{ - buf_pool.page_hash.lock_get(chain)}; - got_latch = (latch_mode == BTR_SEARCH_LEAF) - ? block->page.lock.s_lock_try() - : block->page.lock.x_lock_try(); - } - - if (!got_latch) { - goto ahi_release_and_fail; - } - - const auto state = block->page.state(); - if (UNIV_UNLIKELY(state < buf_page_t::UNFIXED)) { - ut_ad(state == buf_page_t::REMOVE_HASH); -block_and_ahi_release_and_fail: - if (latch_mode == BTR_SEARCH_LEAF) { - block->page.lock.s_unlock(); - } else { - block->page.lock.x_unlock(); - } - goto ahi_release_and_fail; - } + if (!btr_search.enabled) + { + ahi_release_and_fail: + part.latch.rd_unlock(); + fail: + cursor->flag= BTR_CUR_HASH_FAIL; - ut_ad(state < buf_page_t::READ_FIX || state >= buf_page_t::WRITE_FIX); - ut_ad(state < buf_page_t::READ_FIX || latch_mode == BTR_SEARCH_LEAF); +#ifdef UNIV_SEARCH_PERF_STAT + ++index->search_info.n_hash_fail; + if (index->search_info.n_hash_succ > 0) + --index->search_info.n_hash_succ; +#endif /* UNIV_SEARCH_PERF_STAT */ + index->search_info.last_hash_succ= false; + return false; + } - if (index != block->index && index_id == block->index->id) { - ut_a(block->index->freed()); - goto block_and_ahi_release_and_fail; - } + const ahi_node *node= part.table.cell_get(fold)-> + find(&ahi_node::next, [fold](const ahi_node* node) + { return node->fold == fold; }); - block->page.fix(); - buf_page_make_young_if_needed(&block->page); - static_assert(ulint{MTR_MEMO_PAGE_S_FIX} == ulint{BTR_SEARCH_LEAF}, - ""); - static_assert(ulint{MTR_MEMO_PAGE_X_FIX} == ulint{BTR_MODIFY_LEAF}, - ""); + if (!node) + goto ahi_release_and_fail; - part->latch.rd_unlock(); + const rec_t *rec= node->rec; + buf_block_t *block= buf_pool.block_from_ahi(rec); +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + ut_a(block == node->block); +#endif + { + buf_pool_t::hash_chain &chain= + buf_pool.page_hash.cell_get(block->page.id().fold()); + /* We must hold the cell latch while attempting to acquire + block->page.lock, because buf_LRU_block_remove_hashed() assumes + that block->page.can_relocate() will not cease to hold. */ + transactional_shared_lock_guard g + {buf_pool.page_hash.lock_get(chain)}; + if (latch_mode == BTR_SEARCH_LEAF + ? !block->page.lock.s_lock_try() + : !block->page.lock.x_lock_try()) + goto ahi_release_and_fail; + } - ++buf_pool.stat.n_page_gets; + const uint32_t state{block->page.state()}; - mtr->memo_push(block, mtr_memo_type_t(latch_mode)); + if (UNIV_UNLIKELY(state < buf_page_t::UNFIXED)) + { + ut_ad(state == buf_page_t::REMOVE_HASH); + block_and_ahi_release_and_fail: + if (latch_mode == BTR_SEARCH_LEAF) + block->page.lock.s_unlock(); + else + block->page.lock.x_unlock(); + goto ahi_release_and_fail; + } - ut_ad(page_rec_is_user_rec(rec)); + ut_ad(!block->page.is_read_fixed(state)); + ut_ad(!block->page.is_write_fixed(state) || latch_mode == BTR_SEARCH_LEAF); - btr_cur_position(index, (rec_t*) rec, block, cursor); + const dict_index_t *block_index= block->index; + if (index != block_index && index_id == block_index->id) + { + ut_a(block_index->freed()); + goto block_and_ahi_release_and_fail; + } - /* Check the validity of the guess within the page */ + /* We successfully validated the state of the block and that it + actually belongs to our index. Now it is safe to release part.latch. + Because we are holding block->page.lock, the page cannot be + modified or evicted (buf_page_t::can_relocate() will not hold) while + we validate the guessed rec. */ + part.latch.rd_unlock(); + + block->page.fix(); + buf_page_make_young_if_needed(&block->page); + static_assert(ulint{MTR_MEMO_PAGE_S_FIX} == ulint{BTR_SEARCH_LEAF}, ""); + static_assert(ulint{MTR_MEMO_PAGE_X_FIX} == ulint{BTR_MODIFY_LEAF}, ""); + + ++buf_pool.stat.n_page_gets; + + mtr->memo_push(block, mtr_memo_type_t(latch_mode)); + + ut_ad(page_rec_is_user_rec(rec)); + ut_ad(page_is_leaf(block->page.frame)); + + btr_cur_position(index, const_cast(rec), block, cursor); + const ulint comp{page_is_comp(block->page.frame)}; + if (UNIV_LIKELY(comp != 0)) + switch (rec_get_status(rec)) { + case REC_STATUS_INSTANT: + case REC_STATUS_ORDINARY: + break; + default: + mismatch: + mtr->release_last_page(); + goto fail; + } - /* If we only have the latch on search system, not on the - page, it only protects the columns of the record the cursor - is positioned on. We cannot look at the next of the previous - record to determine if our guess for the cursor position is - right. */ - if (index_id != btr_page_get_index_id(block->page.frame) - || !btr_search_check_guess(cursor, false, tuple, mode)) { - mtr->release_last_page(); - goto fail; - } + /* Check the validity of the guess within the page */ + if (index_id != btr_page_get_index_id(block->page.frame) || + !btr_search_check_guess(cursor, tuple, mode)) + goto mismatch; - if (info->n_hash_potential < BTR_SEARCH_BUILD_LIMIT + 5) { + uint8_t n_hash_potential= index->search_info.n_hash_potential; - info->n_hash_potential++; - } + if (n_hash_potential++ < BTR_SEARCH_BUILD_LIMIT) + index->search_info.n_hash_potential= n_hash_potential; - info->last_hash_succ = TRUE; + index->search_info.last_hash_succ= true; #ifdef UNIV_SEARCH_PERF_STAT - btr_search_n_succ++; + btr_search_n_succ++; #endif - return true; + return true; } -/** Drop any adaptive hash index entries that point to an index page. -@param[in,out] block block containing index page, s- or x-latched, or an - index page for which we know that - block->buf_fix_count == 0 or it is an index page which - has already been removed from the buf_pool.page_hash - i.e.: it is in state BUF_BLOCK_REMOVE_HASH -@param[in] garbage_collect drop ahi only if the index is marked - as freed */ -void btr_search_drop_page_hash_index(buf_block_t* block, - bool garbage_collect) +void btr_search_drop_page_hash_index(buf_block_t *block, + bool garbage_collect) noexcept { ulint n_fields; ulint n_bytes; @@ -1215,43 +1300,46 @@ void btr_search_drop_page_hash_index(buf_block_t* block, /* We must not dereference block->index here, because it could be freed if (!index->table->get_ref_count() && !dict_sys.frozen()). Determine the ahi_slot based on the block contents. */ + const index_id_t index_id= btr_page_get_index_id(block->page.frame); + btr_sea::partition &part= btr_search.get_part(index_id); - const index_id_t index_id - = btr_page_get_index_id(block->page.frame); + part.latch.rd_lock(SRW_LOCK_CALL); - auto part = btr_search_sys.get_part(index_id, - block->page.id().space()); + dict_index_t* index= block->index; - part->latch.rd_lock(SRW_LOCK_CALL); + if (!index || !btr_search.enabled) { +unlock_and_return: + part.latch.rd_unlock(); + return; + } - dict_index_t* index = block->index; - bool is_freed = index && index->freed(); + const bool is_freed= index->freed(); if (is_freed) { - part->latch.rd_unlock(); - part->latch.wr_lock(SRW_LOCK_CALL); + part.latch.rd_unlock(); + part.latch.wr_lock(SRW_LOCK_CALL); if (index != block->index) { - part->latch.wr_unlock(); + part.latch.wr_unlock(); goto retry; } - } else if (garbage_collect) { - part->latch.rd_unlock(); - return; + } + else if (garbage_collect) { + goto unlock_and_return; } assert_block_ahi_valid(block); - if (!index || !btr_search_enabled) { + if (!index || !btr_search.enabled) { if (is_freed) { - part->latch.wr_unlock(); + part.latch.wr_unlock(); } else { - part->latch.rd_unlock(); + part.latch.rd_unlock(); } return; } ut_ad(!index->table->is_temporary()); - ut_ad(btr_search_enabled); + ut_ad(btr_search.enabled); ut_ad(block->page.id().space() == index->table->space_id); ut_a(index_id == index->id); @@ -1260,11 +1348,8 @@ void btr_search_drop_page_hash_index(buf_block_t* block, n_fields = block->curr_n_fields; n_bytes = block->curr_n_bytes; - /* NOTE: The AHI fields of block must not be accessed after - releasing search latch, as the index page might only be s-latched! */ - if (!is_freed) { - part->latch.rd_unlock(); + part.latch.rd_unlock(); } ut_a(n_fields > 0 || n_bytes > 0); @@ -1353,7 +1438,7 @@ void btr_search_drop_page_hash_index(buf_block_t* block, all_deleted: if (!is_freed) { - part->latch.wr_lock(SRW_LOCK_CALL); + part.latch.wr_lock(SRW_LOCK_CALL); if (UNIV_UNLIKELY(!block->index)) { /* Someone else has meanwhile dropped the @@ -1370,18 +1455,17 @@ void btr_search_drop_page_hash_index(buf_block_t* block, /* Someone else has meanwhile built a new hash index on the page, with different parameters */ - part->latch.wr_unlock(); + part.latch.wr_unlock(); ut_free(folds); goto retry; } for (ulint i = 0; i < n_cached; i++) { - ha_remove_all_nodes_to_page(&part->table, part->heap, - folds[i], page); + ha_remove_all_nodes_to_page(part, folds[i], page); } - switch (index->search_info->ref_count--) { + switch (index->search_info.ref_count--) { case 0: ut_error; case 1: @@ -1397,15 +1481,11 @@ void btr_search_drop_page_hash_index(buf_block_t* block, cleanup: assert_block_ahi_valid(block); - part->latch.wr_unlock(); - + part.latch.wr_unlock(); ut_free(folds); } -/** Drop possible adaptive hash index entries when a page is evicted -from the buffer pool or freed in a file, or the index is being dropped. -@param[in] page_id page id */ -void btr_search_drop_page_hash_when_freed(const page_id_t page_id) +void btr_search_drop_page_hash_when_freed(const page_id_t page_id) noexcept { buf_block_t* block; mtr_t mtr; @@ -1439,7 +1519,7 @@ If index is non-NULL, this function checks if n_fields and n_bytes are sensible, and does not build a hash index if not. @param[in,out] index index for which to build. @param[in,out] block index page, s-/x- latched. -@param[in,out] ahi_latch the adaptive search latch +@param[in,out] part the adaptive search partition @param[in] n_fields hash this many full fields @param[in] n_bytes hash this many bytes of the next field @param[in] left_side hash for searches from left side */ @@ -1448,10 +1528,10 @@ void btr_search_build_page_hash_index( dict_index_t* index, buf_block_t* block, - srw_spin_lock* ahi_latch, + btr_sea::partition& part, uint16_t n_fields, uint16_t n_bytes, - bool left_side) + bool left_side) noexcept { const rec_t* rec; ulint fold; @@ -1466,12 +1546,12 @@ btr_search_build_page_hash_index( ut_ad(!index->table->is_temporary()); - if (!btr_search_enabled) { + if (!btr_search.enabled) { return; } rec_offs_init(offsets_); - ut_ad(ahi_latch == &btr_search_sys.get_part(*index)->latch); + ut_ad(&part == &btr_search.get_part(*index)); ut_ad(index); ut_ad(block->page.id().space() == index->table->space_id); ut_ad(!dict_index_is_ibuf(index)); @@ -1480,15 +1560,15 @@ btr_search_build_page_hash_index( ut_ad(block->page.lock.have_x() || block->page.lock.have_s()); ut_ad(block->page.id().page_no() >= 3); - ahi_latch->rd_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); - const bool enabled = btr_search_enabled; - const bool rebuild = enabled && block->index + const bool enabled = btr_search.enabled; + const bool rebuild = block->index && (block->curr_n_fields != n_fields || block->curr_n_bytes != n_bytes || block->curr_left_side != left_side); - ahi_latch->rd_unlock(); + part.latch.rd_unlock(); if (!enabled) { return; @@ -1591,11 +1671,11 @@ btr_search_build_page_hash_index( fold = next_fold; } - btr_search_check_free_space_in_heap(index); + part.prepare_insert(); - ahi_latch->wr_lock(SRW_LOCK_CALL); + part.latch.wr_lock(SRW_LOCK_CALL); - if (!btr_search_enabled) { + if (!btr_search.enabled) { goto exit_func; } @@ -1606,7 +1686,7 @@ btr_search_build_page_hash_index( case. */ if (!block->index) { assert_block_ahi_empty(block); - index->search_info->ref_count++; + index->search_info.ref_count++; } else if (block->curr_n_fields != n_fields || block->curr_n_bytes != n_bytes || block->curr_left_side != left_side) { @@ -1620,20 +1700,15 @@ btr_search_build_page_hash_index( block->curr_left_side = left_side; block->index = index; - { - auto part = btr_search_sys.get_part(*index); - for (ulint i = 0; i < n_cached; i++) { - ha_insert_for_fold(&part->table, part->heap, - folds[i], block, recs[i]); - } + for (ulint i = 0; i < n_cached; i++) { + ha_insert_for_fold(part, folds[i], block, recs[i]); } MONITOR_INC(MONITOR_ADAPTIVE_HASH_PAGE_ADDED); MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_ADDED, n_cached); exit_func: assert_block_ahi_valid(block); - ahi_latch->wr_unlock(); - + part.latch.wr_unlock(); ut_free(folds); ut_free(recs); if (UNIV_LIKELY_NULL(heap)) { @@ -1641,66 +1716,45 @@ btr_search_build_page_hash_index( } } -/** Updates the search info. -@param[in,out] info search info -@param[in,out] cursor cursor which was just positioned */ -void btr_search_info_update_slow(btr_search_t *info, btr_cur_t *cursor) +void btr_cur_t::search_info_update() const noexcept { - srw_spin_lock* ahi_latch = &btr_search_sys.get_part(*cursor->index()) - ->latch; - buf_block_t* block = btr_cur_get_block(cursor); - /* NOTE that the following two function calls do NOT protect info or block->n_fields etc. with any semaphore, to save CPU time! We cannot assume the fields are consistent when we return from those functions! */ - btr_search_info_update_hash(info, cursor); - - bool build_index = btr_search_update_block_hash_info(info, block); + btr_search_info_update_hash(this); - if (build_index || (cursor->flag == BTR_CUR_HASH_FAIL)) { - - btr_search_check_free_space_in_heap(cursor->index()); - } + bool build_index = btr_search_update_block_hash_info( + &index()->search_info, page_cur.block); - if (cursor->flag == BTR_CUR_HASH_FAIL) { + if (flag == BTR_CUR_HASH_FAIL) { /* Update the hash node reference, if appropriate */ - #ifdef UNIV_SEARCH_PERF_STAT btr_search_n_hash_fail++; #endif /* UNIV_SEARCH_PERF_STAT */ - - btr_search_update_hash_ref(info, block, cursor); + btr_search_update_hash_ref(this); } if (build_index) { /* Note that since we did not protect block->n_fields etc. with any semaphore, the values can be inconsistent. We have to check inside the function call that they make sense. */ - btr_search_build_page_hash_index(cursor->index(), block, - ahi_latch, - block->n_fields, - block->n_bytes, - block->left_side); + btr_search_build_page_hash_index(index(), page_cur.block, + btr_search.get_part(*index()), + page_cur.block->n_fields, + page_cur.block->n_bytes, + page_cur.block->left_side); } } -/** Move or delete hash entries for moved records, usually in a page split. -If new_block is already hashed, then any hash index for block is dropped. -If new_block is not hashed, and block is hashed, then a new hash index is -built to new_block with the same parameters as block. -@param[in,out] new_block destination page -@param[in,out] block source page (subject to deletion later) */ -void -btr_search_move_or_delete_hash_entries( - buf_block_t* new_block, - buf_block_t* block) +void btr_search_move_or_delete_hash_entries(buf_block_t *new_block, + buf_block_t *block) noexcept { ut_ad(block->page.lock.have_x()); ut_ad(new_block->page.lock.have_x()); - if (!btr_search_enabled) { + if (!btr_search.enabled) { return; } @@ -1713,10 +1767,6 @@ btr_search_move_or_delete_hash_entries( assert_block_ahi_valid(block); assert_block_ahi_valid(new_block); - srw_spin_lock* ahi_latch = index - ? &btr_search_sys.get_part(*index)->latch - : nullptr; - if (new_block->index) { drop_exit: btr_search_drop_page_hash_index(block, false); @@ -1727,10 +1777,11 @@ btr_search_move_or_delete_hash_entries( return; } - ahi_latch->rd_lock(SRW_LOCK_CALL); + btr_sea::partition& part = btr_search.get_part(*index); + part.latch.rd_lock(SRW_LOCK_CALL); if (index->freed()) { - ahi_latch->rd_unlock(); + part.latch.rd_unlock(); goto drop_exit; } @@ -1743,12 +1794,12 @@ btr_search_move_or_delete_hash_entries( new_block->n_bytes = block->curr_n_bytes; new_block->left_side = left_side; - ahi_latch->rd_unlock(); + part.latch.rd_unlock(); ut_a(n_fields > 0 || n_bytes > 0); btr_search_build_page_hash_index( - index, new_block, ahi_latch, + index, new_block, part, n_fields, n_bytes, left_side); ut_ad(n_fields == block->curr_n_fields); ut_ad(n_bytes == block->curr_n_bytes); @@ -1756,13 +1807,10 @@ btr_search_move_or_delete_hash_entries( return; } - ahi_latch->rd_unlock(); + part.latch.rd_unlock(); } -/** Updates the page hash index when a single record is deleted from a page. -@param[in] cursor cursor which was positioned on the record to delete - using btr_cur_search_, the record is not yet deleted.*/ -void btr_search_update_hash_on_delete(btr_cur_t *cursor) +void btr_search_update_hash_on_delete(btr_cur_t *cursor) noexcept { buf_block_t* block; const rec_t* rec; @@ -1774,7 +1822,7 @@ void btr_search_update_hash_on_delete(btr_cur_t *cursor) ut_ad(page_is_leaf(btr_cur_get_page(cursor))); - if (!btr_search_enabled) { + if (!btr_search.enabled) { return; } @@ -1812,45 +1860,31 @@ void btr_search_update_hash_on_delete(btr_cur_t *cursor) mem_heap_free(heap); } - auto part = btr_search_sys.get_part(*index); + btr_sea::partition& part = btr_search.get_part(*index); - part->latch.wr_lock(SRW_LOCK_CALL); + part.latch.wr_lock(SRW_LOCK_CALL); assert_block_ahi_valid(block); - if (block->index && btr_search_enabled) { + if (block->index) { + ut_ad(btr_search.enabled); ut_a(block->index == index); - if (ha_search_and_delete_if_found(&part->table, part->heap, - fold, rec)) { + if (part.erase(fold, rec)) { MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_REMOVED); } else { MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_REMOVE_NOT_FOUND); } - - assert_block_ahi_valid(block); + } else { + part.latch.wr_unlock(); } - - part->latch.wr_unlock(); } -/** Updates the page hash index when a single record is inserted on a page. -@param[in] cursor cursor which was positioned to the place to insert - using btr_cur_search_, and the new record has been - inserted next to the cursor. -@param[in] ahi_latch the adaptive hash index latch */ -void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, - srw_spin_lock *ahi_latch) +void btr_search_update_hash_node_on_insert(btr_cur_t *cursor) noexcept { buf_block_t* block; dict_index_t* index; rec_t* rec; - ut_ad(ahi_latch == &btr_search_sys.get_part(*cursor->index())->latch); - - if (!btr_search_enabled) { - return; - } - rec = btr_cur_get_rec(cursor); block = btr_cur_get_block(cursor); @@ -1860,10 +1894,10 @@ void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, index = block->index; if (!index) { - return; } + ut_ad(btr_search.enabled); ut_ad(!cursor->index()->table->is_temporary()); if (index != cursor->index()) { @@ -1872,11 +1906,11 @@ void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, return; } - ut_a(cursor->index() == index); ut_ad(!dict_index_is_ibuf(index)); - ahi_latch->wr_lock(SRW_LOCK_CALL); + btr_sea::partition& part = btr_search.get_part(*index); + part.latch.wr_lock(SRW_LOCK_CALL); - if (!block->index || !btr_search_enabled) { + if (!block->index || !btr_search.enabled) { goto func_exit; } @@ -1889,8 +1923,7 @@ void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, && !block->curr_left_side) { if (const rec_t *new_rec = page_rec_get_next_const(rec)) { if (ha_search_and_update_if_found( - &btr_search_sys.get_part(*cursor->index()) - ->table, + &part.table, cursor->fold, rec, block, new_rec)) { MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_UPDATED); } @@ -1900,22 +1933,14 @@ void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, func_exit: assert_block_ahi_valid(block); - ahi_latch->wr_unlock(); + part.latch.wr_unlock(); } else { - ahi_latch->wr_unlock(); - - btr_search_update_hash_on_insert(cursor, ahi_latch); + part.latch.wr_unlock(); + btr_search_update_hash_on_insert(cursor); } } -/** Updates the page hash index when a single record is inserted on a page. -@param[in,out] cursor cursor which was positioned to the - place to insert using btr_cur_search_..., - and the new record has been inserted next - to the cursor -@param[in] ahi_latch the adaptive hash index latch */ -void btr_search_update_hash_on_insert(btr_cur_t *cursor, - srw_spin_lock *ahi_latch) +void btr_search_update_hash_on_insert(btr_cur_t *cursor) noexcept { buf_block_t* block; dict_index_t* index; @@ -1932,10 +1957,9 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, rec_offs* offsets = offsets_; rec_offs_init(offsets_); - ut_ad(ahi_latch == &btr_search_sys.get_part(*cursor->index())->latch); ut_ad(page_is_leaf(btr_cur_get_page(cursor))); - if (!btr_search_enabled) { + if (!btr_search.enabled) { return; } @@ -1952,7 +1976,6 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, } ut_ad(block->page.id().space() == index->table->space_id); - btr_search_check_free_space_in_heap(index); rec = btr_cur_get_rec(cursor); @@ -1990,9 +2013,9 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, n_bytes, index->id); } - /* We must not look up "part" before acquiring ahi_latch. */ - btr_search_sys_t::partition* part= nullptr; + btr_sea::partition &part= btr_search.get_part(*index); bool locked = false; + part.prepare_insert(); if (!page_rec_is_infimum(rec) && !rec_is_metadata(rec, *index)) { offsets = rec_get_offsets( @@ -2002,15 +2025,13 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, } else { if (left_side) { locked = true; - ahi_latch->wr_lock(SRW_LOCK_CALL); + part.latch.wr_lock(SRW_LOCK_CALL); - if (!btr_search_enabled || !block->index) { + if (!block->index) { goto function_exit; } - part = btr_search_sys.get_part(*index); - ha_insert_for_fold(&part->table, part->heap, - ins_fold, block, ins_rec); + ha_insert_for_fold(part, ins_fold, block, ins_rec); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } @@ -2021,21 +2042,17 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, if (!locked) { locked = true; - ahi_latch->wr_lock(SRW_LOCK_CALL); + part.latch.wr_lock(SRW_LOCK_CALL); - if (!btr_search_enabled || !block->index) { + if (!block->index) { goto function_exit; } - - part = btr_search_sys.get_part(*index); } if (!left_side) { - ha_insert_for_fold(&part->table, part->heap, - fold, block, rec); + ha_insert_for_fold(part, fold, block, rec); } else { - ha_insert_for_fold(&part->table, part->heap, - ins_fold, block, ins_rec); + ha_insert_for_fold(part, ins_fold, block, ins_rec); } MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } @@ -2046,17 +2063,14 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, if (!left_side) { if (!locked) { locked = true; - ahi_latch->wr_lock(SRW_LOCK_CALL); + part.latch.wr_lock(SRW_LOCK_CALL); - if (!btr_search_enabled || !block->index) { + if (!block->index) { goto function_exit; } - - part = btr_search_sys.get_part(*index); } - ha_insert_for_fold(&part->table, part->heap, - ins_fold, block, ins_rec); + ha_insert_for_fold(part, ins_fold, block, ins_rec); MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } @@ -2066,21 +2080,17 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, if (ins_fold != next_fold) { if (!locked) { locked = true; - ahi_latch->wr_lock(SRW_LOCK_CALL); + part.latch.wr_lock(SRW_LOCK_CALL); - if (!btr_search_enabled || !block->index) { + if (!block->index) { goto function_exit; } - - part = btr_search_sys.get_part(*index); } if (!left_side) { - ha_insert_for_fold(&part->table, part->heap, - ins_fold, block, ins_rec); + ha_insert_for_fold(part, ins_fold, block, ins_rec); } else { - ha_insert_for_fold(&part->table, part->heap, - next_fold, block, next_rec); + ha_insert_for_fold(part, next_fold, block, next_rec); } MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); } @@ -2090,11 +2100,11 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, mem_heap_free(heap); } if (locked) { - ahi_latch->wr_unlock(); + part.latch.wr_unlock(); } } -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG +# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG __attribute__((nonnull)) /** @return whether a range of the cells is valid */ static bool ha_validate(const hash_table_t *table, @@ -2107,7 +2117,7 @@ static bool ha_validate(const hash_table_t *table, for (ulint i= start_index; i <= end_index; i++) { - for (auto node= static_cast(table->array[i].node); node; + for (auto node= static_cast(table->array[i].node); node; node= node->next) { if (table->calc_hash(node->fold) != i) { @@ -2121,13 +2131,28 @@ static bool ha_validate(const hash_table_t *table, return ok; } +/** Lock all search latches in exclusive mode. */ +static void btr_search_x_lock_all() noexcept +{ + for (ulong i= 0; i < btr_search.n_parts; i++) + btr_search.parts[i].latch.wr_lock(SRW_LOCK_CALL); +} + +/** Unlock all search latches from exclusive mode. */ +static void btr_search_x_unlock_all() noexcept +{ + for (ulong i= 0; i < btr_search.n_parts; i++) + btr_search.parts[i].latch.wr_unlock(); +} + /** Validates the search system for given hash table. @param thd connection, for checking if CHECK TABLE has been killed @param hash_table_id hash table to validate @return true if ok */ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) + noexcept { - ha_node_t* node; + ahi_node* node; bool ok = true; ulint i; ulint cell_count; @@ -2136,7 +2161,7 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) rec_offs* offsets = offsets_; btr_search_x_lock_all(); - if (!btr_search_enabled || (thd && thd_kill_level(thd))) { + if (!btr_search.enabled || (thd && thd_kill_level(thd))) { func_exit: btr_search_x_unlock_all(); @@ -2155,7 +2180,7 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) mysql_mutex_lock(&buf_pool.mutex); - auto &part = btr_search_sys.parts[hash_table_id]; + btr_sea::partition& part = btr_search.parts[hash_table_id]; cell_count = part.table.n_cells; @@ -2171,7 +2196,7 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) btr_search_x_lock_all(); - if (!btr_search_enabled + if (!btr_search.enabled || (thd && thd_kill_level(thd))) { goto func_exit; } @@ -2190,11 +2215,11 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) } } - node = static_cast(part.table.array[i].node); + node = static_cast(part.table.array[i].node); for (; node != NULL; node = node->next) { const buf_block_t* block - = buf_pool.block_from_ahi((byte*) node->data); + = buf_pool.block_from_ahi(node->rec); index_id_t page_index_id; if (UNIV_LIKELY(block->page.in_file())) { @@ -2229,14 +2254,14 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) page_index_id = btr_page_get_index_id(page); offsets = rec_get_offsets( - node->data, block->index, offsets, + node->rec, block->index, offsets, block->index->n_core_fields, btr_search_get_n_fields(block->curr_n_fields, block->curr_n_bytes), &heap); const ulint fold = rec_fold( - node->data, offsets, + node->rec, offsets, block->curr_n_fields, block->curr_n_bytes, page_index_id); @@ -2249,13 +2274,13 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) << block->page.id() << ", ptr mem address " << reinterpret_cast( - node->data) + node->rec) << ", index id " << page_index_id << ", node fold " << node->fold << ", rec fold " << fold; fputs("InnoDB: Record ", stderr); - rec_print_new(stderr, node->data, offsets); + rec_print_new(stderr, node->rec, offsets); fprintf(stderr, "\nInnoDB: on that page." " Page mem address %p, is hashed %p," " n fields %lu\n" @@ -2279,7 +2304,7 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) btr_search_x_lock_all(); - if (!btr_search_enabled + if (!btr_search.enabled || (thd && thd_kill_level(thd))) { goto func_exit; } @@ -2312,28 +2337,28 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) /** Validates the search system. @param thd connection, for checking if CHECK TABLE has been killed @return true if ok */ -bool btr_search_validate(THD *thd) +bool btr_search_validate(THD *thd) noexcept { - for (ulint i= 0; i < btr_ahi_parts; ++i) + for (ulint i= 0; i < btr_search.n_parts; ++i) if (!btr_search_hash_table_validate(thd, i)) - return(false); + return false; return true; } +# endif /* defined UNIV_AHI_DEBUG || defined UNIV_DEBUG */ -#ifdef UNIV_DEBUG -bool btr_search_check_marked_free_index(const buf_block_t *block) +# ifdef UNIV_DEBUG +bool btr_search_check_marked_free_index(const buf_block_t *block) noexcept { const index_id_t index_id= btr_page_get_index_id(block->page.frame); - auto part= btr_search_sys.get_part(index_id, block->page.id().space()); + btr_sea::partition &part= btr_search.get_part(index_id); - part->latch.rd_lock(SRW_LOCK_CALL); + part.latch.rd_lock(SRW_LOCK_CALL); bool is_freed= block->index && block->index->freed(); - part->latch.rd_unlock(); + part.latch.rd_unlock(); return is_freed; } -#endif /* UNIV_DEBUG */ -#endif /* defined UNIV_AHI_DEBUG || defined UNIV_DEBUG */ +# endif /* UNIV_DEBUG */ #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index f9388a11d2392..a30c7e3ff5bd6 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1593,11 +1593,9 @@ inline void buf_pool_t::resize() /* disable AHI if needed */ buf_resize_status("Disabling adaptive hash index."); - btr_search_s_lock_all(); - const bool btr_search_disabled = btr_search_enabled; - btr_search_s_unlock_all(); + const bool btr_search_disabled = btr_search.enabled; - btr_search_disable(); + btr_search.disable(); if (btr_search_disabled) { ib::info() << "disabled adaptive hash index."; @@ -1900,7 +1898,7 @@ inline void buf_pool_t::resize() #ifdef BTR_CUR_HASH_ADAPT /* enable AHI if needed */ if (btr_search_disabled) { - btr_search_enable(true); + btr_search.enable(true); ib::info() << "Re-enabled adaptive hash index."; } #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 2819943ebb913..fd67cf11c8afd 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -273,6 +273,10 @@ buf_block_t* buf_LRU_get_free_only() while (block != NULL) { ut_ad(block->page.in_free_list); ut_d(block->page.in_free_list = FALSE); +#ifdef BTR_CUR_HASH_ADAPT + ut_ad(!block->n_pointers); + ut_ad(!block->index); +#endif ut_ad(!block->page.oldest_modification()); ut_ad(!block->page.in_LRU_list); ut_a(!block->page.in_file()); @@ -282,10 +286,6 @@ buf_block_t* buf_LRU_get_free_only() || UT_LIST_GET_LEN(buf_pool.withdraw) >= buf_pool.withdraw_target || !buf_pool.will_be_withdrawn(block->page)) { - /* No adaptive hash index entries may point to - a free block. */ - assert_block_ahi_empty(block); - block->page.set_state(buf_page_t::MEMORY); MEM_MAKE_ADDRESSABLE(block->page.frame, srv_page_size); break; @@ -975,7 +975,10 @@ buf_LRU_block_free_non_file_page( void* data; ut_ad(block->page.state() == buf_page_t::MEMORY); +#ifdef BTR_CUR_HASH_ADAPT assert_block_ahi_empty(block); + block->n_hash_helps = 0; +#endif ut_ad(!block->page.in_free_list); ut_ad(!block->page.oldest_modification()); ut_ad(!block->page.in_LRU_list); diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 371c50e57c2b0..ef33ba0d93969 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -1282,7 +1282,7 @@ dict_create_index_step( } #ifdef BTR_CUR_HASH_ADAPT - ut_ad(!node->index->search_info->ref_count); + ut_ad(!node->index->search_info.ref_count); #endif /* BTR_CUR_HASH_ADAPT */ dict_index_remove_from_cache(table, node->index); node->index = NULL; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index d97e26f7fcac5..0bac83b925c91 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1210,7 +1210,7 @@ static bool dict_table_can_be_evicted(dict_table_t *table) for (const dict_index_t* index = dict_table_get_first_index(table); index; index = dict_table_get_next_index(index)) { - if (index->n_ahi_pages()) { + if (index->any_ahi_pages()) { return false; } } @@ -1237,9 +1237,6 @@ dict_index_t *dict_index_t::clone() const ut_ad(!rtr_track); const size_t size= sizeof *this + n_fields * sizeof(*fields) + -#ifdef BTR_CUR_ADAPT - sizeof *search_info + -#endif 1 + strlen(name) + n_uniq * (sizeof *stat_n_diff_key_vals + sizeof *stat_n_sample_sizes + @@ -1254,9 +1251,6 @@ dict_index_t *dict_index_t::clone() const index->name= mem_heap_strdup(heap, name); index->fields= static_cast (mem_heap_dup(heap, fields, n_fields * sizeof *fields)); -#ifdef BTR_CUR_ADAPT - index->search_info= btr_search_info_create(index->heap); -#endif /* BTR_CUR_ADAPT */ index->stat_n_diff_key_vals= static_cast (mem_heap_zalloc(heap, n_uniq * sizeof *stat_n_diff_key_vals)); index->stat_n_sample_sizes= static_cast @@ -1271,7 +1265,7 @@ dict_index_t *dict_index_t::clone() const @return this or a clone */ dict_index_t *dict_index_t::clone_if_needed() { - if (!search_info->ref_count) + if (!search_info.ref_count) return this; dict_index_t *prev= UT_LIST_GET_PREV(indexes, this); @@ -2028,9 +2022,6 @@ dict_index_add_to_cache( /* Add the new index as the last index for the table */ UT_LIST_ADD_LAST(new_index->table->indexes, new_index); -#ifdef BTR_CUR_ADAPT - new_index->search_info = btr_search_info_create(new_index->heap); -#endif /* BTR_CUR_ADAPT */ new_index->page = unsigned(page_no); new_index->lock.SRW_LOCK_INIT(index_tree_rw_lock_key); @@ -2096,7 +2087,7 @@ dict_index_remove_from_cache_low( only free the dict_index_t struct when this count drops to zero. See also: dict_table_can_be_evicted() */ - if (index->n_ahi_pages()) { + if (index->any_ahi_pages()) { table->autoinc_mutex.wr_lock(); index->set_freed(); UT_LIST_ADD_LAST(table->freed_indexes, index); diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index a4d169f923362..b6a6897191a04 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -583,8 +583,7 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, #ifndef BTR_CUR_ADAPT buf_block_t *guess= nullptr; #else - btr_search_t *const info= btr_search_get_info(index); - buf_block_t *guess= info->root_guess; + buf_block_t *&guess= index->search_info.root_guess; #endif /* Store the position of the tree latch we push to mtr so that we @@ -733,7 +732,7 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, rtr_get_mbr_from_tuple(tuple, &cur->rtr_info->mbr); #ifdef BTR_CUR_ADAPT - info->root_guess= block; + guess= block; #endif } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4fedd79790b11..b7de655a5dd93 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -17615,9 +17615,9 @@ innodb_adaptive_hash_index_update(THD*, st_mysql_sys_var*, void*, { mysql_mutex_unlock(&LOCK_global_system_variables); if (*(my_bool*) save) { - btr_search_enable(); + btr_search.enable(); } else { - btr_search_disable(); + btr_search.disable(); } mysql_mutex_lock(&LOCK_global_system_variables); } @@ -19145,18 +19145,15 @@ static MYSQL_SYSVAR_BOOL(stats_traditional, srv_stats_sample_traditional, NULL, NULL, TRUE); #ifdef BTR_CUR_HASH_ADAPT -static MYSQL_SYSVAR_BOOL(adaptive_hash_index, btr_search_enabled, +static MYSQL_SYSVAR_BOOL(adaptive_hash_index, *(my_bool*) &btr_search.enabled, PLUGIN_VAR_OPCMDARG, "Enable InnoDB adaptive hash index (disabled by default).", NULL, innodb_adaptive_hash_index_update, false); -/** Number of distinct partitions of AHI. -Each partition is protected by its own latch and so we have parts number -of latches protecting complete search system. */ -static MYSQL_SYSVAR_ULONG(adaptive_hash_index_parts, btr_ahi_parts, +static MYSQL_SYSVAR_ULONG(adaptive_hash_index_parts, btr_search.n_parts, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, "Number of InnoDB Adaptive Hash Index Partitions (default 8)", - NULL, NULL, 8, 1, 512, 0); + NULL, NULL, 8, 1, array_elements(btr_search.parts), 0); #endif /* BTR_CUR_HASH_ADAPT */ static MYSQL_SYSVAR_UINT(compression_level, page_zip_level, diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index ff42729737589..bd1382d7cdc62 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -5968,12 +5968,12 @@ static bool innobase_instant_try( #ifdef BTR_CUR_HASH_ADAPT /* Acquire the ahi latch to avoid a race condition between ahi access and instant alter table */ - srw_spin_lock* ahi_latch = btr_search_sys.get_latch(*index); - ahi_latch->wr_lock(SRW_LOCK_CALL); + btr_sea::partition& part = btr_search.get_part(*index); + part.latch.wr_lock(SRW_LOCK_CALL); #endif /* BTR_CUR_HASH_ADAPT */ const bool metadata_changed = ctx->instant_column(); #ifdef BTR_CUR_HASH_ADAPT - ahi_latch->wr_unlock(); + part.latch.wr_unlock(); #endif /* BTR_CUR_HASH_ADAPT */ DBUG_ASSERT(index->n_fields >= n_old_fields); diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 3e8f78889597b..86eb72aa50a0d 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -471,9 +471,6 @@ ibuf_init_at_db_start(void) ibuf.index->id = DICT_IBUF_ID_MIN + IBUF_SPACE_ID; ibuf.index->n_uniq = REC_MAX_N_FIELDS; ibuf.index->lock.SRW_LOCK_INIT(index_tree_rw_lock_key); -#ifdef BTR_CUR_ADAPT - ibuf.index->search_info = btr_search_info_create(ibuf.index->heap); -#endif /* BTR_CUR_ADAPT */ ibuf.index->page = FSP_IBUF_TREE_ROOT_PAGE_NO; ut_d(ibuf.index->cached = TRUE); diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 935be50543677..e3fd37c069e7c 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -33,9 +33,6 @@ Created 10/16/1994 Heikki Tuuri #include "rem0types.h" #include "gis0type.h" #include "my_base.h" -#ifdef BTR_CUR_HASH_ADAPT -# include "srw_lock.h" -#endif /** Mode flags for btr_cur operations; these can be ORed */ enum { @@ -770,6 +767,10 @@ struct btr_cur_t { @return error code */ inline dberr_t open_random_leaf(rec_offs *&offsets, mem_heap_t *& heap, mtr_t &mtr); + +#ifdef BTR_CUR_HASH_ADAPT + void search_info_update() const noexcept; +#endif }; /** Modify the delete-mark flag of a record. diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index b75cad1018019..1c18533e23984 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -24,43 +24,24 @@ The index tree adaptive search Created 2/17/1996 Heikki Tuuri *************************************************************************/ -#ifndef btr0sea_h -#define btr0sea_h +#pragma once #include "dict0dict.h" #ifdef BTR_CUR_HASH_ADAPT -#include "ha0ha.h" -#include "srw_lock.h" +#include "buf0buf.h" #ifdef UNIV_PFS_RWLOCK extern mysql_pfs_key_t btr_search_latch_key; #endif /* UNIV_PFS_RWLOCK */ -#define btr_search_sys_create() btr_search_sys.create() -#define btr_search_sys_free() btr_search_sys.free() - -/** Disable the adaptive hash search system and empty the index. */ -void btr_search_disable(); - -/** Enable the adaptive hash search system. -@param resize whether buf_pool_t::resize() is the caller */ -void btr_search_enable(bool resize= false); - -/*********************************************************************//** -Updates the search info. */ -UNIV_INLINE -void -btr_search_info_update( -/*===================*/ - dict_index_t* index, /*!< in: index of the cursor */ - btr_cur_t* cursor);/*!< in: cursor which was just positioned */ +#define btr_search_sys_create() btr_search.create() +#define btr_search_sys_free() btr_search.free() /** Tries to guess the right search position based on the hash search info of the index. Note that if mode is PAGE_CUR_LE, which is used in inserts, and the function returns TRUE, then cursor->up_match and cursor->low_match both have sensible values. @param[in,out] index index -@param[in,out] info index search info @param[in] tuple logical record @param[in] mode PAGE_CUR_L, .... @param[in] latch_mode BTR_SEARCH_LEAF, ... @@ -70,12 +51,11 @@ both have sensible values. bool btr_search_guess_on_hash( dict_index_t* index, - btr_search_t* info, const dtuple_t* tuple, ulint mode, ulint latch_mode, btr_cur_t* cursor, - mtr_t* mtr); + mtr_t* mtr) noexcept; /** Move or delete hash entries for moved records, usually in a page split. If new_block is already hashed, then any hash index for block is dropped. @@ -83,10 +63,8 @@ If new_block is not hashed, and block is hashed, then a new hash index is built to new_block with the same parameters as block. @param[in,out] new_block destination page @param[in,out] block source page (subject to deletion later) */ -void -btr_search_move_or_delete_hash_entries( - buf_block_t* new_block, - buf_block_t* block); +void btr_search_move_or_delete_hash_entries(buf_block_t *new_block, + buf_block_t *block) noexcept; /** Drop any adaptive hash index entries that point to an index page. @param[in,out] block block containing index page, s- or x-latched, or an @@ -97,307 +75,143 @@ btr_search_move_or_delete_hash_entries( @param[in] garbage_collect drop ahi only if the index is marked as freed */ void btr_search_drop_page_hash_index(buf_block_t* block, - bool garbage_collect); + bool garbage_collect) noexcept; /** Drop possible adaptive hash index entries when a page is evicted from the buffer pool or freed in a file, or the index is being dropped. @param[in] page_id page id */ -void btr_search_drop_page_hash_when_freed(const page_id_t page_id); - -/** Updates the page hash index when a single record is inserted on a page. -@param[in] cursor cursor which was positioned to the place to insert - using btr_cur_search_, and the new record has been - inserted next to the cursor. -@param[in] ahi_latch the adaptive hash index latch */ -void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, - srw_spin_lock *ahi_latch); - -/** Updates the page hash index when a single record is inserted on a page. -@param[in,out] cursor cursor which was positioned to the - place to insert using btr_cur_search_..., - and the new record has been inserted next - to the cursor -@param[in] ahi_latch the adaptive hash index latch */ -void btr_search_update_hash_on_insert(btr_cur_t *cursor, - srw_spin_lock *ahi_latch); - -/** Updates the page hash index when a single record is deleted from a page. -@param[in] cursor cursor which was positioned on the record to delete - using btr_cur_search_, the record is not yet deleted.*/ -void btr_search_update_hash_on_delete(btr_cur_t *cursor); - -/** Validates the search system. -@param thd connection, for checking if CHECK TABLE has been killed -@return true if ok */ -bool btr_search_validate(THD *thd); +void btr_search_drop_page_hash_when_freed(const page_id_t page_id) noexcept; -/** Lock all search latches in exclusive mode. */ -static inline void btr_search_x_lock_all(); +/** Update the page hash index after a single record is inserted on a page. +@param cursor cursor which was positioned before the inserted record */ +void btr_search_update_hash_node_on_insert(btr_cur_t *cursor) noexcept; -/** Unlock all search latches from exclusive mode. */ -static inline void btr_search_x_unlock_all(); +/** Update the page hash index after a single record is inserted on a page. +@param cursor cursor which was positioned before the inserted record */ +void btr_search_update_hash_on_insert(btr_cur_t *cursor) noexcept; -/** Lock all search latches in shared mode. */ -static inline void btr_search_s_lock_all(); +/** Updates the page hash index before a single record is deleted from a page. +@param cursor cursor positioned on the to-be-deleted record */ +void btr_search_update_hash_on_delete(btr_cur_t *cursor) noexcept; -/** Unlock all search latches from shared mode. */ -static inline void btr_search_s_unlock_all(); +/** Validates the search system. +@param thd connection, for checking if CHECK TABLE has been killed +@return true if ok */ +bool btr_search_validate(THD *thd) noexcept; # ifdef UNIV_DEBUG /** @return if the index is marked as freed */ -bool btr_search_check_marked_free_index(const buf_block_t *block); -# endif /* UNIV_DEBUG */ -#else /* BTR_CUR_HASH_ADAPT */ -# define btr_search_sys_create() -# define btr_search_sys_free() -# define btr_search_drop_page_hash_index(block, garbage_collect) -# define btr_search_s_lock_all(index) -# define btr_search_s_unlock_all(index) -# define btr_search_info_update(index, cursor) -# define btr_search_move_or_delete_hash_entries(new_block, block) -# define btr_search_update_hash_on_insert(cursor, ahi_latch) -# define btr_search_update_hash_on_delete(cursor) -# ifdef UNIV_DEBUG -# define btr_search_check_marked_free_index(block) +bool btr_search_check_marked_free_index(const buf_block_t *block) noexcept; # endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ -#ifdef BTR_CUR_ADAPT -/** Create and initialize search info. -@param[in,out] heap heap where created -@return own: search info struct */ -static inline btr_search_t* btr_search_info_create(mem_heap_t* heap) - MY_ATTRIBUTE((nonnull, warn_unused_result)); +struct ahi_node; -/** @return the search info of an index */ -static inline btr_search_t* btr_search_get_info(dict_index_t* index) -{ - return(index->search_info); -} -#endif /* BTR_CUR_ADAPT */ - -/** The search info struct in an index */ -struct btr_search_t{ - /* @{ The following fields are not protected by any latch. - Unfortunately, this means that they must be aligned to - the machine word, i.e., they cannot be turned into bit-fields. */ - buf_block_t* root_guess;/*!< the root page frame when it was last time - fetched, or NULL */ -#ifdef BTR_CUR_HASH_ADAPT - ulint hash_analysis; /*!< when this exceeds - BTR_SEARCH_HASH_ANALYSIS, the hash - analysis starts; this is reset if no - success noticed */ - ibool last_hash_succ; /*!< TRUE if the last search would have - succeeded, or did succeed, using the hash - index; NOTE that the value here is not exact: - it is not calculated for every search, and the - calculation itself is not always accurate! */ - ulint n_hash_potential; - /*!< number of consecutive searches - which would have succeeded, or did succeed, - using the hash index; - the range is 0 .. BTR_SEARCH_BUILD_LIMIT + 5 */ - /* @} */ - ulint ref_count; /*!< Number of blocks in this index tree - that have search index built - i.e. block->index points to this index. - Protected by search latch except - when during initialization in - btr_search_info_create(). */ - - /*---------------------- @{ */ - uint16_t n_fields; /*!< recommended prefix length for hash search: - number of full fields */ - uint16_t n_bytes; /*!< recommended prefix: number of bytes in - an incomplete field - @see BTR_PAGE_MAX_REC_SIZE */ - bool left_side; /*!< true or false, depending on whether - the leftmost record of several records with - the same prefix should be indexed in the - hash index */ - /*---------------------- @} */ -#ifdef UNIV_SEARCH_PERF_STAT - ulint n_hash_succ; /*!< number of successful hash searches thus - far */ - ulint n_hash_fail; /*!< number of failed hash searches */ - ulint n_patt_succ; /*!< number of successful pattern searches thus - far */ - ulint n_searches; /*!< number of searches */ -#endif /* UNIV_SEARCH_PERF_STAT */ -#endif /* BTR_CUR_HASH_ADAPT */ -#ifdef UNIV_DEBUG - ulint magic_n; /*!< magic number @see BTR_SEARCH_MAGIC_N */ -/** value of btr_search_t::magic_n, used in assertions */ -# define BTR_SEARCH_MAGIC_N 1112765 -#endif /* UNIV_DEBUG */ -}; - -#ifdef BTR_CUR_HASH_ADAPT /** The hash index system */ -struct btr_search_sys_t +struct btr_sea { + /** the actual value of innodb_adaptive_hash_index */ + Atomic_relaxed enabled; + + /** Disable the adaptive hash search system and empty the index. */ + void disable() noexcept; + + /** Enable the adaptive hash search system. + @param resize whether buf_pool_t::resize() is the caller */ + void enable(bool resize= false) noexcept; + /** Partition of the hash table */ struct partition { - /** latches protecting hash_table */ - srw_spin_lock latch; - /** mapping of dtuple_fold() to rec_t* in buf_block_t::frame */ + /** latch protecting the hash table */ + alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_spin_lock latch; + /** map of dtuple_fold() or rec_fold() to rec_t* in buf_page_t::frame */ hash_table_t table; - /** memory heap for table */ - mem_heap_t *heap; - -#ifdef _MSC_VER -#pragma warning(push) -// nonstandard extension - zero sized array, if perfschema is not compiled -#pragma warning(disable : 4200) -#endif - - char pad[(CPU_LEVEL1_DCACHE_LINESIZE - sizeof latch - - sizeof table - sizeof heap) & - (CPU_LEVEL1_DCACHE_LINESIZE - 1)]; - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - - void init() - { - memset((void*) this, 0, sizeof *this); - latch.SRW_LOCK_INIT(btr_search_latch_key); - } - - void alloc(ulint hash_size) - { - table.create(hash_size); - heap= mem_heap_create_typed(std::min(4096, - MEM_MAX_ALLOC_IN_BUF / 2 - - MEM_BLOCK_HEADER_SIZE - - MEM_SPACE_NEEDED(0)), - MEM_HEAP_FOR_BTR_SEARCH); - } - - void clear() - { - mem_heap_free(heap); - heap= nullptr; - ut_free(table.array); - } - - void free() - { - latch.destroy(); - if (heap) - clear(); - } + /** latch protecting blocks, spare; may be acquired while holding latch */ + srw_mutex blocks_mutex; + /** allocated blocks */ + UT_LIST_BASE_NODE_T(buf_page_t) blocks; + /** a cached block to extend blocks */ + Atomic_relaxed spare; + + inline void init() noexcept; + + inline void alloc(ulint hash_size) noexcept; + + inline void clear() noexcept; + + inline void free() noexcept; + + /** Ensure that there is a spare block for a future insert() */ + void prepare_insert() noexcept; + + /** Clean up after erasing an AHI node + @param erase node being erased + @return buffer block to be freed + @retval nullptr if no buffer block was freed */ + buf_block_t *cleanup_after_erase(ahi_node *erase) noexcept; + + __attribute__((nonnull)) +# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + /** Insert or replace an entry into the hash table. + @param fold hash value of rec + @param rec B-tree leaf page record + @param block the buffer block that contains rec */ + void insert(ulint fold, const rec_t *rec, buf_block_t *block) noexcept; +# else + /** Insert or replace an entry into the hash table. + @param fold hash value of data + @param rec B-tree leaf page record */ + void insert(ulint fold, const rec_t *rec) noexcept; +# endif + + /** Delete a pointer to a record if it exists. + @param fold hash value of rec + @param rec B-tree leaf page record + @return whether a record existed and was removed */ + inline bool erase(ulint fold, const rec_t *rec) noexcept; }; + /** innodb_adaptive_hash_index_parts */ + ulong n_parts; /** Partitions of the adaptive hash index */ - partition *parts; + partition parts[512]; /** Get an adaptive hash index partition */ - partition *get_part(index_id_t id, ulint space_id) const - { - return parts + ut_fold_ulint_pair(ulint(id), space_id) % btr_ahi_parts; - } + partition &get_part(index_id_t id) noexcept { return parts[id % n_parts]; } /** Get an adaptive hash index partition */ - partition *get_part(const dict_index_t &index) const - { - ut_ad(!index.table->space || - index.table->space->id == index.table->space_id); - return get_part(ulint(index.id), index.table->space_id); - } - - /** Get the search latch for the adaptive hash index partition */ - srw_spin_lock *get_latch(const dict_index_t &index) const - { return &get_part(index)->latch; } + partition &get_part(const dict_index_t &index) noexcept + { return get_part(index.id); } /** Create and initialize at startup */ - void create() - { - parts= static_cast(ut_malloc(btr_ahi_parts * sizeof *parts, - mem_key_ahi)); - for (ulong i= 0; i < btr_ahi_parts; ++i) - parts[i].init(); - if (btr_search_enabled) - btr_search_enable(); - } - - void alloc(ulint hash_size) - { - hash_size/= btr_ahi_parts; - for (ulong i= 0; i < btr_ahi_parts; ++i) - parts[i].alloc(hash_size); - } + void create() noexcept; + + void alloc(ulint hash_size) noexcept; /** Clear when disabling the adaptive hash index */ - void clear() { for (ulong i= 0; i < btr_ahi_parts; ++i) parts[i].clear(); } + inline void clear() noexcept; /** Free at shutdown */ - void free() - { - if (parts) - { - for (ulong i= 0; i < btr_ahi_parts; ++i) - parts[i].free(); - ut_free(parts); - parts= nullptr; - } - } + void free() noexcept; }; /** The adaptive hash index */ -extern btr_search_sys_t btr_search_sys; +extern btr_sea btr_search; -/** @return number of leaf pages pointed to by the adaptive hash index */ -TRANSACTIONAL_INLINE inline ulint dict_index_t::n_ahi_pages() const -{ - if (!btr_search_enabled) - return 0; - srw_spin_lock *latch= &btr_search_sys.get_part(*this)->latch; -#if !defined NO_ELISION && !defined SUX_LOCK_GENERIC - if (xbegin()) - { - if (latch->is_locked()) - xabort(); - ulint ref_count= search_info->ref_count; - xend(); - return ref_count; - } -#endif - latch->rd_lock(SRW_LOCK_CALL); - ulint ref_count= search_info->ref_count; - latch->rd_unlock(); - return ref_count; -} - -#ifdef UNIV_SEARCH_PERF_STAT +# ifdef UNIV_SEARCH_PERF_STAT /** Number of successful adaptive hash index lookups */ extern ulint btr_search_n_succ; /** Number of failed adaptive hash index lookups */ extern ulint btr_search_n_hash_fail; -#endif /* UNIV_SEARCH_PERF_STAT */ - -/** After change in n_fields or n_bytes in info, this many rounds are waited -before starting the hash analysis again: this is to save CPU time when there -is no hope in building a hash index. */ -#define BTR_SEARCH_HASH_ANALYSIS 17 - -/** Limit of consecutive searches for trying a search shortcut on the search -pattern */ -#define BTR_SEARCH_ON_PATTERN_LIMIT 3 - -/** Limit of consecutive searches for trying a search shortcut using -the hash index */ -#define BTR_SEARCH_ON_HASH_LIMIT 3 - -/** We do this many searches before trying to keep the search latch -over calls from MySQL. If we notice someone waiting for the latch, we -again set this much timeout. This is to reduce contention. */ -#define BTR_SEA_TIMEOUT 10000 +# endif /* UNIV_SEARCH_PERF_STAT */ +#else /* BTR_CUR_HASH_ADAPT */ +# define btr_search_sys_create() +# define btr_search_sys_free() +# define btr_search_drop_page_hash_index(block, garbage_collect) +# define btr_search_move_or_delete_hash_entries(new_block, block) +# define btr_search_update_hash_on_insert(cursor, ahi_latch) +# define btr_search_update_hash_on_delete(cursor) +# ifdef UNIV_DEBUG +# define btr_search_check_marked_free_index(block) +# endif /* UNIV_DEBUG */ #endif /* BTR_CUR_HASH_ADAPT */ - -#include "btr0sea.inl" - -#endif diff --git a/storage/innobase/include/btr0sea.inl b/storage/innobase/include/btr0sea.inl deleted file mode 100644 index 5a8d648029a97..0000000000000 --- a/storage/innobase/include/btr0sea.inl +++ /dev/null @@ -1,117 +0,0 @@ -/***************************************************************************** - -Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2018, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/********************************************************************//** -@file include/btr0sea.ic -The index tree adaptive search - -Created 2/17/1996 Heikki Tuuri -*************************************************************************/ - -#include "dict0mem.h" -#include "btr0cur.h" -#include "buf0buf.h" - -/** Create and initialize search info. -@param[in,out] heap heap where created -@return own: search info struct */ -static inline btr_search_t* btr_search_info_create(mem_heap_t* heap) -{ - btr_search_t* info = static_cast( - mem_heap_zalloc(heap, sizeof(btr_search_t))); - ut_d(info->magic_n = BTR_SEARCH_MAGIC_N); -#ifdef BTR_CUR_HASH_ADAPT - info->n_fields = 1; - info->left_side = TRUE; -#endif /* BTR_CUR_HASH_ADAPT */ - return(info); -} - -#ifdef BTR_CUR_HASH_ADAPT -/** Updates the search info. -@param[in,out] info search info -@param[in,out] cursor cursor which was just positioned */ -void btr_search_info_update_slow(btr_search_t *info, btr_cur_t *cursor); - -/*********************************************************************//** -Updates the search info. */ -static inline -void -btr_search_info_update( -/*===================*/ - dict_index_t* index, /*!< in: index of the cursor */ - btr_cur_t* cursor) /*!< in: cursor which was just positioned */ -{ - ut_ad(!index->is_spatial()); - ut_ad(!index->table->is_temporary()); - - if (!btr_search_enabled) { - return; - } - - btr_search_t* info; - info = btr_search_get_info(index); - - info->hash_analysis++; - - if (info->hash_analysis < BTR_SEARCH_HASH_ANALYSIS) { - - /* Do nothing */ - - return; - - } - - ut_ad(cursor->flag != BTR_CUR_HASH); - - btr_search_info_update_slow(info, cursor); -} - -/** Lock all search latches in exclusive mode. */ -static inline void btr_search_x_lock_all() -{ - for (ulint i = 0; i < btr_ahi_parts; ++i) { - btr_search_sys.parts[i].latch.wr_lock(SRW_LOCK_CALL); - } -} - -/** Unlock all search latches from exclusive mode. */ -static inline void btr_search_x_unlock_all() -{ - for (ulint i = 0; i < btr_ahi_parts; ++i) { - btr_search_sys.parts[i].latch.wr_unlock(); - } -} - -/** Lock all search latches in shared mode. */ -static inline void btr_search_s_lock_all() -{ - for (ulint i = 0; i < btr_ahi_parts; ++i) { - btr_search_sys.parts[i].latch.rd_lock(SRW_LOCK_CALL); - } -} - -/** Unlock all search latches from shared mode. */ -static inline void btr_search_s_unlock_all() -{ - for (ulint i = 0; i < btr_ahi_parts; ++i) { - btr_search_sys.parts[i].latch.rd_unlock(); - } -} -#endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/include/btr0types.h b/storage/innobase/include/btr0types.h index fc829e7857a37..a9b68feeb35d1 100644 --- a/storage/innobase/include/btr0types.h +++ b/storage/innobase/include/btr0types.h @@ -33,17 +33,6 @@ Created 2/17/1996 Heikki Tuuri struct btr_pcur_t; /** B-tree cursor */ struct btr_cur_t; -/** B-tree search information for the adaptive hash index */ -struct btr_search_t; - -#ifdef BTR_CUR_HASH_ADAPT -/** Is search system enabled. -Search system is protected by array of latches. */ -extern char btr_search_enabled; - -/** Number of adaptive hash index partition. */ -extern ulong btr_ahi_parts; -#endif /* BTR_CUR_HASH_ADAPT */ /** The size of a reference to data stored on a different page. The reference is stored at the end of the prefix of the field diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index eed4111365920..2d29f00ba7f9d 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -949,12 +949,12 @@ struct buf_block_t{ Another exception is that ha_insert_for_fold() may decrement n_pointers without holding the appropriate latch - in btr_search_latches[]. Thus, n_pointers must be + in btr_search.parts. Thus, n_pointers must be protected by atomic memory access. This implies that the fields may be read without race condition whenever any of the following hold: - - the btr_search_sys.partition[].latch is being held, or + - the btr_search.parts.latch is being held, or - state() == NOT_USED || state() == MEMORY, and holding some latch prevents the state from changing to that. @@ -980,7 +980,6 @@ struct buf_block_t{ ut_a((block)->index || (block)->n_pointers == 0) # else /* UNIV_AHI_DEBUG || UNIV_DEBUG */ # define assert_block_ahi_empty(block) /* nothing */ -# define assert_block_ahi_empty_on_init(block) /* nothing */ # define assert_block_ahi_valid(block) /* nothing */ # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ unsigned curr_n_fields:10;/*!< prefix length for hash indexing: @@ -1000,7 +999,6 @@ struct buf_block_t{ /* @} */ #else /* BTR_CUR_HASH_ADAPT */ # define assert_block_ahi_empty(block) /* nothing */ -# define assert_block_ahi_empty_on_init(block) /* nothing */ # define assert_block_ahi_valid(block) /* nothing */ #endif /* BTR_CUR_HASH_ADAPT */ void fix() noexcept { page.fix(); } diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 609aaf951cabd..7c3c532577d12 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1084,8 +1084,66 @@ struct dict_index_t { UT_LIST_NODE_T(dict_index_t) indexes;/*!< list of indexes of the table */ #ifdef BTR_CUR_ADAPT - btr_search_t* search_info; - /*!< info used in optimistic searches */ + /** The search info struct in an index */ + struct ahi { + ahi()= default; + ahi(const ahi&)= default; + ~ahi()= default; + /** Dummy assignment operator for dict_index_t::clone(), which + will return a clone where these fields are reset to default values + (because no AHI entries exist yet for the clone) */ + ahi &operator=(const ahi&) { new(this) ahi(); return *this; } + /** the root page when it was last time fetched, or nullptr */ + buf_block_t *root_guess= nullptr; +# ifdef BTR_CUR_HASH_ADAPT + private: + /** After change in n_fields or n_bytes, this many rounds are + waited before starting the hash analysis again: this is to save + CPU time when there is no hope in building a hash index. */ + static constexpr uint8_t HASH_ANALYSIS= 16; + /** the number of calls to hash_analysis_useful() */ + Atomic_relaxed hash_analysis{0}; + public: + bool hash_analysis_useful() noexcept + { + return hash_analysis > HASH_ANALYSIS || + hash_analysis.fetch_add(1) >= HASH_ANALYSIS; + } + void hash_analysis_reset() noexcept { hash_analysis= 0; } + + /** number of consecutive searches which would have succeeded, or + did succeed, using the hash index; the range is 0 + .. BTR_SEARCH_BUILD_LIMIT + 5 */ + Atomic_relaxed n_hash_potential{0}; + + /** whether the last search would have succeeded, or + did succeed, using the hash index; NOTE that the value + here is not exact: it is not calculated for every + search, and the calculation itself is not always accurate! */ + Atomic_relaxed last_hash_succ{false}; + + /** recommended full field prefix */ + uint16_t n_fields= 1; + /** recommended number of bytes in an incomplete field */ + uint16_t n_bytes= 0; + + /** whether the leftmost record of several records with the same + prefix should be indexed */ + bool left_side= true; + + /** number of buf_block_t::index pointers to this index */ + Atomic_counter ref_count{0}; + +# ifdef UNIV_SEARCH_PERF_STAT + /** number of successful hash searches */ + size_t n_hash_succ{0}; + /** number of failed hash searches */ + size_t n_hash_fail{0}; + /** number of searches */ + size_t n_searches{0}; +# endif /* UNIV_SEARCH_PERF_STAT */ +# endif /* BTR_CUR_HASH_ADAPT */ + } search_info; #endif /* BTR_CUR_ADAPT */ row_log_t* online_log; /*!< the log of modifications @@ -1376,8 +1434,8 @@ struct dict_index_t { /** Clone this index for lazy dropping of the adaptive hash index. @return this or a clone */ dict_index_t* clone_if_needed(); - /** @return number of leaf pages pointed to by the adaptive hash index */ - inline ulint n_ahi_pages() const; + /** @return whether any leaf pages may be in the adaptive hash index */ + bool any_ahi_pages() const noexcept { return search_info.ref_count; } /** @return whether mark_freed() had been invoked */ bool freed() const { return UNIV_UNLIKELY(page == 1); } /** Note that the index is waiting for btr_search_lazy_free() */ diff --git a/storage/innobase/include/ha0ha.h b/storage/innobase/include/ha0ha.h deleted file mode 100644 index 5aaa559b88568..0000000000000 --- a/storage/innobase/include/ha0ha.h +++ /dev/null @@ -1,60 +0,0 @@ -/***************************************************************************** - -Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2018, 2020, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/ha0ha.h -The hash table interface for the adaptive hash index - -Created 8/18/1994 Heikki Tuuri -*******************************************************/ - -#ifndef ha0ha_h -#define ha0ha_h - -#include "hash0hash.h" -#include "page0types.h" -#include "buf0types.h" -#include "rem0types.h" - -#ifdef BTR_CUR_HASH_ADAPT -/*************************************************************//** -Looks for an element in a hash table. -@return pointer to the data of the first hash table node in chain -having the fold number, NULL if not found */ -UNIV_INLINE -const rec_t* -ha_search_and_get_data( -/*===================*/ - hash_table_t* table, /*!< in: hash table */ - ulint fold); /*!< in: folded value of the searched data */ - -/** The hash table external chain node */ -struct ha_node_t { - ulint fold; /*!< fold value for the data */ - ha_node_t* next; /*!< next chain node or NULL if none */ -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - buf_block_t* block; /*!< buffer block containing the data, or NULL */ -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - const rec_t* data; /*!< pointer to the data */ -}; - -#include "ha0ha.inl" -#endif /* BTR_CUR_HASH_ADAPT */ - -#endif diff --git a/storage/innobase/include/ha0ha.inl b/storage/innobase/include/ha0ha.inl deleted file mode 100644 index 0b25625721498..0000000000000 --- a/storage/innobase/include/ha0ha.inl +++ /dev/null @@ -1,154 +0,0 @@ -/***************************************************************************** - -Copyright (c) 1994, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2018, 2020, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/********************************************************************//** -@file include/ha0ha.ic -The hash table interface for the adaptive hash index - -Created 8/18/1994 Heikki Tuuri -*************************************************************************/ - -#ifdef BTR_CUR_HASH_ADAPT -#include "btr0types.h" - -/******************************************************************//** -Gets a hash node data. -@return pointer to the data */ -UNIV_INLINE -const rec_t* -ha_node_get_data( -/*=============*/ - const ha_node_t* node) /*!< in: hash chain node */ -{ - return(node->data); -} - -/******************************************************************//** -Sets hash node data. */ -UNIV_INLINE -void -ha_node_set_data_func( -/*==================*/ - ha_node_t* node, /*!< in: hash chain node */ -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - buf_block_t* block, /*!< in: buffer block containing the data */ -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - const rec_t* data) /*!< in: pointer to the data */ -{ -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - node->block = block; -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - node->data = data; -} - -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG -/** Sets hash node data. -@param n in: hash chain node -@param b in: buffer block containing the data -@param d in: pointer to the data */ -# define ha_node_set_data(n,b,d) ha_node_set_data_func(n,b,d) -#else /* UNIV_AHI_DEBUG || UNIV_DEBUG */ -/** Sets hash node data. -@param n in: hash chain node -@param b in: buffer block containing the data -@param d in: pointer to the data */ -# define ha_node_set_data(n,b,d) ha_node_set_data_func(n,d) -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - -/******************************************************************//** -Gets the next node in a hash chain. -@return next node, NULL if none */ -UNIV_INLINE -ha_node_t* -ha_chain_get_next( -/*==============*/ - const ha_node_t* node) /*!< in: hash chain node */ -{ - return(node->next); -} - -/******************************************************************//** -Gets the first node in a hash chain. -@return first node, NULL if none */ -UNIV_INLINE -ha_node_t* -ha_chain_get_first( -/*===============*/ - hash_table_t* table, /*!< in: hash table */ - ulint fold) /*!< in: fold value determining the chain */ -{ - return static_cast(table->array[table->calc_hash(fold)].node); -} - -/*************************************************************//** -Looks for an element in a hash table. -@return pointer to the data of the first hash table node in chain -having the fold number, NULL if not found */ -UNIV_INLINE -const rec_t* -ha_search_and_get_data( -/*===================*/ - hash_table_t* table, /*!< in: hash table */ - ulint fold) /*!< in: folded value of the searched data */ -{ - ut_ad(btr_search_enabled); - - for (const ha_node_t* node = ha_chain_get_first(table, fold); - node != NULL; - node = ha_chain_get_next(node)) { - - if (node->fold == fold) { - - return(node->data); - } - } - - return(NULL); -} - -/*********************************************************//** -Looks for an element when we know the pointer to the data. -@return pointer to the hash table node, NULL if not found in the table */ -UNIV_INLINE -ha_node_t* -ha_search_with_data( -/*================*/ - hash_table_t* table, /*!< in: hash table */ - ulint fold, /*!< in: folded value of the searched data */ - const rec_t* data) /*!< in: pointer to the data */ -{ - ha_node_t* node; - - ut_ad(btr_search_enabled); - - node = ha_chain_get_first(table, fold); - - while (node) { - if (node->data == data) { - - return(node); - } - - node = ha_chain_get_next(node); - } - - return(NULL); -} - -#endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/include/mem0mem.h b/storage/innobase/include/mem0mem.h index 959147a61fcf9..6e80979c7997a 100644 --- a/storage/innobase/include/mem0mem.h +++ b/storage/innobase/include/mem0mem.h @@ -28,8 +28,6 @@ Created 6/9/1994 Heikki Tuuri #define mem0mem_h #include "ut0mem.h" -#include "ut0rnd.h" -#include "mach0data.h" #include @@ -42,22 +40,14 @@ typedef struct mem_block_info_t mem_block_t; /** A memory heap is a nonempty linear list of memory blocks */ typedef mem_block_t mem_heap_t; +struct buf_block_t; + /** Types of allocation for memory heaps: DYNAMIC means allocation from the dynamic memory pool of the C compiler, BUFFER means allocation from the buffer pool; the latter method is used for very big heaps */ #define MEM_HEAP_DYNAMIC 0 /* the most common type */ #define MEM_HEAP_BUFFER 1 -#define MEM_HEAP_BTR_SEARCH 2 /* this flag can optionally be - ORed to MEM_HEAP_BUFFER, in which - case heap->free_block is used in - some cases for memory allocations, - and if it's NULL, the memory - allocation functions can return - NULL. */ - -/** Different type of heaps in terms of which datastructure is using them */ -#define MEM_HEAP_FOR_BTR_SEARCH (MEM_HEAP_BTR_SEARCH | MEM_HEAP_BUFFER) #define MEM_HEAP_FOR_LOCK_HEAP (MEM_HEAP_BUFFER) /** The following start size is used for the first block in the memory heap if @@ -110,8 +100,7 @@ A single user buffer of 'size' will fit in the block. @param[in] file_name File name where created @param[in] line Line where created @param[in] type Heap type -@return own: memory heap, NULL if did not succeed (only possible for -MEM_HEAP_BTR_SEARCH type heaps) */ +@return own: memory heap */ UNIV_INLINE mem_heap_t* mem_heap_create_func( @@ -145,8 +134,7 @@ mem_heap_zalloc( @param[in] heap memory heap @param[in] n number of bytes; if the heap is allowed to grow into the buffer pool, this must be <= MEM_MAX_ALLOC_IN_BUF -@return allocated storage, NULL if did not succeed (only possible for -MEM_HEAP_BTR_SEARCH type heaps) */ +@return allocated storage */ UNIV_INLINE void* mem_heap_alloc( @@ -180,26 +168,6 @@ void mem_heap_empty( mem_heap_t* heap); -/** Returns a pointer to the topmost element in a memory heap. -The size of the element must be given. -@param[in] heap memory heap -@param[in] n size of the topmost element -@return pointer to the topmost element */ -UNIV_INLINE -void* -mem_heap_get_top( - mem_heap_t* heap, - ulint n); - -/*****************************************************************//** -Frees the topmost element in a memory heap. -The size of the element must be given. */ -UNIV_INLINE -void -mem_heap_free_top( -/*==============*/ - mem_heap_t* heap, /*!< in: memory heap */ - ulint n); /*!< in: size of the topmost element */ /*****************************************************************//** Returns the space in bytes occupied by a memory heap. */ UNIV_INLINE @@ -319,19 +287,13 @@ struct mem_block_info_t { in the heap. This is defined only in the base node and is set to ULINT_UNDEFINED in others. */ ulint type; /*!< type of heap: MEM_HEAP_DYNAMIC, or - MEM_HEAP_BUF possibly ORed to MEM_HEAP_BTR_SEARCH */ + MEM_HEAP_BUFFER */ ulint free; /*!< offset in bytes of the first free position for user data in the block */ ulint start; /*!< the value of the struct field 'free' at the creation of the block */ - void* free_block; - /* if the MEM_HEAP_BTR_SEARCH bit is set in type, - and this is the heap root, this can contain an - allocated buffer frame, which can be appended as a - free block to the heap, if we need more space; - otherwise, this is NULL */ - void* buf_block; + buf_block_t* buf_block; /* if this block has been allocated from the buffer pool, this contains the buf_block_t handle; otherwise, this is NULL */ diff --git a/storage/innobase/include/mem0mem.inl b/storage/innobase/include/mem0mem.inl index 9906daf3eb90a..fd6dc1e713bc2 100644 --- a/storage/innobase/include/mem0mem.inl +++ b/storage/innobase/include/mem0mem.inl @@ -39,8 +39,7 @@ Created 6/8/1994 Heikki Tuuri #endif /* UNIV_DEBUG */ /***************************************************************//** Creates a memory heap block where data can be allocated. -@return own: memory heap block, NULL if did not succeed (only possible -for MEM_HEAP_BTR_SEARCH type heaps) */ +@return own: memory heap block */ mem_block_t* mem_heap_create_block_func( /*=======================*/ @@ -62,19 +61,11 @@ mem_heap_block_free( mem_heap_t* heap, /*!< in: heap */ mem_block_t* block); /*!< in: block to free */ -/******************************************************************//** -Frees the free_block field from a memory heap. */ -void -mem_heap_free_block_free( -/*=====================*/ - mem_heap_t* heap); /*!< in: heap */ - /***************************************************************//** Adds a new block to a memory heap. @param[in] heap memory heap @param[in] n number of bytes needed -@return created block, NULL if did not succeed (only possible for -MEM_HEAP_BTR_SEARCH type heaps) */ +@return created block */ mem_block_t* mem_heap_add_block( mem_heap_t* heap, @@ -100,9 +91,7 @@ UNIV_INLINE void mem_block_set_type(mem_block_t* block, ulint type) { - ut_ad((type == MEM_HEAP_DYNAMIC) || (type == MEM_HEAP_BUFFER) - || (type == MEM_HEAP_BUFFER + MEM_HEAP_BTR_SEARCH)); - + ut_ad(type == MEM_HEAP_DYNAMIC || type == MEM_HEAP_BUFFER); block->type = type; } @@ -157,8 +146,6 @@ mem_heap_zalloc( mem_heap_t* heap, ulint n) { - ut_ad(heap); - ut_ad(!(heap->type & MEM_HEAP_BTR_SEARCH)); return(memset(mem_heap_alloc(heap, n), 0, n)); } @@ -166,8 +153,7 @@ mem_heap_zalloc( @param[in] heap memory heap @param[in] n number of bytes; if the heap is allowed to grow into the buffer pool, this must be <= MEM_MAX_ALLOC_IN_BUF -@return allocated storage, NULL if did not succeed (only possible for -MEM_HEAP_BTR_SEARCH type heaps) */ +@return allocated storage */ UNIV_INLINE void* mem_heap_alloc( @@ -290,62 +276,6 @@ mem_heap_empty( mem_heap_t* heap) { mem_heap_free_heap_top(heap, (byte*) heap + mem_block_get_start(heap)); - - if (heap->free_block) { - mem_heap_free_block_free(heap); - } -} - -/** Returns a pointer to the topmost element in a memory heap. -The size of the element must be given. -@param[in] heap memory heap -@param[in] n size of the topmost element -@return pointer to the topmost element */ -UNIV_INLINE -void* -mem_heap_get_top( - mem_heap_t* heap, - ulint n) -{ - mem_block_t* block; - byte* buf; - - block = UT_LIST_GET_LAST(heap->base); - - buf = (byte*) block + mem_block_get_free(block) - MEM_SPACE_NEEDED(n); - - return((void*) buf); -} - -/*****************************************************************//** -Frees the topmost element in a memory heap. The size of the element must be -given. */ -UNIV_INLINE -void -mem_heap_free_top( -/*==============*/ - mem_heap_t* heap, /*!< in: memory heap */ - ulint n) /*!< in: size of the topmost element */ -{ - mem_block_t* block; - - n += REDZONE_SIZE; - - block = UT_LIST_GET_LAST(heap->base); - - /* Subtract the free field of block */ - mem_block_set_free(block, mem_block_get_free(block) - - MEM_SPACE_NEEDED(n)); - - /* If free == start, we may free the block if it is not the first - one */ - - if ((heap != block) && (mem_block_get_free(block) - == mem_block_get_start(block))) { - mem_heap_block_free(heap, block); - } else { - MEM_NOACCESS((byte*) block + mem_block_get_free(block), n); - } } /** Creates a memory heap. @@ -356,8 +286,7 @@ A single user buffer of 'size' will fit in the block. @param[in] file_name File name where created @param[in] line Line where created @param[in] type Heap type -@return own: memory heap, NULL if did not succeed (only possible for -MEM_HEAP_BTR_SEARCH type heaps) */ +@return own: memory heap */ UNIV_INLINE mem_heap_t* mem_heap_create_func( @@ -406,10 +335,6 @@ mem_heap_free( block = UT_LIST_GET_LAST(heap->base); - if (heap->free_block) { - mem_heap_free_block_free(heap); - } - while (block != NULL) { /* Store the contents of info before freeing current block (it is erased in freeing) */ @@ -430,13 +355,7 @@ mem_heap_get_size( /*==============*/ mem_heap_t* heap) /*!< in: heap */ { - ulint size = heap->total_size; - - if (heap->free_block) { - size += srv_page_size; - } - - return(size); + return heap->total_size; } /**********************************************************************//** diff --git a/storage/innobase/mem/mem0mem.cc b/storage/innobase/mem/mem0mem.cc index 8a342275a46db..01e119b9f2781 100644 --- a/storage/innobase/mem/mem0mem.cc +++ b/storage/innobase/mem/mem0mem.cc @@ -215,7 +215,6 @@ mem_heap_validate( case MEM_HEAP_DYNAMIC: break; case MEM_HEAP_BUFFER: - case MEM_HEAP_BUFFER | MEM_HEAP_BTR_SEARCH: ut_ad(block->len <= srv_page_size); break; default: @@ -242,8 +241,7 @@ static void ut_strlcpy_rev(char* dst, const char* src, ulint size) /***************************************************************//** Creates a memory heap block where data can be allocated. -@return own: memory heap block, NULL if did not succeed (only possible -for MEM_HEAP_BTR_SEARCH type heaps) */ +@return own: memory heap block */ mem_block_t* mem_heap_create_block_func( /*=======================*/ @@ -257,12 +255,11 @@ mem_heap_create_block_func( ulint type) /*!< in: type of heap: MEM_HEAP_DYNAMIC or MEM_HEAP_BUFFER */ { - buf_block_t* buf_block = NULL; + buf_block_t* buf_block; mem_block_t* block; ulint len; - ut_ad((type == MEM_HEAP_DYNAMIC) || (type == MEM_HEAP_BUFFER) - || (type == MEM_HEAP_BUFFER + MEM_HEAP_BTR_SEARCH)); + ut_ad(type == MEM_HEAP_DYNAMIC || type == MEM_HEAP_BUFFER); if (heap != NULL) { ut_d(mem_heap_validate(heap)); @@ -276,24 +273,11 @@ mem_heap_create_block_func( ut_ad(type == MEM_HEAP_DYNAMIC || n <= MEM_MAX_ALLOC_IN_BUF); block = static_cast(ut_malloc_nokey(len)); + buf_block = nullptr; } else { len = srv_page_size; - if ((type & MEM_HEAP_BTR_SEARCH) && heap) { - /* We cannot allocate the block from the - buffer pool, but must get the free block from - the heap header free block field */ - - buf_block = static_cast(heap->free_block); - heap->free_block = NULL; - - if (UNIV_UNLIKELY(!buf_block)) { - - return(NULL); - } - } else { - buf_block = buf_block_alloc(); - } + buf_block = buf_block_alloc(); block = (mem_block_t*) buf_block->page.frame; } @@ -304,7 +288,6 @@ mem_heap_create_block_func( } block->buf_block = buf_block; - block->free_block = NULL; ut_d(ut_strlcpy_rev(block->file_name, file_name, sizeof(block->file_name))); @@ -340,8 +323,7 @@ mem_heap_create_block_func( /***************************************************************//** Adds a new block to a memory heap. -@return created block, NULL if did not succeed (only possible for -MEM_HEAP_BTR_SEARCH type heaps) */ +@return created block */ mem_block_t* mem_heap_add_block( /*===============*/ @@ -400,9 +382,6 @@ mem_heap_block_free( { ulint type; ulint len; - buf_block_t* buf_block; - - buf_block = static_cast(block->buf_block); UT_LIST_REMOVE(heap->base, block); @@ -413,25 +392,10 @@ mem_heap_block_free( len = block->len; if (type == MEM_HEAP_DYNAMIC || len < srv_page_size / 2) { - ut_ad(!buf_block); + ut_ad(!block->buf_block); ut_free(block); } else { ut_ad(type & MEM_HEAP_BUFFER); - buf_block_free(buf_block); - } -} - -/******************************************************************//** -Frees the free_block field from a memory heap. */ -void -mem_heap_free_block_free( -/*=====================*/ - mem_heap_t* heap) /*!< in: heap */ -{ - if (UNIV_LIKELY_NULL(heap->free_block)) { - - buf_block_free(static_cast(heap->free_block)); - - heap->free_block = NULL; + buf_block_free(block->buf_block); } } diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index bd0ef0eee1bbf..8a9fa9c060ddb 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -34,9 +34,8 @@ Created 11/26/1995 Heikki Tuuri #include "my_cpu.h" #ifdef BTR_CUR_HASH_ADAPT # include "btr0sea.h" -#else -# include "btr0cur.h" #endif +#include "btr0cur.h" #include "srv0start.h" #include "log.h" #include "mariadb_stats.h" diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index a755704405661..2f25c30840c13 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3229,10 +3229,6 @@ static void add_fts_index(dict_table_t *table) for (ulint i= 0; i < clust_index->n_uniq; i++) dict_index_add_col(fts_index, table, clust_index->fields[i].col, clust_index->fields[i].prefix_len); -#ifdef BTR_CUR_HASH_ADAPT - fts_index->search_info= btr_search_info_create(fts_index->heap); - fts_index->search_info->ref_count= 0; -#endif /* BTR_CUR_HASH_ADAPT */ UT_LIST_ADD_LAST(fts_index->table->indexes, fts_index); } @@ -3335,9 +3331,6 @@ static dict_table_t *build_fts_hidden_table( new_index->fields[old_index->n_fields].fixed_len= sizeof(doc_id_t); } -#ifdef BTR_CUR_HASH_ADAPT - new_index->search_info= btr_search_info_create(new_index->heap); -#endif /* BTR_CUR_HASH_ADAPT */ UT_LIST_ADD_LAST(new_index->table->indexes, new_index); old_index= UT_LIST_GET_NEXT(indexes, old_index); if (UT_LIST_GET_LEN(new_table->indexes) diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 0502568b31241..da3db84440b49 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2740,13 +2740,10 @@ row_ins_clust_index_entry_low( #endif /* WITH_WSREP */ #ifdef BTR_CUR_HASH_ADAPT - if (btr_search_enabled) { - btr_search_x_lock_all(); - index->table->bulk_trx_id = trx->id; - btr_search_x_unlock_all(); - } else { - index->table->bulk_trx_id = trx->id; - } + auto &part = btr_search.get_part(*index); + part.latch.wr_lock(SRW_LOCK_CALL); + index->table->bulk_trx_id = trx->id; + part.latch.wr_unlock(); #else /* BTR_CUR_HASH_ADAPT */ index->table->bulk_trx_id = trx->id; #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 2345b853b541c..a82f053f5cf11 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3859,7 +3859,7 @@ row_merge_drop_indexes( prebuilt->ins_node->entry_list in ins_node_create_entry_list(). */ #ifdef BTR_CUR_HASH_ADAPT - ut_ad(!index->search_info->ref_count); + ut_ad(!index->search_info.ref_count); #endif /* BTR_CUR_HASH_ADAPT */ dict_index_remove_from_cache( table, index); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index bb3b03aef14b3..db525fb3f8a42 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2194,7 +2194,7 @@ row_create_index_for_mysql( err = dict_create_index_tree_in_mem(index, trx); #ifdef BTR_CUR_HASH_ADAPT - ut_ad(!index->search_info->ref_count); + ut_ad(!index->search_info.ref_count); #endif /* BTR_CUR_HASH_ADAPT */ if (err != DB_SUCCESS) { diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 38b79af8da406..2a174ecd1a4a3 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -4543,7 +4543,7 @@ row_search_mvcc( if (UNIV_UNLIKELY(direction == 0) && unique_search - && btr_search_enabled + && btr_search.enabled && dict_index_is_clust(index) && !index->table->is_temporary() && !prebuilt->templ_contains_blob diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index cc35a659a71f2..115d3aaca81eb 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -804,15 +804,13 @@ srv_printf_innodb_monitor( ibuf_print(file); #ifdef BTR_CUR_HASH_ADAPT - for (ulint i = 0; i < btr_ahi_parts && btr_search_enabled; ++i) { - const auto part= &btr_search_sys.parts[i]; - part->latch.rd_lock(SRW_LOCK_CALL); - ut_ad(part->heap->type == MEM_HEAP_FOR_BTR_SEARCH); + for (ulong i = 0; i < btr_search.n_parts; ++i) { + btr_sea::partition& part= btr_search.parts[i]; + part.blocks_mutex.wr_lock(); fprintf(file, "Hash table size " ULINTPF ", node heap has " ULINTPF " buffer(s)\n", - part->table.n_cells, - part->heap->base.count - !part->heap->free_block); - part->latch.rd_unlock(); + part.table.n_cells, part.blocks.count + !!part.spare); + part.blocks_mutex.wr_unlock(); } /* btr_cur_n_sea_old and btr_cur_n_non_sea_old are protected by @@ -941,17 +939,17 @@ srv_export_innodb_status(void) export_vars.innodb_ahi_miss = btr_cur_n_non_sea; ulint mem_adaptive_hash = 0; - for (ulong i = 0; i < btr_ahi_parts; i++) { - const auto part= &btr_search_sys.parts[i]; - part->latch.rd_lock(SRW_LOCK_CALL); - if (part->heap) { - ut_ad(part->heap->type == MEM_HEAP_FOR_BTR_SEARCH); - - mem_adaptive_hash += mem_heap_get_size(part->heap) - + part->table.n_cells * sizeof(hash_cell_t); - } - part->latch.rd_unlock(); + for (ulong i = 0; i < btr_search.n_parts; i++) { + btr_sea::partition& part= btr_search.parts[i]; + part.blocks_mutex.wr_lock(); + mem_adaptive_hash += part.blocks.count + !!part.spare; + part.blocks_mutex.wr_unlock(); } + mem_adaptive_hash <<= srv_page_size_shift; + btr_search.parts[0].latch.rd_lock(SRW_LOCK_CALL); + mem_adaptive_hash += btr_search.parts[0].table.n_cells + * sizeof *btr_search.parts[0].table.array * btr_search.n_parts; + btr_search.parts[0].latch.rd_unlock(); export_vars.innodb_mem_adaptive_hash = mem_adaptive_hash; #endif diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index ff1a363565a5a..dc738e7318954 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2093,7 +2093,7 @@ void innodb_shutdown() #ifdef BTR_CUR_HASH_ADAPT if (dict_sys.is_initialised()) { - btr_search_disable(); + btr_search.disable(); } #endif /* BTR_CUR_HASH_ADAPT */ ibuf_close(); From e245c6ab11747dd35b4ef0ce0e69f365f428f11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 9 Jan 2025 14:15:14 +0200 Subject: [PATCH 02/10] MDEV-35049: Use CRC-32C and avoid allocating heap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For the adaptive hash index, dtuple_fold() and rec_fold() were employing a slow rolling hash algorithm, computing hash values ("fold") for one field and one byte at a time, while depending on calls to rec_get_offsets(). We already have optimized implementations of CRC-32C and have been successfully using that function in some other InnoDB tables, but not yet in the adaptive hash index. Any linear function such as any CRC will fail the avalanche test that any cryptographically secure hash function is expected to pass: any single-bit change in the input key should affect on average half the bits in the output. But we always were happy with less than cryptographically secure: in fact, ut_fold_ulint_pair() or ut_fold_binary() are just about as linear as any CRC, using a combination of multiplication and addition, partly carry-less. It is worth noting that exclusive-or corresponds to carry-less subtraction or addition in a binary Galois field, or GF(2). We only need some way of reducing key prefixes into hash values. The CRC-32C should be better than a Rabin–Karp rolling hash algorithm. Compared to the old hash algorithm, it has the drawback that there will be only 32 bits of entropy before we choose the hash table cell by a modulus operation. The size of each adaptive hash index array is (innodb_buffer_pool_size / 512) / innodb_adaptive_hash_index_parts. With the maximum number of partitions (512), we would not exceed 1<<32 elements per array until the buffer pool size exceeds 1<<50 bytes (1 PiB). We would hit other limits before that: the virtual address space on many contemporary 64-bit processor implementations is only 48 bits (256 TiB). So, we can simply go for the SIMD accelerated CRC-32C. rec_fold(): Take a combined parameter n_bytes_fields. Determine the length of each field on the fly, and compute CRC-32C over a single contiguous range of bytes, from the start of the record payload area to the end of the last full or partial field. For secondary index records in ROW_FORMAT=REDUNDANT, also the data area that is reserved for NULL values (to facilitate in-place updates between NULL and NOT NULL values) will be included in the count. Luckily, InnoDB always zero-initialized such unused area; refer to data_write_sql_null() in rec_convert_dtuple_to_rec_old(). For other than ROW_FORMAT=REDUNDANT, no space is allocated for NULL values, and therefore the CRC-32C will only cover the actual payload of the key prefix. dtuple_fold(): For ROW_FORMAT=REDUNDANT, include the dummy NULL values in the CRC-32C, so that the values will be comparable with rec_fold(). innodb_ahi-t: A unit test for rec_fold() and dtuple_fold(). btr_search_build_page_hash_index(), btr_search_drop_page_hash_index(): Use a fixed-size stack buffer for computing the fold values, to avoid dynamic memory allocation. btr_search_drop_page_hash_index(): Do not release part.latch if we need to invoke multiple batches of rec_fold(). dtuple_t: Allocate fewer bits for the fields. The maximum number of data fields is about 1023, so uint16_t will be fine for them. The info_bits is stored in less than 1 byte. ut_pair_min(), ut_pair_cmp(): Remove. We can actually combine and compare int(n_fields << 16 | n_bytes). PAGE_CUR_LE_OR_EXTENDS, PAGE_CUR_DBG: Remove. These were never defined, because they would only work with latin1_swedish_ci if at all. btr_cur_t::check_mismatch(): Replaces !btr_search_check_guess(). cmp_dtuple_rec_bytes(): Replaces cmp_dtuple_rec_with_match_bytes(). Determine the offsets of fields on the fly. page_cur_try_search_shortcut_bytes(): This caller of cmp_dtuple_rec_bytes() will not be invoked on the change buffer tree. cmp_dtuple_rec_leaf(): Replaces cmp_dtuple_rec_with_match() for comparing leaf-page records. buf_block_t::ahi_left_bytes_fields: Consolidated Atomic_relaxed of curr_left_side << 31 | curr_n_bytes << 16 | curr_n_fields. The other set of parameters (n_fields, n_bytes, left_side) was removed as redundant. btr_search_update_hash_node_on_insert(): Merged to btr_search_update_hash_on_insert(). btr_search_build_page_hash_index(): Take combined left_bytes_fields instead of n_fields, n_bytes, left_side. btr_search_update_block_hash_info(), btr_search_update_hash_ref(): Merged to btr_search_info_update_hash(). btr_cur_t::n_bytes_fields: Replaces n_bytes << 16 | n_fields. We also remove many redundant checks of btr_search.enabled. If we are holding any btr_sea::partition::latch, then a nonnull pointer in buf_block_t::index must imply that the adaptive hash index is enabled. Reviewed by: Vladislav Lesin --- storage/innobase/CMakeLists.txt | 1 - storage/innobase/btr/btr0btr.cc | 12 +- storage/innobase/btr/btr0cur.cc | 74 +- storage/innobase/btr/btr0pcur.cc | 2 +- storage/innobase/btr/btr0sea.cc | 2192 +++++++++------------ storage/innobase/buf/buf0buf.cc | 62 +- storage/innobase/data/data0data.cc | 19 +- storage/innobase/dict/dict0dict.cc | 8 +- storage/innobase/dict/dict0load.cc | 18 +- storage/innobase/fts/fts0fts.cc | 6 +- storage/innobase/gis/gis0rtree.cc | 11 +- storage/innobase/gis/gis0sea.cc | 26 +- storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/handler/handler0alter.cc | 12 +- storage/innobase/ibuf/ibuf0ibuf.cc | 16 +- storage/innobase/include/btr0cur.h | 38 +- storage/innobase/include/btr0pcur.inl | 4 +- storage/innobase/include/btr0sea.h | 67 +- storage/innobase/include/buf0buf.h | 100 +- storage/innobase/include/data0data.h | 29 +- storage/innobase/include/data0data.inl | 77 +- storage/innobase/include/dict0dict.h | 4 +- storage/innobase/include/dict0dict.inl | 4 +- storage/innobase/include/dict0mem.h | 15 +- storage/innobase/include/page0cur.h | 46 +- storage/innobase/include/page0types.h | 5 - storage/innobase/include/rem0cmp.h | 48 +- storage/innobase/include/rem0rec.h | 6 +- storage/innobase/include/rem0rec.inl | 22 +- storage/innobase/include/ut0ut.h | 63 - storage/innobase/include/ut0ut.inl | 143 -- storage/innobase/log/log0sync.cc | 1 + storage/innobase/mtr/mtr0mtr.cc | 3 +- storage/innobase/page/page0cur.cc | 1274 ++++++------ storage/innobase/rem/rem0cmp.cc | 295 +-- storage/innobase/row/row0ins.cc | 13 +- storage/innobase/row/row0log.cc | 52 +- storage/innobase/row/row0row.cc | 6 +- storage/innobase/row/row0sel.cc | 24 +- storage/innobase/row/row0vers.cc | 2 +- storage/innobase/trx/trx0rec.cc | 5 +- storage/innobase/unittest/CMakeLists.txt | 6 + storage/innobase/unittest/innodb_ahi-t.cc | 220 +++ 43 files changed, 2213 insertions(+), 2820 deletions(-) delete mode 100644 storage/innobase/include/ut0ut.inl create mode 100644 storage/innobase/unittest/innodb_ahi-t.cc diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index ba1fee9d4072d..a067bce2678ad 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -372,7 +372,6 @@ SET(INNOBASE_SOURCES include/ut0sort.h include/ut0stage.h include/ut0ut.h - include/ut0ut.inl include/ut0vec.h include/ut0vec.inl include/ut0wqueue.h diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 7befde5fa7c18..c6fba3422f77b 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -597,8 +597,8 @@ buf_block_t *btr_root_block_sx(dict_index_t *index, mtr_t *mtr, dberr_t *err) return root; } #ifdef BTR_CUR_HASH_ADAPT - else - ut_ad(!root->index || !root->index->freed()); + ut_d(else if (dict_index_t *index= root->index)) + ut_ad(!index->freed()); #endif return root; } @@ -863,7 +863,7 @@ static rec_offs *btr_page_get_parent(rec_offs *offsets, mem_heap_t *heap, { ut_ad(block->page.lock.have_u_or_x() || (!block->page.lock.have_s() && index->lock.have_x())); - ulint up_match= 0, low_match= 0; + uint16_t up_match= 0, low_match= 0; cursor->page_cur.block= block; if (page_cur_search_with_match(tuple, PAGE_CUR_LE, &up_match, &low_match, &cursor->page_cur, @@ -2149,7 +2149,7 @@ btr_root_raise_and_insert( ut_ad(dtuple_check_typed(tuple)); /* Reposition the cursor to the child node */ - ulint low_match = 0, up_match = 0; + uint16_t low_match = 0, up_match = 0; if (page_cur_search_with_match(tuple, PAGE_CUR_LE, &up_match, &low_match, @@ -2834,7 +2834,7 @@ btr_insert_into_right_sibling( return nullptr; } - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; if (page_cur_search_with_match(tuple, PAGE_CUR_LE, &up_match, &low_match, @@ -3365,7 +3365,7 @@ btr_page_split_and_insert( page_cursor = btr_cur_get_page_cur(cursor); page_cursor->block = insert_block; - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; if (page_cur_search_with_match(tuple, PAGE_CUR_LE, &up_match, &low_match, diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 4d11052ba6d0f..a023ccdfb6aa2 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1114,8 +1114,7 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, MEM_UNDEFINED(&up_bytes, sizeof up_bytes); MEM_UNDEFINED(&low_match, sizeof low_match); MEM_UNDEFINED(&low_bytes, sizeof low_bytes); - ut_d(up_match= ULINT_UNDEFINED); - ut_d(low_match= ULINT_UNDEFINED); + ut_d(up_match= low_match= uint16_t(~0u)); ut_ad(!(latch_mode & BTR_ALREADY_S_LATCHED) || mtr->memo_contains_flagged(&index()->lock, @@ -1180,9 +1179,9 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, latch_mode, this, mtr)) { /* Search using the hash index succeeded */ - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_GE); - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); - ut_ad(low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_GE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_LE); + ut_ad(low_match != uint16_t(~0U) || mode != PAGE_CUR_LE); ++btr_cur_n_sea; return DB_SUCCESS; @@ -1457,9 +1456,9 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, if (page_cur_search_with_match(tuple, mode, &up_match, &low_match, &page_cur, nullptr)) goto corrupted; - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_GE); - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); - ut_ad(low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_GE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_LE); + ut_ad(low_match != uint16_t(~0U) || mode != PAGE_CUR_LE); goto func_exit; } @@ -1509,22 +1508,22 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, reached_latched_leaf: #ifdef BTR_CUR_HASH_ADAPT - if (btr_search.enabled && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG)) + if (!(tuple->info_bits & REC_INFO_MIN_REC_FLAG) && !index()->is_ibuf() && + btr_search.enabled) { - if (page_cur_search_with_match_bytes(tuple, mode, - &up_match, &up_bytes, - &low_match, &low_bytes, &page_cur)) + if (page_cur_search_with_match_bytes(*tuple, mode, &up_match, &low_match, + &page_cur, &up_bytes, &low_bytes)) goto corrupted; } else #endif /* BTR_CUR_HASH_ADAPT */ - if (page_cur_search_with_match(tuple, mode, &up_match, &low_match, - &page_cur, nullptr)) - goto corrupted; + if (page_cur_search_with_match(tuple, mode, &up_match, &low_match, + &page_cur, nullptr)) + goto corrupted; - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_GE); - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); - ut_ad(low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_GE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_LE); + ut_ad(low_match != uint16_t(~0U) || mode != PAGE_CUR_LE); if (latch_mode == BTR_MODIFY_TREE && btr_cur_need_opposite_intention(block->page, index()->is_clust(), @@ -1776,9 +1775,9 @@ dberr_t btr_cur_t::pessimistic_search_leaf(const dtuple_t *tuple, err= DB_CORRUPTION; else { - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_GE); - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); - ut_ad(low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_GE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_LE); + ut_ad(low_match != uint16_t(~0U) || mode != PAGE_CUR_LE); #ifdef BTR_CUR_HASH_ADAPT /* We do a dirty read of btr_search.enabled here. We will recheck in @@ -2650,13 +2649,8 @@ btr_cur_optimistic_insert( ut_ad(entry->is_metadata()); ut_ad(index->is_instant()); ut_ad(flags == BTR_NO_LOCKING_FLAG); - } else if (index->table->is_temporary()) { - } else { - if (!reorg && cursor->flag == BTR_CUR_HASH) { - btr_search_update_hash_node_on_insert(cursor); - } else { - btr_search_update_hash_on_insert(cursor); - } + } else if (!index->table->is_temporary()) { + btr_search_update_hash_on_insert(cursor, reorg); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -2846,9 +2840,8 @@ btr_cur_pessimistic_insert( ut_ad(index->is_instant()); ut_ad(flags & BTR_NO_LOCKING_FLAG); ut_ad(!(flags & BTR_CREATE_FLAG)); - } else if (index->table->is_temporary()) { - } else { - btr_search_update_hash_on_insert(cursor); + } else if (!index->table->is_temporary()) { + btr_search_update_hash_on_insert(cursor, false); } #endif /* BTR_CUR_HASH_ADAPT */ if (inherit && !(flags & BTR_NO_LOCKING_FLAG)) { @@ -3523,7 +3516,7 @@ static void btr_cur_trim_alter_metadata(dtuple_t* entry, if (n_fields != index->n_uniq) { ut_ad(n_fields >= index->n_core_fields); - entry->n_fields = n_fields; + entry->n_fields = uint16_t(n_fields); return; } @@ -3559,7 +3552,7 @@ static void btr_cur_trim_alter_metadata(dtuple_t* entry, ut_ad(n_fields >= index->n_core_fields); mtr.commit(); - entry->n_fields = n_fields + 1; + entry->n_fields = uint16_t(n_fields + 1); } /** Trim an update tuple due to instant ADD COLUMN, if needed. @@ -3615,7 +3608,7 @@ btr_cur_trim( ulint n_fields = upd_get_nth_field(update, 0) ->field_no; ut_ad(n_fields + 1 >= entry->n_fields); - entry->n_fields = n_fields; + entry->n_fields = uint16_t(n_fields); } } else { entry->trim(*index); @@ -5127,10 +5120,10 @@ class btr_est_cur_t /** Matched fields and bytes which are used for on-page search, see btr_cur_t::(up|low)_(match|bytes) comments for details */ - ulint m_up_match= 0; - ulint m_up_bytes= 0; - ulint m_low_match= 0; - ulint m_low_bytes= 0; + uint16_t m_up_match= 0; + uint16_t m_up_bytes= 0; + uint16_t m_low_match= 0; + uint16_t m_low_bytes= 0; public: btr_est_cur_t(dict_index_t *index, const dtuple_t &tuple, @@ -5154,12 +5147,7 @@ class btr_est_cur_t m_page_mode= PAGE_CUR_LE; break; default: -#ifdef PAGE_CUR_LE_OR_EXTENDS - ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE || - mode == PAGE_CUR_LE_OR_EXTENDS); -#else /* PAGE_CUR_LE_OR_EXTENDS */ ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE); -#endif /* PAGE_CUR_LE_OR_EXTENDS */ m_page_mode= mode; break; } diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index c7fcfd205fbc1..529f463a70342 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -450,7 +450,7 @@ btr_pcur_t::restore_position(btr_latch_mode restore_latch_mode, mtr_t *mtr) rec_offs_init(offsets); restore_status ret_val= restore_status::NOT_SAME; if (rel_pos == BTR_PCUR_ON && btr_pcur_is_on_user_rec(this)) { - ulint n_matched_fields= 0; + uint16_t n_matched_fields= 0; if (!cmp_dtuple_rec_with_match( tuple, btr_pcur_get_rec(this), rec_get_offsets(btr_pcur_get_rec(this), index, offsets, diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index e2cc2739832b9..7ae1fa399fc5b 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -38,21 +38,21 @@ Created 2/17/1996 Heikki Tuuri #ifdef UNIV_SEARCH_PERF_STAT /** Number of successful adaptive hash index lookups */ -ulint btr_search_n_succ = 0; +ulint btr_search_n_succ; /** Number of failed adaptive hash index lookups */ -ulint btr_search_n_hash_fail = 0; +ulint btr_search_n_hash_fail; #endif /* UNIV_SEARCH_PERF_STAT */ #ifdef UNIV_PFS_RWLOCK -mysql_pfs_key_t btr_search_latch_key; +mysql_pfs_key_t btr_search_latch_key; #endif /* UNIV_PFS_RWLOCK */ /** The adaptive hash index */ btr_sea btr_search; struct ahi_node { - /** rec_fold(rec) */ - ulint fold; + /** CRC-32C of the rec prefix */ + uint32_t fold; /** pointer to next record in the hash bucket chain, or nullptr */ ahi_node *next; /** B-tree index leaf page record */ @@ -143,101 +143,156 @@ void btr_sea::free() noexcept /** If the number of records on the page divided by this parameter would have been successfully accessed using a hash index, the index is then built on the page, assuming the global limit has been reached */ -#define BTR_SEARCH_PAGE_BUILD_LIMIT 16U +static constexpr unsigned BTR_SEARCH_PAGE_BUILD_LIMIT= 16; /** The global limit for consecutive potentially successful hash searches, before hash index building is started */ -#define BTR_SEARCH_BUILD_LIMIT 100U +static constexpr uint8_t BTR_SEARCH_BUILD_LIMIT= 100; -/** Compute a hash value of a record in a page. -@param[in] rec index record -@param[in] offsets return value of rec_get_offsets() -@param[in] n_fields number of complete fields to fold -@param[in] n_bytes number of bytes to fold in the last field -@param[in] index_id index tree ID -@return the hash value */ -static inline -ulint -rec_fold( - const rec_t* rec, - const rec_offs* offsets, - ulint n_fields, - ulint n_bytes, - index_id_t tree_id) +/** Determine the number of accessed key fields. +@param n_bytes_fields number of complete fields | incomplete_bytes << 16 +@return number of complete or incomplete fields */ +inline size_t btr_search_get_n_fields(ulint n_bytes_fields) noexcept { - ulint i; - const byte* data; - ulint len; - ulint fold; - ulint n_fields_rec; - - ut_ad(rec_offs_validate(rec, NULL, offsets)); - ut_ad(rec_validate(rec, offsets)); - ut_ad(page_rec_is_leaf(rec)); - ut_ad(!page_rec_is_metadata(rec)); - ut_ad(n_fields > 0 || n_bytes > 0); - - n_fields_rec = rec_offs_n_fields(offsets); - ut_ad(n_fields <= n_fields_rec); - ut_ad(n_fields < n_fields_rec || n_bytes == 0); - - if (n_fields > n_fields_rec) { - n_fields = n_fields_rec; - } + return uint16_t(n_bytes_fields) + (n_bytes_fields >= 1U << 16); +} - if (n_fields == n_fields_rec) { - n_bytes = 0; - } +/** Determine the number of accessed key fields. +@param cursor b-tree cursor +@return number of complete or incomplete fields */ +inline size_t btr_search_get_n_fields(const btr_cur_t *cursor) noexcept +{ + return btr_search_get_n_fields(cursor->n_bytes_fields); +} - fold = ut_fold_ull(tree_id); +/** Compute a hash value of a record in a page. +@tparam comp whether ROW_FORMAT=REDUNDANT is not being used +@param rec index record +@param index index tree +@param n_bytes_fields bytes << 16 | number of complete fields +@return CRC-32C of the record prefix */ +template +static uint32_t rec_fold(const rec_t *rec, const dict_index_t &index, + uint32_t n_bytes_fields) noexcept +{ + ut_ad(page_rec_is_leaf(rec)); + ut_ad(page_rec_is_user_rec(rec)); + ut_ad(!rec_is_metadata(rec, index)); + ut_ad(index.n_uniq <= index.n_core_fields); + size_t n_f= btr_search_get_n_fields(n_bytes_fields); + ut_ad(n_f > 0); + ut_ad(n_f <= index.n_core_fields); + ut_ad(comp == index.table->not_redundant()); - for (i = 0; i < n_fields; i++) { - data = rec_get_nth_field(rec, offsets, i, &len); + size_t n; - if (len != UNIV_SQL_NULL) { - fold = ut_fold_ulint_pair(fold, - ut_fold_binary(data, len)); - } - } + if (comp) + { + const byte *nulls= rec - REC_N_NEW_EXTRA_BYTES; + const byte *lens; + if (rec_get_status(rec) == REC_STATUS_INSTANT) + { + ulint n_fields= index.n_core_fields + rec_get_n_add_field(nulls) + 1; + ut_ad(n_fields <= index.n_fields); + const ulint n_nullable= index.get_n_nullable(n_fields); + ut_ad(n_nullable <= index.n_nullable); + lens= --nulls - UT_BITS_IN_BYTES(n_nullable); + } + else + lens= --nulls - index.n_core_null_bytes; + byte null_mask= 1; + n= 0; - if (n_bytes > 0) { - data = rec_get_nth_field(rec, offsets, i, &len); + const dict_field_t *field= index.fields; + size_t len; + do + { + const dict_col_t *col= field->col; + if (col->is_nullable()) + { + const int is_null{*nulls & null_mask}; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic push +# if __GNUC__ < 12 || defined WITH_UBSAN +# pragma GCC diagnostic ignored "-Wconversion" +# endif +#endif + null_mask<<= 1; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic pop +#endif + if (UNIV_UNLIKELY(!null_mask)) + null_mask= 1, nulls--; + if (is_null) + { + len= 0; + continue; + } + } - if (len != UNIV_SQL_NULL) { - if (len > n_bytes) { - len = n_bytes; - } + len= field->fixed_len; - fold = ut_fold_ulint_pair(fold, - ut_fold_binary(data, len)); - } - } + if (!len) + { + len= *lens--; + if (UNIV_UNLIKELY(len & 0x80) && DATA_BIG_COL(col)) + { + len<<= 8; + len|= *lens--; + ut_ad(!(len & 0x4000)); + len&= 0x3fff; + } + } - return(fold); -} + n+= len; + } + while (field++, --n_f); -/** Determine the number of accessed key fields. -@param[in] n_fields number of complete fields -@param[in] n_bytes number of bytes in an incomplete last field -@return number of complete or incomplete fields */ -inline MY_ATTRIBUTE((warn_unused_result)) -ulint -btr_search_get_n_fields( - ulint n_fields, - ulint n_bytes) -{ - return(n_fields + (n_bytes > 0 ? 1 : 0)); + if (size_t n_bytes= n_bytes_fields >> 16) + n+= std::min(n_bytes, len) - len; + } + else + { + const size_t n_bytes= n_bytes_fields >> 16; + ut_ad(n_f <= rec_get_n_fields_old(rec)); + if (rec_get_1byte_offs_flag(rec)) + { + n= rec_1_get_field_end_info(rec, n_f - 1) & ~REC_1BYTE_SQL_NULL_MASK; + if (!n_bytes); + else if (!uint16_t(n_bytes_fields)) + n= std::min(n_bytes, n); + else + { + size_t len= n - (rec_1_get_field_end_info(rec, n_f - 2) & + ~REC_1BYTE_SQL_NULL_MASK); + n+= std::min(n_bytes, n - len) - len; + } + } + else + { + n= rec_2_get_field_end_info(rec, n_f - 1) & ~REC_2BYTE_SQL_NULL_MASK; + ut_ad(n < REC_2BYTE_EXTERN_MASK); /* keys never are BLOBs */ + if (!n_bytes); + else if (!uint16_t(n_bytes_fields)) + n= std::min(n_bytes, n); + else + { + size_t len= n - (rec_2_get_field_end_info(rec, n_f - 2) & + ~REC_2BYTE_SQL_NULL_MASK); + n+= std::min(n_bytes, n - len) - len; + } + } + } + + return my_crc32c(uint32_t(ut_fold_ull(index.id)), rec, n); } -/** Determine the number of accessed key fields. -@param[in] cursor b-tree cursor -@return number of complete or incomplete fields */ -inline MY_ATTRIBUTE((warn_unused_result)) -ulint -btr_search_get_n_fields( - const btr_cur_t* cursor) +static uint32_t rec_fold(const rec_t *rec, const dict_index_t &index, + uint32_t n_bytes_fields, ulint comp) noexcept { - return(btr_search_get_n_fields(cursor->n_fields, cursor->n_bytes)); + return comp + ? rec_fold(rec, index, n_bytes_fields) + : rec_fold(rec, index, n_bytes_fields); } void btr_sea::partition::prepare_insert() noexcept @@ -362,170 +417,223 @@ void btr_sea::enable(bool resize) noexcept } } -/** Updates the search info of an index about hash successes. NOTE that info -is NOT protected by any semaphore, to save CPU time! Do not assume its fields -are consistent. -@param[in] cursor cursor which was just positioned */ -static void btr_search_info_update_hash(const btr_cur_t *cursor) -{ - ut_ad(cursor->flag != BTR_CUR_HASH); - - dict_index_t* index = cursor->index(); - int cmp; - - if (dict_index_is_ibuf(index)) { - /* So many deletes are performed on an insert buffer tree - that we do not consider a hash index useful on it: */ - - return; - } - - uint16_t n_unique = dict_index_get_n_unique_in_tree(index); - dict_index_t::ahi* info = &index->search_info; - uint8_t n_hash_potential = info->n_hash_potential; - - if (n_hash_potential == 0) { - - goto set_new_recomm; - } +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG +# define ha_insert_for_fold(p,f,b,d) (p).insert(f,d,b) +#else +# define ha_insert_for_fold(p,f,b,d) (p).insert(f,d) +#endif - /* Test if the search would have succeeded using the recommended - hash prefix */ +ATTRIBUTE_NOINLINE +/** Update a hash node reference when it has been unsuccessfully used in a +search which could have succeeded with the used hash parameters. This can +happen because when building a hash index for a page, we do not check +what happens at page boundaries, and therefore there can be misleading +hash nodes. Also, collisions in the fold value can lead to misleading +references. This function lazily fixes these imperfections in the hash +index. +@param cursor B-tree cursor +@param block cursor block +@param left_bytes_fields AHI paramaters */ +static void btr_search_update_hash_ref(const btr_cur_t &cursor, + buf_block_t *block, + uint32_t left_bytes_fields) noexcept +{ + ut_ad(block == cursor.page_cur.block); +#ifdef UNIV_SEARCH_PERF_STAT + btr_search_n_hash_fail++; +#endif /* UNIV_SEARCH_PERF_STAT */ - if (info->n_fields >= n_unique && cursor->up_match >= n_unique) { -increment_potential: - if (n_hash_potential < BTR_SEARCH_BUILD_LIMIT) { - info->n_hash_potential = ++n_hash_potential; - } - return; - } + dict_index_t *const index= cursor.index(); + ut_ad(block->page.id().space() == index->table->space_id); + ut_ad(!index->is_ibuf()); + btr_sea::partition &part= btr_search.get_part(index->id); + part.prepare_insert(); + part.latch.wr_lock(SRW_LOCK_CALL); - cmp = ut_pair_cmp(info->n_fields, info->n_bytes, - cursor->low_match, cursor->low_bytes); + if (ut_d(const dict_index_t *block_index=) block->index) + { + ut_ad(block_index == index); + ut_ad(btr_search.enabled); + uint32_t bytes_fields{block->ahi_left_bytes_fields}; + if (bytes_fields != left_bytes_fields) + goto skip; + if (UNIV_UNLIKELY(index->search_info.left_bytes_fields != + left_bytes_fields)) + goto skip; + bytes_fields&= ~buf_block_t::LEFT_SIDE; + const rec_t *rec= cursor.page_cur.rec; + uint32_t fold; + if (page_is_comp(block->page.frame)) + { + switch (rec - block->page.frame) { + case PAGE_NEW_INFIMUM: + case PAGE_NEW_SUPREMUM: + goto skip; + default: + fold= rec_fold(rec, *index, bytes_fields); + } + } + else + { + switch (rec - block->page.frame) { + case PAGE_OLD_INFIMUM: + case PAGE_OLD_SUPREMUM: + goto skip; + default: + fold= rec_fold(rec, *index, bytes_fields); + } + } - if (info->left_side ? cmp <= 0 : cmp > 0) { + ha_insert_for_fold(part, fold, block, rec); + MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); + } +# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + else + ut_a(!block->n_pointers); +# endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - goto set_new_recomm; - } +skip: + part.latch.wr_unlock(); +} - cmp = ut_pair_cmp(info->n_fields, info->n_bytes, - cursor->up_match, cursor->up_bytes); +/** Updates the search info of an index about hash successes. +@param cursor freshly positioned cursor +@return AHI parameters +@retval 0 if the adaptive hash index should not be rebuilt */ +static uint32_t btr_search_info_update_hash(const btr_cur_t &cursor) noexcept +{ + ut_ad(cursor.flag != BTR_CUR_HASH); - if (info->left_side ? cmp <= 0 : cmp > 0) { + dict_index_t *const index= cursor.index(); - goto increment_potential; - } + if (index->is_ibuf()) + /* Too many deletes are performed on the change buffer */ + return 0; -set_new_recomm: - /* We have to set a new recommendation; skip the hash analysis - for a while to avoid unnecessary CPU time usage when there is no - chance for success */ - - info->hash_analysis_reset(); - - cmp = ut_pair_cmp(cursor->up_match, cursor->up_bytes, - cursor->low_match, cursor->low_bytes); - info->left_side = cmp >= 0; - info->n_hash_potential = cmp != 0; - - if (cmp == 0) { - /* For extra safety, we set some sensible values here */ - info->n_fields = 1; - info->n_bytes = 0; - } else if (cmp > 0) { - if (cursor->up_match >= n_unique) { - - info->n_fields = n_unique; - info->n_bytes = 0; - - } else if (cursor->low_match < cursor->up_match) { - - info->n_fields = static_cast( - cursor->low_match + 1); - info->n_bytes = 0; - } else { - info->n_fields = static_cast( - cursor->low_match); - info->n_bytes = static_cast( - cursor->low_bytes + 1); - } - } else { - if (cursor->low_match >= n_unique) { - - info->n_fields = n_unique; - info->n_bytes = 0; - } else if (cursor->low_match > cursor->up_match) { - - info->n_fields = static_cast( - cursor->up_match + 1); - info->n_bytes = 0; - } else { - info->n_fields = static_cast( - cursor->up_match); - info->n_bytes = static_cast( - cursor->up_bytes + 1); - } - } -} + const uint16_t n_uniq{dict_index_get_n_unique_in_tree(index)}; + dict_index_t::ahi &info= index->search_info; -/** Update the block search info on hash successes. NOTE that info and -block->n_hash_helps, n_fields, n_bytes, left_side are NOT protected by any -semaphore, to save CPU time! Do not assume the fields are consistent. -@return TRUE if building a (new) hash index on the block is recommended -@param[in,out] info search info -@param[in,out] block buffer block */ -static -bool -btr_search_update_block_hash_info(dict_index_t::ahi* info, buf_block_t* block) -{ - ut_ad(block->page.lock.have_x() || block->page.lock.have_s()); + uint32_t left_bytes_fields{info.left_bytes_fields}; + uint8_t n_hash_potential= info.n_hash_potential; + buf_block_t *const block= cursor.page_cur.block; + ut_ad(block->page.lock.have_any()); + ut_d(const uint32_t state= block->page.state()); + ut_ad(state >= buf_page_t::UNFIXED); + ut_ad(!block->page.is_read_fixed(state)); + const dict_index_t *const block_index= block->index; + uint16_t n_hash_helps{block->n_hash_helps}; + uint32_t ret; - info->last_hash_succ = FALSE; - ut_ad(block->page.frame); + if (!n_hash_potential) + { + info.left_bytes_fields= left_bytes_fields= buf_block_t::LEFT_SIDE | 1; + info.hash_analysis_reset(); + increment_potential: + if (n_hash_potential < BTR_SEARCH_BUILD_LIMIT) + info.n_hash_potential= ++n_hash_potential; + if (n_hash_helps) + goto got_help; + goto no_help; + } + else if (uint16_t(left_bytes_fields) >= n_uniq && cursor.up_match >= n_uniq) + /* The search would have succeeded using the recommended prefix */ + goto increment_potential; + else + { + const bool left_side{!!(left_bytes_fields & buf_block_t::LEFT_SIDE)}; + const int info_cmp= + int(uint16_t((left_bytes_fields & ~buf_block_t::LEFT_SIDE) >> 16) | + int{uint16_t(left_bytes_fields)} << 16); + const int low_cmp = int(cursor.low_match << 16 | cursor.low_bytes); + const int up_cmp = int(cursor.up_match << 16 | cursor.up_bytes); - if ((block->n_hash_helps > 0) - && (info->n_hash_potential > 0) - && (block->n_fields == info->n_fields) - && (block->n_bytes == info->n_bytes) - && (block->left_side == info->left_side)) { + if (left_side == (info_cmp > low_cmp) && left_side == (info_cmp <= up_cmp)) + goto increment_potential; - if ((block->index) - && (block->curr_n_fields == info->n_fields) - && (block->curr_n_bytes == info->n_bytes) - && (block->curr_left_side == info->left_side)) { + const int cmp= up_cmp - low_cmp; + static_assert(buf_block_t::LEFT_SIDE == 1U << 31, ""); + left_bytes_fields= (cmp >= 0) << 31; - /* The search would presumably have succeeded using - the hash index */ + if (left_bytes_fields) + { + if (cursor.up_match >= n_uniq) + left_bytes_fields|= n_uniq; + else if (cursor.low_match < cursor.up_match) + left_bytes_fields|= uint32_t(cursor.low_match + 1); + else + { + left_bytes_fields|= cursor.low_match; + left_bytes_fields|= uint32_t(cursor.low_bytes + 1) << 16; + } + } + else + { + if (cursor.low_match >= n_uniq) + left_bytes_fields|= n_uniq; + else if (cursor.low_match > cursor.up_match) + left_bytes_fields|= uint32_t(cursor.up_match + 1); + else + { + left_bytes_fields|= cursor.up_match; + left_bytes_fields|= uint32_t(cursor.up_bytes + 1) << 16; + } + } + /* We have to set a new recommendation; skip the hash analysis for a + while to avoid unnecessary CPU time usage when there is no chance + for success */ + info.hash_analysis_reset(); + info.left_bytes_fields= left_bytes_fields; + info.n_hash_potential= cmp != 0; + if (cmp == 0) + goto no_help; + } - info->last_hash_succ = TRUE; - } + ut_ad(block->page.lock.have_x() || block->page.lock.have_s()); + ut_ad(btr_cur_get_page(&cursor) == page_align(btr_cur_get_rec(&cursor))); + ut_ad(page_is_leaf(btr_cur_get_page(&cursor))); - block->n_hash_helps++; - } else { - block->n_hash_helps = 1; - block->n_fields = info->n_fields; - block->n_bytes = info->n_bytes; - block->left_side = info->left_side; - } + if (!n_hash_helps) + { + no_help: + info.last_hash_succ= false; + block->n_hash_helps= 1; + ret= 0; + } + else + { + got_help: + const uint32_t ahi_left_bytes_fields= block->ahi_left_bytes_fields; - if ((block->n_hash_helps > page_get_n_recs(block->page.frame) - / BTR_SEARCH_PAGE_BUILD_LIMIT) - && (info->n_hash_potential >= BTR_SEARCH_BUILD_LIMIT)) { + ret= left_bytes_fields; + info.last_hash_succ= + block_index && ahi_left_bytes_fields == left_bytes_fields; - if ((!block->index) - || (block->n_hash_helps - > 2U * page_get_n_recs(block->page.frame)) - || (block->n_fields != block->curr_n_fields) - || (block->n_bytes != block->curr_n_bytes) - || (block->left_side != block->curr_left_side)) { + if (n_hash_potential >= BTR_SEARCH_BUILD_LIMIT) + { + const auto n_recs= page_get_n_recs(block->page.frame); + if (n_hash_helps / 2 > n_recs) + goto func_exit; + if (n_hash_helps >= n_recs / BTR_SEARCH_PAGE_BUILD_LIMIT && + (!block_index || left_bytes_fields != ahi_left_bytes_fields)) + goto func_exit; + } - /* Build a new hash index on the page */ + if (++n_hash_helps) + block->n_hash_helps= n_hash_helps; + ret= 0; + } - return(true); - } - } +func_exit: + if (!block_index); + else if (UNIV_UNLIKELY(block_index != index)) + { + ut_ad(block_index->id == index->id); + btr_search_drop_page_hash_index(block, false); + } + else if (cursor.flag == BTR_CUR_HASH_FAIL) + btr_search_update_hash_ref(cursor, block, left_bytes_fields); - return(false); + return ret; } #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG @@ -534,10 +642,10 @@ constexpr ulint MAX_N_POINTERS = UNIV_PAGE_SIZE_MAX / REC_N_NEW_EXTRA_BYTES; #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG -void btr_sea::partition::insert(ulint fold, const rec_t *rec, +void btr_sea::partition::insert(uint32 fold, const rec_t *rec, buf_block_t *block) noexcept #else -void btr_sea::partition::insert(ulint fold, const rec_t *rec) noexcept +void btr_sea::partition::insert(uint32_t fold, const rec_t *rec) noexcept #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ { #ifndef SUX_LOCK_GENERIC @@ -682,10 +790,10 @@ buf_block_t *btr_sea::partition::cleanup_after_erase(ahi_node *erase) noexcept __attribute__((nonnull)) /** Delete all pointers to a page. @param part hash table partition -@param fold fold value +@param fold CRC-32C value @param page page of a record to be deleted */ static void ha_remove_all_nodes_to_page(btr_sea::partition &part, - ulint fold, const page_t *page) + uint32_t fold, const page_t *page) noexcept { hash_cell_t *cell= part.table.cell_get(fold); @@ -711,7 +819,7 @@ static void ha_remove_all_nodes_to_page(btr_sea::partition &part, { return page_align(node->rec) == page; })); } -inline bool btr_sea::partition::erase(ulint fold, const rec_t *rec) noexcept +inline bool btr_sea::partition::erase(uint32_t fold, const rec_t *rec) noexcept { #ifndef SUX_LOCK_GENERIC ut_ad(latch.is_write_locked()); @@ -745,20 +853,18 @@ updates the pointer to data if found. @param data pointer to the data @param new_data new pointer to the data @return whether the element was found */ -static bool ha_search_and_update_if_found(hash_table_t *table, ulint fold, +static bool ha_search_and_update_if_found(hash_table_t *table, uint32_t fold, const rec_t *data, #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG /** block containing new_data */ buf_block_t *new_block, #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - const rec_t *new_data) + const rec_t *new_data) noexcept { #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG ut_a(new_block->page.frame == page_align(new_data)); #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - - if (!btr_search.enabled) - return false; + ut_ad(btr_search.enabled); if (ahi_node *node= table->cell_get(fold)-> find(&ahi_node::next, [data](const ahi_node *node) @@ -779,249 +885,11 @@ static bool ha_search_and_update_if_found(hash_table_t *table, ulint fold, return false; } -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG -# define ha_insert_for_fold(p,f,b,d) (p).insert(f,d,b) -#else -# define ha_insert_for_fold(p,f,b,d) (p).insert(f,d) +#if !defined UNIV_AHI_DEBUG && !defined UNIV_DEBUG # define ha_search_and_update_if_found(table,fold,data,new_block,new_data) \ - ha_search_and_update_if_found(table,fold,data,new_data) + ha_search_and_update_if_found(table,fold,data,new_data) #endif -/** Updates a hash node reference when it has been unsuccessfully used in a -search which could have succeeded with the used hash parameters. This can -happen because when building a hash index for a page, we do not check -what happens at page boundaries, and therefore there can be misleading -hash nodes. Also, collisions in the fold value can lead to misleading -references. This function lazily fixes these imperfections in the hash -index. -@param[in] cursor cursor */ -static void btr_search_update_hash_ref(const btr_cur_t* cursor) noexcept -{ - ut_ad(cursor->flag == BTR_CUR_HASH_FAIL); - - buf_block_t* const block = cursor->page_cur.block; - ut_ad(block->page.lock.have_x() || block->page.lock.have_s()); - ut_ad(btr_cur_get_page(cursor) == block->page.frame); - ut_ad(page_is_leaf(block->page.frame)); - assert_block_ahi_valid(block); - - dict_index_t* index = block->index; - - if (!index || !index->search_info.n_hash_potential) { - return; - } - - if (index != cursor->index()) { - ut_ad(index->id == cursor->index()->id); - btr_search_drop_page_hash_index(block, false); - return; - } - - ut_ad(block->page.id().space() == index->table->space_id); - ut_ad(index == cursor->index()); - ut_ad(!dict_index_is_ibuf(index)); - btr_sea::partition& part = btr_search.get_part(*index); - part.latch.wr_lock(SRW_LOCK_CALL); - ut_ad(!block->index || block->index == index); - - if (block->index - && (block->curr_n_fields == index->search_info.n_fields) - && (block->curr_n_bytes == index->search_info.n_bytes) - && (block->curr_left_side == index->search_info.left_side) - && btr_search.enabled) { - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs_init(offsets_); - - const rec_t* rec = btr_cur_get_rec(cursor); - - if (!page_rec_is_user_rec(rec)) { - goto func_exit; - } - - ulint fold = rec_fold( - rec, - rec_get_offsets(rec, index, offsets_, - index->n_core_fields, - ULINT_UNDEFINED, &heap), - block->curr_n_fields, - block->curr_n_bytes, index->id); - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - - ha_insert_for_fold(part, fold, block, rec); - - MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); - } - -func_exit: - part.latch.wr_unlock(); -} - -/** Checks if a guessed position for a tree cursor is right. Note that if -mode is PAGE_CUR_LE, which is used in inserts, and the function returns -TRUE, then cursor->up_match and cursor->low_match both have sensible values. -@param[in,out] cursor guess cursor position -@param[in] tuple data tuple -@param[in] mode PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, PAGE_CUR_GE -@return whether a match was found */ -static -bool -btr_search_check_guess( - btr_cur_t* cursor, - const dtuple_t* tuple, - ulint mode) -{ - rec_t* rec; - ulint n_unique; - ulint match; - int cmp; - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - bool success = false; - rec_offs_init(offsets_); - - n_unique = dict_index_get_n_unique_in_tree(cursor->index()); - - rec = btr_cur_get_rec(cursor); - - if (UNIV_UNLIKELY(!page_rec_is_user_rec(rec) - || !page_rec_is_leaf(rec))) { - ut_ad("corrupted index" == 0); - return false; - } else if (cursor->index()->table->not_redundant()) { - switch (rec_get_status(rec)) { - case REC_STATUS_INSTANT: - case REC_STATUS_ORDINARY: - break; - default: - ut_ad("corrupted index" == 0); - return false; - } - } - - match = 0; - - offsets = rec_get_offsets(rec, cursor->index(), offsets, - cursor->index()->n_core_fields, - n_unique, &heap); - cmp = cmp_dtuple_rec_with_match(tuple, rec, offsets, &match); - - if (mode == PAGE_CUR_GE) { - if (cmp > 0) { - goto exit_func; - } - - cursor->up_match = match; - - if (match >= n_unique) { - success = true; - goto exit_func; - } - } else if (mode == PAGE_CUR_LE) { - if (cmp < 0) { - goto exit_func; - } - - cursor->low_match = match; - - } else if (mode == PAGE_CUR_G) { - if (cmp >= 0) { - goto exit_func; - } - } else if (mode == PAGE_CUR_L) { - if (cmp <= 0) { - goto exit_func; - } - } - - match = 0; - - if ((mode == PAGE_CUR_G) || (mode == PAGE_CUR_GE)) { - const rec_t* prev_rec = page_rec_get_prev(rec); - - if (UNIV_UNLIKELY(!prev_rec)) { - ut_ad("corrupted index" == 0); - goto exit_func; - } - - if (page_rec_is_infimum(prev_rec)) { - success = !page_has_prev(page_align(prev_rec)); - goto exit_func; - } - - if (cursor->index()->table->not_redundant()) { - switch (rec_get_status(prev_rec)) { - case REC_STATUS_INSTANT: - case REC_STATUS_ORDINARY: - break; - default: - ut_ad("corrupted index" == 0); - goto exit_func; - } - } - - offsets = rec_get_offsets(prev_rec, cursor->index(), offsets, - cursor->index()->n_core_fields, - n_unique, &heap); - cmp = cmp_dtuple_rec_with_match( - tuple, prev_rec, offsets, &match); - if (mode == PAGE_CUR_GE) { - success = cmp > 0; - } else { - success = cmp >= 0; - } - } else { - ut_ad(!page_rec_is_supremum(rec)); - - const rec_t* next_rec = page_rec_get_next(rec); - - if (UNIV_UNLIKELY(!next_rec)) { - ut_ad("corrupted index" == 0); - goto exit_func; - } - - if (page_rec_is_supremum(next_rec)) { - if (!page_has_next(page_align(next_rec))) { - cursor->up_match = 0; - success = true; - } - - goto exit_func; - } - - if (cursor->index()->table->not_redundant()) { - switch (rec_get_status(next_rec)) { - case REC_STATUS_INSTANT: - case REC_STATUS_ORDINARY: - break; - default: - ut_ad("corrupted index" == 0); - goto exit_func; - } - } - - offsets = rec_get_offsets(next_rec, cursor->index(), offsets, - cursor->index()->n_core_fields, - n_unique, &heap); - cmp = cmp_dtuple_rec_with_match( - tuple, next_rec, offsets, &match); - if (mode == PAGE_CUR_LE) { - success = cmp < 0; - cursor->up_match = match; - } else { - success = cmp <= 0; - } - } -exit_func: - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - return(success); -} - /** Clear the adaptive hash index on all pages in the buffer pool. */ inline void buf_pool_t::clear_hash_index() noexcept { @@ -1105,6 +973,60 @@ inline buf_block_t* buf_pool_t::block_from_ahi(const byte *ptr) const noexcept return block; } +/** Fold a prefix given as the number of fields of a tuple. +@param tuple index record +@param cursor B-tree cursor +@return CRC-32C of the record prefix */ +static uint32_t dtuple_fold(const dtuple_t *tuple, const btr_cur_t *cursor) +{ + ut_ad(tuple); + ut_ad(tuple->magic_n == DATA_TUPLE_MAGIC_N); + ut_ad(dtuple_check_typed(tuple)); + + const bool comp= cursor->index()->table->not_redundant(); + uint32_t fold= uint32_t(ut_fold_ull(cursor->index()->id)); + const size_t n_fields= uint16_t(cursor->n_bytes_fields); + + + for (unsigned i= 0; i < n_fields; i++) + { + const dfield_t *field= dtuple_get_nth_field(tuple, i); + const void *data= dfield_get_data(field); + size_t len= dfield_get_len(field); + if (len == UNIV_SQL_NULL) + { + if (UNIV_UNLIKELY(!comp)) + { + len= dtype_get_sql_null_size(dfield_get_type(field), 0); + data= field_ref_zero; + } + else + continue; + } + fold= my_crc32c(fold, data, len); + } + + if (size_t n_bytes= cursor->n_bytes_fields >> 16) + { + const dfield_t *field= dtuple_get_nth_field(tuple, n_fields); + const void *data= dfield_get_data(field); + size_t len= dfield_get_len(field); + if (len == UNIV_SQL_NULL) + { + if (UNIV_UNLIKELY(!comp)) + { + len= dtype_get_sql_null_size(dfield_get_type(field), 0); + data= field_ref_zero; + } + else + return fold; + } + fold= my_crc32c(fold, data, std::min(n_bytes, len)); + } + + return fold; +} + /** Tries to guess the right search position based on the hash search info of the index. Note that if mode is PAGE_CUR_LE, which is used in inserts, and the function returns TRUE, then cursor->up_match and cursor->low_match @@ -1121,8 +1043,8 @@ bool btr_search_guess_on_hash( dict_index_t* index, const dtuple_t* tuple, - ulint mode, - ulint latch_mode, + page_cur_mode_t mode, + btr_latch_mode latch_mode, btr_cur_t* cursor, mtr_t* mtr) noexcept { @@ -1141,8 +1063,8 @@ btr_search_guess_on_hash( static_assert(ulint{BTR_SEARCH_LEAF} == ulint{RW_S_LATCH}, ""); static_assert(ulint{BTR_MODIFY_LEAF} == ulint{RW_X_LATCH}, ""); - cursor->n_fields = index->search_info.n_fields; - cursor->n_bytes = index->search_info.n_bytes; + cursor->n_bytes_fields= index->search_info.left_bytes_fields & + ~buf_block_t::LEFT_SIDE; if (dtuple_get_n_fields(tuple) < btr_search_get_n_fields(cursor)) return false; @@ -1152,7 +1074,7 @@ btr_search_guess_on_hash( #ifdef UNIV_SEARCH_PERF_STAT index->search_info.n_hash_succ++; #endif - ulint fold= dtuple_fold(tuple, cursor->n_fields, cursor->n_bytes, index_id); + const uint32_t fold= dtuple_fold(tuple, cursor); cursor->fold= fold; cursor->flag= BTR_CUR_HASH; btr_sea::partition &part= btr_search.get_part(*index); @@ -1258,7 +1180,7 @@ btr_search_guess_on_hash( /* Check the validity of the guess within the page */ if (index_id != btr_page_get_index_id(block->page.frame) || - !btr_search_check_guess(cursor, tuple, mode)) + cursor->check_mismatch(*tuple, mode, comp)) goto mismatch; uint8_t n_hash_potential= index->search_info.n_hash_potential; @@ -1274,834 +1196,656 @@ btr_search_guess_on_hash( return true; } -void btr_search_drop_page_hash_index(buf_block_t *block, - bool garbage_collect) noexcept +/** The maximum number of rec_fold() values to allocate in stack. +The actual maximum number of records per page is 8189, limited by +the 13-bit heap number field in the record header. */ +static constexpr size_t REC_FOLD_IN_STACK= 128; + +/** Drop any adaptive hash index entries that point to an index page. +@param block latched block containing index page, or a buffer-unfixed + index page or a block in state BUF_BLOCK_REMOVE_HASH +@param garbage_collect drop ahi only if the index is marked as freed +@param folds work area for REC_FOLD_IN_STACK rec_fold() values */ +static void btr_search_drop_page_hash_index(buf_block_t *block, + bool garbage_collect, + uint32_t *folds) noexcept { - ulint n_fields; - ulint n_bytes; - const rec_t* rec; - mem_heap_t* heap; - rec_offs* offsets; - retry: - if (!block->index) { - return; - } - - ut_d(const auto state = block->page.state()); - ut_ad(state == buf_page_t::REMOVE_HASH - || state >= buf_page_t::UNFIXED); - ut_ad(state == buf_page_t::REMOVE_HASH - || !(~buf_page_t::LRU_MASK & state) - || block->page.lock.have_any()); - ut_ad(state < buf_page_t::READ_FIX || state >= buf_page_t::WRITE_FIX); - ut_ad(page_is_leaf(block->page.frame)); - - /* We must not dereference block->index here, because it could be freed - if (!index->table->get_ref_count() && !dict_sys.frozen()). - Determine the ahi_slot based on the block contents. */ - const index_id_t index_id= btr_page_get_index_id(block->page.frame); - btr_sea::partition &part= btr_search.get_part(index_id); - - part.latch.rd_lock(SRW_LOCK_CALL); - - dict_index_t* index= block->index; - - if (!index || !btr_search.enabled) { -unlock_and_return: - part.latch.rd_unlock(); - return; - } + dict_index_t *index= block->index; + if (!index) + return; - const bool is_freed= index->freed(); + ut_d(const auto state= block->page.state()); + ut_ad(state == buf_page_t::REMOVE_HASH || state >= buf_page_t::UNFIXED); + ut_ad(state == buf_page_t::REMOVE_HASH || + !(~buf_page_t::LRU_MASK & state) || block->page.lock.have_any()); + ut_ad(state < buf_page_t::READ_FIX || state >= buf_page_t::WRITE_FIX); + ut_ad(page_is_leaf(block->page.frame)); - if (is_freed) { - part.latch.rd_unlock(); - part.latch.wr_lock(SRW_LOCK_CALL); - if (index != block->index) { - part.latch.wr_unlock(); - goto retry; - } - } - else if (garbage_collect) { - goto unlock_and_return; - } + /* We must not dereference block->index here, because it could be freed + if (!index->table->get_ref_count() && !dict_sys.frozen()). + Determine the ahi_slot based on the block contents. */ + const index_id_t index_id= btr_page_get_index_id(block->page.frame); + btr_sea::partition &part= btr_search.get_part(index_id); - assert_block_ahi_valid(block); + part.latch.rd_lock(SRW_LOCK_CALL); + index= block->index; - if (!index || !btr_search.enabled) { - if (is_freed) { - part.latch.wr_unlock(); - } else { - part.latch.rd_unlock(); - } - return; - } + if (!index) + { + unlock_and_return: + part.latch.rd_unlock(); + return; + } - ut_ad(!index->table->is_temporary()); - ut_ad(btr_search.enabled); + ut_ad(btr_search.enabled); - ut_ad(block->page.id().space() == index->table->space_id); - ut_a(index_id == index->id); - ut_ad(!dict_index_is_ibuf(index)); + bool holding_x= index->freed(); - n_fields = block->curr_n_fields; - n_bytes = block->curr_n_bytes; + if (holding_x) + { + part.latch.rd_unlock(); + part.latch.wr_lock(SRW_LOCK_CALL); + if (index != block->index) + { + part.latch.wr_unlock(); + goto retry; + } + } + else if (garbage_collect) + goto unlock_and_return; - if (!is_freed) { - part.latch.rd_unlock(); - } + assert_block_ahi_valid(block); - ut_a(n_fields > 0 || n_bytes > 0); + ut_ad(!index->table->is_temporary()); - const page_t* const page = block->page.frame; - ulint n_recs = page_get_n_recs(page); - if (!n_recs) { - ut_ad("corrupted adaptive hash index" == 0); - return; - } + ut_ad(block->page.id().space() == index->table->space_id); + ut_a(index_id == index->id); + ut_ad(!index->is_ibuf()); - /* Calculate and cache fold values into an array for fast deletion - from the hash index */ - - const auto comp = page_is_comp(page); - ulint* folds; - ulint n_cached = 0; - ulint prev_fold = 0; - - if (UNIV_LIKELY(comp != 0)) { - rec = page_rec_next_get(page, page + PAGE_NEW_INFIMUM); - if (rec && rec_is_metadata(rec, TRUE)) { - rec = page_rec_next_get(page, rec); -skipped_metadata: - if (!--n_recs) { - /* The page only contains the hidden - metadata record for instant ALTER - TABLE that the adaptive hash index - never points to. */ - folds = nullptr; - goto all_deleted; - } - } - } else { - rec = page_rec_next_get(page, page + PAGE_OLD_INFIMUM); - if (rec && rec_is_metadata(rec, FALSE)) { - rec = page_rec_next_get(page, rec); - goto skipped_metadata; - } - } + const uint32_t left_bytes_fields= block->ahi_left_bytes_fields; - folds = (ulint*) ut_malloc_nokey(n_recs * sizeof(ulint)); - heap = nullptr; - offsets = nullptr; + /* NOTE: block->ahi_left_bytes_fields may change after we release part.latch, + as we might only hold block->page.lock.s_lock()! */ - while (rec) { - if (n_cached >= n_recs) { - ut_ad(page_rec_is_supremum(rec)); - break; - } - ut_ad(page_rec_is_user_rec(rec)); - offsets = rec_get_offsets( - rec, index, offsets, index->n_core_fields, - btr_search_get_n_fields(n_fields, n_bytes), - &heap); - const ulint fold = rec_fold(rec, offsets, n_fields, n_bytes, - index_id); + if (!holding_x) + part.latch.rd_unlock(); - if (fold == prev_fold && prev_fold != 0) { + const uint32_t n_bytes_fields= left_bytes_fields & ~buf_block_t::LEFT_SIDE; + ut_ad(n_bytes_fields); - goto next_rec; - } - - /* Remove all hash nodes pointing to this page from the - hash chain */ - folds[n_cached++] = fold; + const page_t *const page= block->page.frame; + size_t n_folds= 0; + const rec_t *rec; -next_rec: - if (comp) { - rec = page_rec_next_get(page, rec); - if (!rec || rec == page + PAGE_NEW_SUPREMUM) { - break; - } - } else { - rec = page_rec_next_get(page, rec); - if (!rec || rec == page + PAGE_OLD_SUPREMUM) { - break; - } - } - prev_fold = fold; - } + if (page_is_comp(page)) + { + rec= page_rec_next_get(page, page + PAGE_NEW_INFIMUM); - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } + if (rec && rec_is_metadata(rec, true)) + { + ut_ad(index->is_instant()); + rec= page_rec_next_get(page, rec); + } -all_deleted: - if (!is_freed) { - part.latch.wr_lock(SRW_LOCK_CALL); + while (rec && rec != page + PAGE_NEW_SUPREMUM) + { + next_not_redundant: + folds[n_folds]= rec_fold(rec, *index, n_bytes_fields); + rec= page_rec_next_get(page, rec); + if (!n_folds) + n_folds++; + else if (folds[n_folds] == folds[n_folds - 1]); + else if (++n_folds == REC_FOLD_IN_STACK) + break; + } + } + else + { + rec= page_rec_next_get(page, page + PAGE_OLD_INFIMUM); - if (UNIV_UNLIKELY(!block->index)) { - /* Someone else has meanwhile dropped the - hash index */ - goto cleanup; - } + if (rec && rec_is_metadata(rec, false)) + { + ut_ad(index->is_instant()); + rec= page_rec_next_get(page, rec); + } - ut_a(block->index == index); - } + while (rec && rec != page + PAGE_OLD_SUPREMUM) + { + next_redundant: + folds[n_folds]= rec_fold(rec, *index, n_bytes_fields); + rec= page_rec_next_get(page, rec); + if (!n_folds) + n_folds++; + else if (folds[n_folds] == folds[n_folds - 1]); + else if (++n_folds == REC_FOLD_IN_STACK) + break; + } + } - if (block->curr_n_fields != n_fields - || block->curr_n_bytes != n_bytes) { + if (!holding_x) + { + part.latch.wr_lock(SRW_LOCK_CALL); + if (UNIV_UNLIKELY(!block->index)) + /* Someone else has meanwhile dropped the hash index */ + goto cleanup; + ut_a(block->index == index); + } - /* Someone else has meanwhile built a new hash index on the - page, with different parameters */ + if (block->ahi_left_bytes_fields != left_bytes_fields) + { + /* Someone else has meanwhile built a new hash index on the page, + with different parameters */ + part.latch.wr_unlock(); + goto retry; + } - part.latch.wr_unlock(); + MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_REMOVED, n_folds); - ut_free(folds); - goto retry; - } + while (n_folds) + ha_remove_all_nodes_to_page(part, folds[--n_folds], page); - for (ulint i = 0; i < n_cached; i++) { - ha_remove_all_nodes_to_page(part, folds[i], page); - } + if (!rec); + else if (page_is_comp(page)) + { + if (rec != page + PAGE_NEW_SUPREMUM) + { + holding_x= true; + goto next_not_redundant; + } + } + else if (rec != page + PAGE_OLD_SUPREMUM) + { + holding_x= true; + goto next_redundant; + } - switch (index->search_info.ref_count--) { - case 0: - ut_error; - case 1: - if (index->freed()) { - btr_search_lazy_free(index); - } - } + switch (index->search_info.ref_count--) { + case 0: + ut_error; + case 1: + if (index->freed()) + btr_search_lazy_free(index); + } - block->index = nullptr; + ut_ad(!block->n_pointers); + block->index= nullptr; - MONITOR_INC(MONITOR_ADAPTIVE_HASH_PAGE_REMOVED); - MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_REMOVED, n_cached); + MONITOR_INC(MONITOR_ADAPTIVE_HASH_PAGE_REMOVED); cleanup: - assert_block_ahi_valid(block); - part.latch.wr_unlock(); - ut_free(folds); + assert_block_ahi_valid(block); + part.latch.wr_unlock(); +} + +void btr_search_drop_page_hash_index(buf_block_t *block, + bool garbage_collect) noexcept +{ + uint32_t folds[REC_FOLD_IN_STACK]; + btr_search_drop_page_hash_index(block, garbage_collect, folds); } void btr_search_drop_page_hash_when_freed(const page_id_t page_id) noexcept { - buf_block_t* block; - mtr_t mtr; - - mtr_start(&mtr); - - /* If the caller has a latch on the page, then the caller must - have a x-latch on the page and it must have already dropped - the hash index for the page. Because of the x-latch that we - are possibly holding, we cannot s-latch the page, but must - (recursively) x-latch it, even though we are only reading. */ - - block = buf_page_get_gen(page_id, 0, RW_X_LATCH, NULL, - BUF_PEEK_IF_IN_POOL, &mtr); - - if (block && block->index) { - /* In all our callers, the table handle should - be open, or we should be in the process of - dropping the table (preventing eviction). */ - DBUG_ASSERT(block->index->table->get_ref_count() - || dict_sys.locked()); - btr_search_drop_page_hash_index(block, false); - } + mtr_t mtr; + mtr.start(); + /* If the caller has a latch on the page, then the caller must be an + x-latch page and it must have already dropped the hash index for the + page. Because of the x-latch that we are possibly holding, we must + (recursively) x-latch it, even though we are only reading. */ + if (buf_block_t *block= buf_page_get_gen(page_id, 0, RW_X_LATCH, nullptr, + BUF_PEEK_IF_IN_POOL, &mtr)) + { + if (IF_DBUG(dict_index_t *index=,) block->index) + { + /* In all our callers, the table handle should be open, or we + should be in the process of dropping the table (preventing + eviction). */ + DBUG_ASSERT(index->table->get_ref_count() || dict_sys.locked()); + btr_search_drop_page_hash_index(block, false); + } + } - mtr_commit(&mtr); + mtr.commit(); } /** Build a hash index on a page with the given parameters. If the page already has a hash index with different parameters, the old hash index is removed. If index is non-NULL, this function checks if n_fields and n_bytes are sensible, and does not build a hash index if not. -@param[in,out] index index for which to build. -@param[in,out] block index page, s-/x- latched. -@param[in,out] part the adaptive search partition -@param[in] n_fields hash this many full fields -@param[in] n_bytes hash this many bytes of the next field -@param[in] left_side hash for searches from left side */ -static -void -btr_search_build_page_hash_index( - dict_index_t* index, - buf_block_t* block, - btr_sea::partition& part, - uint16_t n_fields, - uint16_t n_bytes, - bool left_side) noexcept +@param index B-tree index +@param block latched B-tree leaf page +@param part the adaptive search partition +@param left_bytes_fields hash parameters */ +static void btr_search_build_page_hash_index(dict_index_t *index, + buf_block_t *block, + btr_sea::partition &part, + const uint32_t left_bytes_fields) + noexcept { - const rec_t* rec; - ulint fold; - ulint next_fold; - ulint n_cached; - ulint n_recs; - ulint* folds; - const rec_t** recs; - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - - ut_ad(!index->table->is_temporary()); - - if (!btr_search.enabled) { - return; - } - - rec_offs_init(offsets_); - ut_ad(&part == &btr_search.get_part(*index)); - ut_ad(index); - ut_ad(block->page.id().space() == index->table->space_id); - ut_ad(!dict_index_is_ibuf(index)); - ut_ad(page_is_leaf(block->page.frame)); - - ut_ad(block->page.lock.have_x() || block->page.lock.have_s()); - ut_ad(block->page.id().page_no() >= 3); - - part.latch.rd_lock(SRW_LOCK_CALL); - - const bool enabled = btr_search.enabled; - const bool rebuild = block->index - && (block->curr_n_fields != n_fields - || block->curr_n_bytes != n_bytes - || block->curr_left_side != left_side); - - part.latch.rd_unlock(); - - if (!enabled) { - return; - } - - if (rebuild) { - btr_search_drop_page_hash_index(block, false); - } - - /* Check that the values for hash index build are sensible */ - - if (n_fields == 0 && n_bytes == 0) { - - return; - } - - if (dict_index_get_n_unique_in_tree(index) - < btr_search_get_n_fields(n_fields, n_bytes)) { - return; - } - - page_t* page = buf_block_get_frame(block); - n_recs = page_get_n_recs(page); - - if (n_recs == 0) { - - return; - } - - rec = page_rec_get_next_const(page_get_infimum_rec(page)); - if (!rec) return; - - if (rec_is_metadata(rec, *index)) { - rec = page_rec_get_next_const(rec); - if (!rec || !--n_recs) return; - } - - /* Calculate and cache fold values and corresponding records into - an array for fast insertion to the hash index */ + ut_ad(!index->table->is_temporary()); - folds = static_cast(ut_malloc_nokey(n_recs * sizeof *folds)); - recs = static_cast( - ut_malloc_nokey(n_recs * sizeof *recs)); + if (!btr_search.enabled) + return; - n_cached = 0; + ut_ad(block->page.id().space() == index->table->space_id); + ut_ad(!index->is_ibuf()); + ut_ad(page_is_leaf(block->page.frame)); + ut_ad(block->page.lock.have_any()); + ut_ad(block->page.id().page_no() >= 3); + ut_ad(&part == &btr_search.get_part(*index)); - ut_a(index->id == btr_page_get_index_id(page)); + part.latch.rd_lock(SRW_LOCK_CALL); - offsets = rec_get_offsets( - rec, index, offsets, index->n_core_fields, - btr_search_get_n_fields(n_fields, n_bytes), - &heap); - ut_ad(page_rec_is_supremum(rec) - || n_fields == rec_offs_n_fields(offsets) - (n_bytes > 0)); + const dict_index_t *const block_index= block->index; + const bool rebuild= block_index && + (block_index != index || + block->ahi_left_bytes_fields != left_bytes_fields); + const bool enabled= btr_search.enabled; - fold = rec_fold(rec, offsets, n_fields, n_bytes, index->id); + part.latch.rd_unlock(); - if (left_side) { + if (!enabled) + return; - folds[n_cached] = fold; - recs[n_cached] = rec; - n_cached++; - } + struct{uint32_t fold;uint32_t offset;} fr[REC_FOLD_IN_STACK / 2]; - while (const rec_t* next_rec = page_rec_get_next_const(rec)) { - if (page_rec_is_supremum(next_rec)) { + if (rebuild) + btr_search_drop_page_hash_index(block, false, &fr[0].fold); - if (!left_side) { + const uint32_t n_bytes_fields{left_bytes_fields & ~buf_block_t::LEFT_SIDE}; - folds[n_cached] = fold; - recs[n_cached] = rec; - n_cached++; - } + /* Check that the values for hash index build are sensible */ + ut_ad(n_bytes_fields); - break; - } + if (dict_index_get_n_unique_in_tree(index) < + btr_search_get_n_fields(n_bytes_fields)) + return; - offsets = rec_get_offsets( - next_rec, index, offsets, index->n_core_fields, - btr_search_get_n_fields(n_fields, n_bytes), &heap); - next_fold = rec_fold(next_rec, offsets, n_fields, - n_bytes, index->id); + const page_t *const page= block->page.frame; + size_t n_cached= 0; + const rec_t *rec; - if (fold != next_fold) { - /* Insert an entry into the hash index */ + if (page_is_comp(page)) + { + rec= page_rec_next_get(page, page + PAGE_NEW_INFIMUM); - if (left_side) { + if (rec && rec_is_metadata(rec, true)) + { + ut_ad(index->is_instant()); + rec= page_rec_next_get(page, rec); + } - folds[n_cached] = next_fold; - recs[n_cached] = next_rec; - n_cached++; - } else { - folds[n_cached] = fold; - recs[n_cached] = rec; - n_cached++; - } - } + while (rec && rec != page + PAGE_NEW_SUPREMUM) + { + next_not_redundant: + const uint32_t offset= uint32_t(uintptr_t(rec)); + fr[n_cached]= {rec_fold(rec, *index, n_bytes_fields), offset}; + rec= page_rec_next_get(page, rec); + if (!n_cached) + n_cached= 1; + else if (fr[n_cached - 1].fold == fr[n_cached].fold) + { + if (!(left_bytes_fields & buf_block_t::LEFT_SIDE)) + fr[n_cached - 1].offset= offset; + } + else if (++n_cached == array_elements(fr)) + break; + } + } + else + { + rec= page_rec_next_get(page, page + PAGE_OLD_INFIMUM); - rec = next_rec; - fold = next_fold; - } + if (rec && rec_is_metadata(rec, false)) + { + ut_ad(index->is_instant()); + rec= page_rec_next_get(page, rec); + } - part.prepare_insert(); + while (rec && rec != page + PAGE_OLD_SUPREMUM) + { + next_redundant: + const uint32_t offset= uint32_t(uintptr_t(rec)); + fr[n_cached]= {rec_fold(rec, *index, n_bytes_fields), offset}; + rec= page_rec_next_get(page, rec); + if (!n_cached) + n_cached= 1; + else if (fr[n_cached - 1].fold == fr[n_cached].fold) + { + if (!(left_bytes_fields & buf_block_t::LEFT_SIDE)) + fr[n_cached - 1].offset= offset; + } + else if (++n_cached == array_elements(fr)) + break; + } + } - part.latch.wr_lock(SRW_LOCK_CALL); + part.prepare_insert(); + part.latch.wr_lock(SRW_LOCK_CALL); - if (!btr_search.enabled) { - goto exit_func; - } + if (!block->index) + { + if (!btr_search.enabled) + goto exit_func; + ut_ad(!block->n_pointers); + index->search_info.ref_count++; + } + else if (block->ahi_left_bytes_fields != left_bytes_fields) + goto exit_func; - /* This counter is decremented every time we drop page - hash index entries and is incremented here. Since we can - rebuild hash index for a page that is already hashed, we - have to take care not to increment the counter in that - case. */ - if (!block->index) { - assert_block_ahi_empty(block); - index->search_info.ref_count++; - } else if (block->curr_n_fields != n_fields - || block->curr_n_bytes != n_bytes - || block->curr_left_side != left_side) { - goto exit_func; - } + block->n_hash_helps= 0; + block->index= index; + block->ahi_left_bytes_fields= left_bytes_fields; - block->n_hash_helps = 0; + MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_ADDED, n_cached); - block->curr_n_fields = n_fields & dict_index_t::MAX_N_FIELDS; - block->curr_n_bytes = n_bytes & ((1U << 15) - 1); - block->curr_left_side = left_side; - block->index = index; + while (n_cached) + { +#if SIZEOF_SIZE_T <= 4 + const auto &f= fr[--n_cached]; + const rec_t *rec= reinterpret_cast(f.offset); +#else + const auto f= fr[--n_cached]; + const rec_t *rec= page + (uint32_t(uintptr_t(page)) ^ f.offset); +#endif + ha_insert_for_fold(part, f.fold, block, rec); + } - for (ulint i = 0; i < n_cached; i++) { - ha_insert_for_fold(part, folds[i], block, recs[i]); - } + if (!rec); + else if (page_is_comp(page)) + { + if (rec != page + PAGE_NEW_SUPREMUM) + { + part.latch.wr_unlock(); + goto next_not_redundant; + } + } + else if (rec != page + PAGE_OLD_SUPREMUM) + { + part.latch.wr_unlock(); + goto next_redundant; + } - MONITOR_INC(MONITOR_ADAPTIVE_HASH_PAGE_ADDED); - MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_ADDED, n_cached); + MONITOR_INC(MONITOR_ADAPTIVE_HASH_PAGE_ADDED); exit_func: - assert_block_ahi_valid(block); - part.latch.wr_unlock(); - ut_free(folds); - ut_free(recs); - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } + assert_block_ahi_valid(block); + part.latch.wr_unlock(); } void btr_cur_t::search_info_update() const noexcept { - /* NOTE that the following two function calls do NOT protect - info or block->n_fields etc. with any semaphore, to save CPU time! - We cannot assume the fields are consistent when we return from - those functions! */ - - btr_search_info_update_hash(this); - - bool build_index = btr_search_update_block_hash_info( - &index()->search_info, page_cur.block); - - if (flag == BTR_CUR_HASH_FAIL) { - /* Update the hash node reference, if appropriate */ -#ifdef UNIV_SEARCH_PERF_STAT - btr_search_n_hash_fail++; -#endif /* UNIV_SEARCH_PERF_STAT */ - btr_search_update_hash_ref(this); - } - - if (build_index) { - /* Note that since we did not protect block->n_fields etc. - with any semaphore, the values can be inconsistent. We have - to check inside the function call that they make sense. */ - btr_search_build_page_hash_index(index(), page_cur.block, - btr_search.get_part(*index()), - page_cur.block->n_fields, - page_cur.block->n_bytes, - page_cur.block->left_side); - } + if (uint32_t left_bytes_fields= btr_search_info_update_hash(*this)) + btr_search_build_page_hash_index(index(), page_cur.block, + btr_search.get_part(*index()), + left_bytes_fields); } void btr_search_move_or_delete_hash_entries(buf_block_t *new_block, buf_block_t *block) noexcept { - ut_ad(block->page.lock.have_x()); - ut_ad(new_block->page.lock.have_x()); - - if (!btr_search.enabled) { - return; - } - - dict_index_t* index = block->index; - if (!index) { - index = new_block->index; - } else { - ut_ad(!new_block->index || index == new_block->index); - } - assert_block_ahi_valid(block); - assert_block_ahi_valid(new_block); - - if (new_block->index) { -drop_exit: - btr_search_drop_page_hash_index(block, false); - return; - } + ut_ad(block->page.lock.have_x()); + ut_ad(new_block->page.lock.have_x()); - if (!index) { - return; - } + if (!btr_search.enabled) + return; - btr_sea::partition& part = btr_search.get_part(*index); - part.latch.rd_lock(SRW_LOCK_CALL); + dict_index_t *index= block->index, *new_block_index= new_block->index; - if (index->freed()) { - part.latch.rd_unlock(); - goto drop_exit; - } + assert_block_ahi_valid(block); + assert_block_ahi_valid(new_block); - if (block->index) { - uint16_t n_fields = block->curr_n_fields; - uint16_t n_bytes = block->curr_n_bytes; - bool left_side = block->curr_left_side; + if (new_block_index) + { + ut_ad(!index || index == new_block_index); +drop_exit: + btr_search_drop_page_hash_index(block, false); + return; + } - new_block->n_fields = block->curr_n_fields; - new_block->n_bytes = block->curr_n_bytes; - new_block->left_side = left_side; + if (!index) + return; - part.latch.rd_unlock(); + btr_sea::partition &part= btr_search.get_part(*index); + part.latch.rd_lock(SRW_LOCK_CALL); - ut_a(n_fields > 0 || n_bytes > 0); + if (index->freed()) + { + part.latch.rd_unlock(); + goto drop_exit; + } - btr_search_build_page_hash_index( - index, new_block, part, - n_fields, n_bytes, left_side); - ut_ad(n_fields == block->curr_n_fields); - ut_ad(n_bytes == block->curr_n_bytes); - ut_ad(left_side == block->curr_left_side); - return; - } + if (ut_d(dict_index_t *block_index=) block->index) + { + ut_ad(block_index == index); + const uint32_t left_bytes_fields{block->ahi_left_bytes_fields}; + ut_ad(left_bytes_fields & ~buf_block_t::LEFT_SIDE); + part.latch.rd_unlock(); + btr_search_build_page_hash_index(index, new_block, part, + left_bytes_fields); + return; + } - part.latch.rd_unlock(); + part.latch.rd_unlock(); } void btr_search_update_hash_on_delete(btr_cur_t *cursor) noexcept { - buf_block_t* block; - const rec_t* rec; - ulint fold; - dict_index_t* index; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - mem_heap_t* heap = NULL; - rec_offs_init(offsets_); - - ut_ad(page_is_leaf(btr_cur_get_page(cursor))); - - if (!btr_search.enabled) { - return; - } - - block = btr_cur_get_block(cursor); - - ut_ad(block->page.lock.have_x()); - - assert_block_ahi_valid(block); - index = block->index; - - if (!index) { - - return; - } - - ut_ad(!cursor->index()->table->is_temporary()); - - if (index != cursor->index()) { - btr_search_drop_page_hash_index(block, false); - return; - } - - ut_ad(block->page.id().space() == index->table->space_id); - ut_a(index == cursor->index()); - ut_a(block->curr_n_fields > 0 || block->curr_n_bytes > 0); - ut_ad(!dict_index_is_ibuf(index)); - - rec = btr_cur_get_rec(cursor); - - fold = rec_fold(rec, rec_get_offsets(rec, index, offsets_, - index->n_core_fields, - ULINT_UNDEFINED, &heap), - block->curr_n_fields, block->curr_n_bytes, index->id); - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - - btr_sea::partition& part = btr_search.get_part(*index); - - part.latch.wr_lock(SRW_LOCK_CALL); - assert_block_ahi_valid(block); - - if (block->index) { - ut_ad(btr_search.enabled); - ut_a(block->index == index); - - if (part.erase(fold, rec)) { - MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_REMOVED); - } else { - MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_REMOVE_NOT_FOUND); - } - } else { - part.latch.wr_unlock(); - } -} - -void btr_search_update_hash_node_on_insert(btr_cur_t *cursor) noexcept -{ - buf_block_t* block; - dict_index_t* index; - rec_t* rec; - - rec = btr_cur_get_rec(cursor); - - block = btr_cur_get_block(cursor); - - ut_ad(block->page.lock.have_x()); - - index = block->index; - - if (!index) { - return; - } + ut_ad(page_is_leaf(btr_cur_get_page(cursor))); + if (!btr_search.enabled) + return; + buf_block_t *block= btr_cur_get_block(cursor); - ut_ad(btr_search.enabled); - ut_ad(!cursor->index()->table->is_temporary()); + ut_ad(block->page.lock.have_x()); - if (index != cursor->index()) { - ut_ad(index->id == cursor->index()->id); - btr_search_drop_page_hash_index(block, false); - return; - } + assert_block_ahi_valid(block); + dict_index_t *index= block->index; + if (!index) + return; + ut_ad(!cursor->index()->table->is_temporary()); - ut_ad(!dict_index_is_ibuf(index)); - btr_sea::partition& part = btr_search.get_part(*index); - part.latch.wr_lock(SRW_LOCK_CALL); + if (UNIV_UNLIKELY(index != cursor->index())) + { + btr_search_drop_page_hash_index(block, false); + return; + } - if (!block->index || !btr_search.enabled) { + ut_ad(block->page.id().space() == index->table->space_id); + const uint32_t n_bytes_fields= + block->ahi_left_bytes_fields & ~buf_block_t::LEFT_SIDE; + ut_ad(n_bytes_fields); + ut_ad(!index->is_ibuf()); - goto func_exit; - } - - ut_a(block->index == index); - - if ((cursor->flag == BTR_CUR_HASH) - && (cursor->n_fields == block->curr_n_fields) - && (cursor->n_bytes == block->curr_n_bytes) - && !block->curr_left_side) { - if (const rec_t *new_rec = page_rec_get_next_const(rec)) { - if (ha_search_and_update_if_found( - &part.table, - cursor->fold, rec, block, new_rec)) { - MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_UPDATED); - } - } else { - ut_ad("corrupted page" == 0); - } + const rec_t *rec= btr_cur_get_rec(cursor); + uint32_t fold= rec_fold(rec, *index, n_bytes_fields, + page_is_comp(btr_cur_get_page(cursor))); + btr_sea::partition &part= btr_search.get_part(*index); + part.latch.wr_lock(SRW_LOCK_CALL); + assert_block_ahi_valid(block); -func_exit: - assert_block_ahi_valid(block); - part.latch.wr_unlock(); - } else { - part.latch.wr_unlock(); - btr_search_update_hash_on_insert(cursor); - } + if (ut_d(dict_index_t *block_index=) block->index) + { + ut_ad(btr_search.enabled); + ut_ad(block_index == index); + if (part.erase(fold, rec)) + { + MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_REMOVED); + } + else + { + MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_REMOVE_NOT_FOUND); + } + } + else + part.latch.wr_unlock(); } -void btr_search_update_hash_on_insert(btr_cur_t *cursor) noexcept +void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept { - buf_block_t* block; - dict_index_t* index; - const rec_t* rec; - const rec_t* ins_rec; - const rec_t* next_rec; - ulint fold; - ulint ins_fold; - ulint next_fold = 0; /* remove warning (??? bug ???) */ - ulint n_fields; - ulint n_bytes; - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - rec_offs_init(offsets_); - - ut_ad(page_is_leaf(btr_cur_get_page(cursor))); - - if (!btr_search.enabled) { - return; - } - - block = btr_cur_get_block(cursor); - - ut_ad(block->page.lock.have_x()); - assert_block_ahi_valid(block); - - index = block->index; + ut_ad(!cursor->index()->table->is_temporary()); + ut_ad(page_is_leaf(btr_cur_get_page(cursor))); - if (!index) { + buf_block_t *block= btr_cur_get_block(cursor); - return; - } - - ut_ad(block->page.id().space() == index->table->space_id); - - rec = btr_cur_get_rec(cursor); - - ut_ad(!cursor->index()->table->is_temporary()); - - if (index != cursor->index()) { - ut_ad(index->id == cursor->index()->id); -drop: - btr_search_drop_page_hash_index(block, false); - return; - } + ut_ad(block->page.lock.have_x()); + assert_block_ahi_valid(block); - ut_a(index == cursor->index()); - ut_ad(!dict_index_is_ibuf(index)); - - n_fields = block->curr_n_fields; - n_bytes = block->curr_n_bytes; - const bool left_side = block->curr_left_side; - - ins_rec = page_rec_get_next_const(rec); - if (UNIV_UNLIKELY(!ins_rec)) goto drop; - next_rec = page_rec_get_next_const(ins_rec); - if (UNIV_UNLIKELY(!next_rec)) goto drop; - - offsets = rec_get_offsets(ins_rec, index, offsets, - index->n_core_fields, - ULINT_UNDEFINED, &heap); - ins_fold = rec_fold(ins_rec, offsets, n_fields, n_bytes, index->id); - - if (!page_rec_is_supremum(next_rec)) { - offsets = rec_get_offsets( - next_rec, index, offsets, index->n_core_fields, - btr_search_get_n_fields(n_fields, n_bytes), &heap); - next_fold = rec_fold(next_rec, offsets, n_fields, - n_bytes, index->id); - } + dict_index_t *index= block->index; - btr_sea::partition &part= btr_search.get_part(*index); - bool locked = false; - part.prepare_insert(); - - if (!page_rec_is_infimum(rec) && !rec_is_metadata(rec, *index)) { - offsets = rec_get_offsets( - rec, index, offsets, index->n_core_fields, - btr_search_get_n_fields(n_fields, n_bytes), &heap); - fold = rec_fold(rec, offsets, n_fields, n_bytes, index->id); - } else { - if (left_side) { - locked = true; - part.latch.wr_lock(SRW_LOCK_CALL); - - if (!block->index) { - goto function_exit; - } - - ha_insert_for_fold(part, ins_fold, block, ins_rec); - MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); - } - - goto check_next_rec; - } - - if (fold != ins_fold) { + if (!index) + return; - if (!locked) { - locked = true; - part.latch.wr_lock(SRW_LOCK_CALL); + ut_ad(block->page.id().space() == index->table->space_id); + const rec_t *rec= btr_cur_get_rec(cursor); - if (!block->index) { - goto function_exit; - } - } - - if (!left_side) { - ha_insert_for_fold(part, fold, block, rec); - } else { - ha_insert_for_fold(part, ins_fold, block, ins_rec); - } - MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); - } + if (UNIV_UNLIKELY(index != cursor->index())) + { + ut_ad(index->id == cursor->index()->id); + drop: + btr_search_drop_page_hash_index(block, false); + return; + } -check_next_rec: - if (page_rec_is_supremum(next_rec)) { + ut_ad(!index->is_ibuf()); - if (!left_side) { - if (!locked) { - locked = true; - part.latch.wr_lock(SRW_LOCK_CALL); + btr_sea::partition &part= btr_search.get_part(*index); + bool locked= false; + const uint32_t left_bytes_fields{block->ahi_left_bytes_fields}; + const uint32_t n_bytes_fields{left_bytes_fields & ~buf_block_t::LEFT_SIDE}; + const page_t *const page= block->page.frame; + const rec_t *ins_rec; + const rec_t *next_rec; + uint32_t ins_fold, next_fold= 0, fold; + bool next_is_supremum, rec_valid; + + if (!reorg && cursor->flag == BTR_CUR_HASH && + left_bytes_fields == cursor->n_bytes_fields) + { + part.latch.wr_lock(SRW_LOCK_CALL); + if (!block->index) + goto unlock_exit; + ut_ad(btr_search.enabled); + locked= true; + if (page_is_comp(page)) + { + ins_rec= page_rec_next_get(page, rec); + update_on_insert: + /* The adaptive hash index is allowed to be a subset of what is + actually present in the index page. It could happen that there are + several INSERT with the same rec_fold() value, especially if the + record prefix identified by n_bytes_fields is being duplicated + in each INSERT. Therefore, we may fail to find the old rec + (and fail to update the AHI to point to to our ins_rec). */ + if (ins_rec && + ha_search_and_update_if_found(&part.table, + cursor->fold, rec, block, ins_rec)) + { + MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_UPDATED); + } - if (!block->index) { - goto function_exit; - } - } + assert_block_ahi_valid(block); + goto unlock_exit; + } + else + { + ins_rec= page_rec_next_get(page, rec); + goto update_on_insert; + } + } - ha_insert_for_fold(part, ins_fold, block, ins_rec); - MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); - } + if (page_is_comp(page)) + { + ins_rec= page_rec_next_get(page, rec); + if (UNIV_UNLIKELY(!ins_rec)) goto drop; + next_rec= page_rec_next_get(page, ins_rec); + if (UNIV_UNLIKELY(!next_rec)) goto drop; + ins_fold= rec_fold(ins_rec, *index, n_bytes_fields); + next_is_supremum= next_rec == page + PAGE_NEW_SUPREMUM; + if (!next_is_supremum) + next_fold= rec_fold(next_rec, *index, n_bytes_fields); + rec_valid= rec != page + PAGE_NEW_INFIMUM && !rec_is_metadata(rec, true); + if (rec_valid) + fold= rec_fold(rec, *index, n_bytes_fields); + } + else + { + ins_rec= page_rec_next_get(page, rec); + if (UNIV_UNLIKELY(!ins_rec)) goto drop; + next_rec= page_rec_next_get(page, ins_rec); + if (UNIV_UNLIKELY(!next_rec)) goto drop; + ins_fold= rec_fold(ins_rec, *index, n_bytes_fields); + next_is_supremum= next_rec == page + PAGE_OLD_SUPREMUM; + if (!next_is_supremum) + next_fold= rec_fold(next_rec, *index, n_bytes_fields); + rec_valid= rec != page + PAGE_OLD_INFIMUM && !rec_is_metadata(rec, false); + if (rec_valid) + fold= rec_fold(rec, *index, n_bytes_fields); + } - goto function_exit; - } + part.prepare_insert(); - if (ins_fold != next_fold) { - if (!locked) { - locked = true; - part.latch.wr_lock(SRW_LOCK_CALL); + if (!rec_valid) + { + if (left_bytes_fields & buf_block_t::LEFT_SIDE) + { + locked= true; + part.latch.wr_lock(SRW_LOCK_CALL); + if (!block->index) + goto unlock_exit; + ha_insert_for_fold(part, ins_fold, block, ins_rec); + MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); + } + } + else if (fold != ins_fold) + { + if (!locked) + { + locked= true; + part.latch.wr_lock(SRW_LOCK_CALL); + if (!block->index) + goto unlock_exit; + } + if (left_bytes_fields & buf_block_t::LEFT_SIDE) + fold= ins_fold, rec= ins_rec; + ha_insert_for_fold(part, fold, block, rec); + MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); + } - if (!block->index) { - goto function_exit; - } - } + if (next_is_supremum) + { + if (!(left_bytes_fields & ~buf_block_t::LEFT_SIDE)) + { + if (!locked) + { + locked= true; + part.latch.wr_lock(SRW_LOCK_CALL); + if (!block->index) + goto unlock_exit; + } + ha_insert_for_fold(part, ins_fold, block, ins_rec); + MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); + } + } + else if (ins_fold != next_fold) + { + if (!locked) + { + locked= true; + part.latch.wr_lock(SRW_LOCK_CALL); + if (!block->index) + goto unlock_exit; + } + if (!(left_bytes_fields & ~buf_block_t::LEFT_SIDE)) + next_fold= ins_fold, next_rec= ins_rec; + ha_insert_for_fold(part, next_fold, block, next_rec); + MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); + } - if (!left_side) { - ha_insert_for_fold(part, ins_fold, block, ins_rec); - } else { - ha_insert_for_fold(part, next_fold, block, next_rec); - } - MONITOR_INC(MONITOR_ADAPTIVE_HASH_ROW_ADDED); - } + ut_ad(!locked || index == block->index); -function_exit: - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - if (locked) { - part.latch.wr_unlock(); - } + if (locked) + unlock_exit: + part.latch.wr_unlock(); } # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG @@ -2121,9 +1865,10 @@ static bool ha_validate(const hash_table_t *table, node= node->next) { if (table->calc_hash(node->fold) != i) { - ib::error() << "Hash table node fold value " << node->fold - << " does not match the cell number " << i; - ok= false; + sql_print_error("InnoDB: Hash table node fold value " UINT32PF + " does not match the cell number %zu", + node->fold, i); + ok= false; } } } @@ -2156,19 +1901,11 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) bool ok = true; ulint i; ulint cell_count; - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; btr_search_x_lock_all(); if (!btr_search.enabled || (thd && thd_kill_level(thd))) { func_exit: btr_search_x_unlock_all(); - - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - return ok; } @@ -2176,8 +1913,6 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) search latches. */ ulint chunk_size = 10000; - rec_offs_init(offsets_); - mysql_mutex_lock(&buf_pool.mutex); btr_sea::partition& part = btr_search.parts[hash_table_id]; @@ -2245,26 +1980,19 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) invokes btr_search_drop_page_hash_index(). */ ut_a(block->page.state() == buf_page_t::REMOVE_HASH); state_ok: - ut_ad(!dict_index_is_ibuf(block->index)); - ut_ad(block->page.id().space() - == block->index->table->space_id); + const dict_index_t* index = block->index; + ut_ad(!index->is_ibuf()); + ut_ad(block->page.id().space() == index->table->space_id); const page_t* page = block->page.frame; page_index_id = btr_page_get_index_id(page); - offsets = rec_get_offsets( - node->rec, block->index, offsets, - block->index->n_core_fields, - btr_search_get_n_fields(block->curr_n_fields, - block->curr_n_bytes), - &heap); - - const ulint fold = rec_fold( - node->rec, offsets, - block->curr_n_fields, - block->curr_n_bytes, - page_index_id); + const uint32_t fold = rec_fold( + node->rec, *block->index, + block->ahi_left_bytes_fields + & ~buf_block_t::LEFT_SIDE, + page_is_comp(page)); if (node->fold != fold) { ok = FALSE; @@ -2278,16 +2006,6 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id) << ", index id " << page_index_id << ", node fold " << node->fold << ", rec fold " << fold; - - fputs("InnoDB: Record ", stderr); - rec_print_new(stderr, node->rec, offsets); - fprintf(stderr, "\nInnoDB: on that page." - " Page mem address %p, is hashed %p," - " n fields %lu\n" - "InnoDB: side %lu\n", - (void*) page, (void*) block->index, - (ulong) block->curr_n_fields, - (ulong) block->curr_left_side); ut_ad(0); } } @@ -2351,13 +2069,11 @@ bool btr_search_check_marked_free_index(const buf_block_t *block) noexcept { const index_id_t index_id= btr_page_get_index_id(block->page.frame); btr_sea::partition &part= btr_search.get_part(index_id); - + bool is_freed= false; part.latch.rd_lock(SRW_LOCK_CALL); - - bool is_freed= block->index && block->index->freed(); - + if (dict_index_t *index= block->index) + is_freed= index->freed(); part.latch.rd_unlock(); - return is_freed; } # endif /* UNIV_DEBUG */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index a30c7e3ff5bd6..ce929527b2125 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -853,6 +853,26 @@ void buf_page_print(const byte *read_buf, ulint zip_size) noexcept #endif } +#ifdef BTR_CUR_HASH_ADAPT + +/** Ensure that some adaptive hash index fields are initialized */ +static void buf_block_init_low(buf_block_t *block) noexcept +{ + /* No adaptive hash index entries may point to a previously unused + (and now freshly allocated) block. */ + MEM_MAKE_DEFINED(&block->index, sizeof block->index); + MEM_MAKE_DEFINED(&block->n_pointers, sizeof block->n_pointers); + MEM_MAKE_DEFINED(&block->n_hash_helps, sizeof block->n_hash_helps); +# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG + ut_a(!block->index); + ut_a(!block->n_pointers); + ut_a(!block->n_hash_helps); +# endif +} +#else /* BTR_CUR_HASH_ADAPT */ +inline void buf_block_init_low(buf_block_t*) {} +#endif /* BTR_CUR_HASH_ADAPT */ + /** Initialize a buffer page descriptor. @param[in,out] block buffer page descriptor @param[in] frame buffer page frame */ @@ -862,8 +882,7 @@ buf_block_init(buf_block_t* block, byte* frame) { /* This function should only be executed at database startup or by buf_pool.resize(). Either way, adaptive hash index must not exist. */ - assert_block_ahi_empty_on_init(block); - + buf_block_init_low(block); block->page.frame = frame; MEM_MAKE_DEFINED(&block->modify_clock, sizeof block->modify_clock); @@ -871,10 +890,6 @@ buf_block_init(buf_block_t* block, byte* frame) MEM_MAKE_DEFINED(&block->page.lock, sizeof block->page.lock); block->page.lock.init(); block->page.init(buf_page_t::NOT_USED, page_id_t(~0ULL)); -#ifdef BTR_CUR_HASH_ADAPT - MEM_MAKE_DEFINED(&block->index, sizeof block->index); - ut_ad(!block->index); -#endif /* BTR_CUR_HASH_ADAPT */ ut_d(block->in_unzip_LRU_list = false); ut_d(block->in_withdraw_list = false); @@ -1295,19 +1310,7 @@ inline bool buf_pool_t::realloc(buf_block_t *block) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); block->page.set_corrupt_id(); - /* set other flags of buf_block_t */ - -#ifdef BTR_CUR_HASH_ADAPT - /* This code should only be executed by resize(), - while the adaptive hash index is disabled. */ - assert_block_ahi_empty(block); - assert_block_ahi_empty_on_init(new_block); - ut_ad(!block->index); - new_block->index = NULL; - new_block->n_hash_helps = 0; - new_block->n_fields = 1; - new_block->left_side = TRUE; -#endif /* BTR_CUR_HASH_ADAPT */ + buf_block_init_low(new_block); ut_d(block->page.set_state(buf_page_t::MEMORY)); /* free block */ new_block = block; @@ -2288,27 +2291,6 @@ buf_page_t *buf_page_get_zip(const page_id_t page_id) noexcept return bpage; } -/********************************************************************//** -Initialize some fields of a control block. */ -UNIV_INLINE -void -buf_block_init_low( -/*===============*/ - buf_block_t* block) /*!< in: block to init */ -{ -#ifdef BTR_CUR_HASH_ADAPT - /* No adaptive hash index entries may point to a previously - unused (and now freshly allocated) block. */ - assert_block_ahi_empty_on_init(block); - block->index = NULL; - - block->n_hash_helps = 0; - block->n_fields = 1; - block->n_bytes = 0; - block->left_side = TRUE; -#endif /* BTR_CUR_HASH_ADAPT */ -} - bool buf_zip_decompress(buf_block_t *block, bool check) noexcept { const byte* frame = block->page.zip.data; diff --git a/storage/innobase/data/data0data.cc b/storage/innobase/data/data0data.cc index 0e485e5821569..975273e475b61 100644 --- a/storage/innobase/data/data0data.cc +++ b/storage/innobase/data/data0data.cc @@ -76,7 +76,7 @@ void dtuple_t::trim(const dict_index_t& index) } } - n_fields = i; + n_fields = uint16_t(i); } /** Compare two data tuples. @@ -113,19 +113,6 @@ dtuple_coll_cmp( return(cmp); } -/*********************************************************************//** -Sets number of fields used in a tuple. Normally this is set in -dtuple_create, but if you want later to set it smaller, you can use this. */ -void -dtuple_set_n_fields( -/*================*/ - dtuple_t* tuple, /*!< in: tuple */ - ulint n_fields) /*!< in: number of fields */ -{ - tuple->n_fields = n_fields; - tuple->n_fields_cmp = n_fields; -} - /**********************************************************//** Checks that a data field is typed. @return TRUE if ok */ @@ -619,8 +606,8 @@ dtuple_convert_big_rec( ulint longest; const bool mblob = entry->is_alter_metadata(); - ut_ad(entry->n_fields - mblob >= index->first_user_field()); - ut_ad(entry->n_fields - mblob <= index->n_fields); + ut_ad(unsigned(entry->n_fields - mblob) >= index->first_user_field()); + ut_ad(unsigned(entry->n_fields - mblob) <= index->n_fields); if (mblob) { longest_i = index->first_user_field(); diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 0bac83b925c91..44327f050ecf1 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -2339,8 +2339,8 @@ dict_table_copy_v_types( /* tuple could have more virtual columns than existing table, if we are calling this for creating index along with adding virtual columns */ - ulint n_fields = ut_min(dtuple_get_n_v_fields(tuple), - static_cast(table->n_v_def)); + ulint n_fields = std::min(dtuple_get_n_v_fields(tuple), + table->n_v_def); for (ulint i = 0; i < n_fields; i++) { @@ -3700,7 +3700,7 @@ dict_index_build_node_ptr( dtuple_t* tuple; dfield_t* field; byte* buf; - ulint n_unique; + uint16_t n_unique; if (dict_index_is_ibuf(index)) { /* In a universal index tree, we take the whole record as @@ -3768,7 +3768,7 @@ dict_index_build_data_tuple( { ut_ad(!index->is_clust()); - dtuple_t* tuple = dtuple_create(heap, n_fields); + dtuple_t* tuple = dtuple_create(heap, uint16_t(n_fields)); dict_index_copy_types(tuple, index, n_fields); diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 917c4d51ee9b5..702eda85ca3fa 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1336,7 +1336,7 @@ static dberr_t dict_load_columns(dict_table_t *table, unsigned use_uncommitted, dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -1463,7 +1463,7 @@ dict_load_virtual_col(dict_table_t *table, bool uncommitted, ulint nth_v_col) dfield_t dfield[2]; dtuple_t tuple{ - 0,2,2,dfield,0,nullptr + 0,2,2,0,dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -1703,7 +1703,7 @@ static dberr_t dict_load_fields(dict_index_t *index, bool uncommitted, dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -1962,7 +1962,7 @@ dberr_t dict_load_indexes(dict_table_t *table, bool uncommitted, dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -2355,7 +2355,7 @@ static dict_table_t *dict_load_table_one(const span &name, dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -2609,7 +2609,7 @@ dict_load_table_on_id( dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -2718,7 +2718,7 @@ static dberr_t dict_load_foreign_cols(dict_foreign_t *foreign, trx_id_t trx_id) dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -2894,7 +2894,7 @@ dict_load_foreign( dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -3105,7 +3105,7 @@ dict_load_foreigns( bool check_recursive = !trx_id; dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index df2cb568d4df5..479b937addbdf 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -3410,7 +3410,7 @@ fts_add_doc_by_id( /* Search based on Doc ID. Here, we'll need to consider the case when there is no primary index on Doc ID */ - const ulint n_uniq = table->fts_n_uniq(); + const auto n_uniq = table->fts_n_uniq(); tuple = dtuple_create(heap, n_uniq); dfield = dtuple_get_nth_field(tuple, 0); dfield->type.mtype = DATA_INT; @@ -3461,9 +3461,7 @@ fts_add_doc_by_id( doc_pcur = &pcur; } else { dtuple_t* clust_ref; - ulint n_fields; - - n_fields = dict_index_get_n_unique(clust_index); + auto n_fields = dict_index_get_n_unique(clust_index); clust_ref = dtuple_create(heap, n_fields); dict_index_copy_types(clust_ref, clust_index, n_fields); diff --git a/storage/innobase/gis/gis0rtree.cc b/storage/innobase/gis/gis0rtree.cc index 0a7a364be6d3c..5eda139719731 100644 --- a/storage/innobase/gis/gis0rtree.cc +++ b/storage/innobase/gis/gis0rtree.cc @@ -141,12 +141,11 @@ rtr_index_build_node_ptr( dtuple_t* tuple; dfield_t* field; byte* buf; - ulint n_unique; ulint info_bits; ut_ad(dict_index_is_spatial(index)); - n_unique = DICT_INDEX_SPATIAL_NODEPTR_SIZE; + uint16_t n_unique = DICT_INDEX_SPATIAL_NODEPTR_SIZE; tuple = dtuple_create(heap, n_unique + 1); @@ -211,8 +210,8 @@ rtr_update_mbr_field( big_rec_t* dummy_big_rec; buf_block_t* block; rec_t* child_rec; - ulint up_match = 0; - ulint low_match = 0; + uint16_t up_match = 0; + uint16_t low_match = 0; ulint child; ulint rec_info; bool ins_suc = true; @@ -607,7 +606,7 @@ rtr_adjust_upper_level( /* Insert the node for the new page. */ node_ptr_upper = rtr_index_build_node_ptr( sea_cur->index(), new_mbr, first, new_page_no, heap); - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; err = page_cur_search_with_match(node_ptr_upper, PAGE_CUR_LE, &up_match, &low_match, @@ -1109,7 +1108,7 @@ rtr_page_split_and_insert( page_cursor->block = cur_split_node->n_node != first_rec_group ? new_block : block; - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; if (page_cur_search_with_match(tuple, PAGE_CUR_LE, &up_match, &low_match, diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index b6a6897191a04..2fc3114f15f81 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -338,7 +338,7 @@ rtr_pcur_getnext_from_path( if (mode == PAGE_CUR_RTREE_LOCATE) { if (target_level == 0 && level == 0) { - ulint low_match = 0, up_match = 0; + uint16_t low_match = 0, up_match = 0; found = false; @@ -565,8 +565,8 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, MEM_UNDEFINED(&cur->up_bytes, sizeof cur->up_bytes); MEM_UNDEFINED(&cur->low_match, sizeof cur->low_match); MEM_UNDEFINED(&cur->low_bytes, sizeof cur->low_bytes); - ut_d(cur->up_match= ULINT_UNDEFINED); - ut_d(cur->low_match= ULINT_UNDEFINED); + ut_d(cur->up_match= uint16_t(~0U)); + ut_d(cur->low_match= uint16_t(~0U)); const bool latch_by_caller= latch_mode & BTR_ALREADY_S_LATCHED; @@ -619,7 +619,7 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, /* Start with the root page. */ page_id_t page_id(index->table->space_id, index->page); - ulint up_match= 0, up_bytes= 0, low_match= 0, low_bytes= 0; + uint16_t up_match= 0, up_bytes= 0, low_match= 0, low_bytes= 0; ulint height= ULINT_UNDEFINED; /* We use these modified search modes on non-leaf levels of the @@ -634,14 +634,8 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, page_mode= PAGE_CUR_LE; break; default: -#ifdef PAGE_CUR_LE_OR_EXTENDS - ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE - || RTREE_SEARCH_MODE(mode) - || mode == PAGE_CUR_LE_OR_EXTENDS); -#else /* PAGE_CUR_LE_OR_EXTENDS */ - ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE - || RTREE_SEARCH_MODE(mode)); -#endif /* PAGE_CUR_LE_OR_EXTENDS */ + ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE || + RTREE_SEARCH_MODE(mode)); page_mode= mode; break; } @@ -975,9 +969,9 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, cur->up_match= up_match; cur->up_bytes= up_bytes; - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_GE); - ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); - ut_ad(low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_GE); + ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_LE); + ut_ad(low_match != uint16_t(~0U) || mode != PAGE_CUR_LE); } goto func_exit; @@ -1675,7 +1669,7 @@ rtr_cur_restore_position( ut_ad(r_cursor == node->cursor); search_again: - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; page_cursor->block = buf_page_get_gen( page_id_t(index->table->space_id, page_no), diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b7de655a5dd93..a12da7db93bf3 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1509,7 +1509,7 @@ static void innodb_drop_database(handlerton*, char *path) dfield_t dfield; dtuple_t tuple{ - 0,1,1,&dfield,0,nullptr + 0,1,1,0,&dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index bd1382d7cdc62..dfe4e614dd42f 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -3626,9 +3626,9 @@ innobase_row_to_mysql( /* The InnoDB row may contain an extra FTS_DOC_ID column at the end. */ ut_ad(row->n_fields == dict_table_get_n_cols(itab)); - ut_ad(n_fields == row->n_fields - DATA_N_SYS_COLS - + dict_table_get_n_v_cols(itab) - - !!(DICT_TF2_FLAG_IS_SET(itab, DICT_TF2_FTS_HAS_DOC_ID))); + ut_ad(row->n_fields == n_fields + DATA_N_SYS_COLS + - dict_table_get_n_v_cols(itab) + + !!(DICT_TF2_FLAG_IS_SET(itab, DICT_TF2_FTS_HAS_DOC_ID))); for (uint i = 0; i < n_fields; i++) { Field* field = table->field[i]; @@ -4686,11 +4686,9 @@ innobase_build_col_map( DBUG_ENTER("innobase_build_col_map"); DBUG_ASSERT(altered_table != table); DBUG_ASSERT(new_table != old_table); - DBUG_ASSERT(dict_table_get_n_cols(new_table) - + dict_table_get_n_v_cols(new_table) + DBUG_ASSERT(unsigned(new_table->n_cols + new_table->n_v_cols) >= altered_table->s->fields + DATA_N_SYS_COLS); - DBUG_ASSERT(dict_table_get_n_cols(old_table) - + dict_table_get_n_v_cols(old_table) + DBUG_ASSERT(unsigned(old_table->n_cols + old_table->n_v_cols) >= table->s->fields + DATA_N_SYS_COLS || ha_innobase::omits_virtual_cols(*table->s)); DBUG_ASSERT(!!defaults == !!(ha_alter_info->handler_flags diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 86eb72aa50a0d..cf572a41e3b0b 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -1355,7 +1355,7 @@ ibuf_build_entry_from_ibuf_rec_func( n_fields = rec_get_n_fields_old(ibuf_rec) - IBUF_REC_FIELD_USER; - tuple = dtuple_create(heap, n_fields); + tuple = dtuple_create(heap, uint16_t(n_fields)); types = rec_get_nth_field_old(ibuf_rec, IBUF_REC_FIELD_METADATA, &len); @@ -1563,7 +1563,7 @@ ibuf_entry_build( n_fields = dtuple_get_n_fields(entry); - tuple = dtuple_create(heap, n_fields + IBUF_REC_FIELD_USER); + tuple = dtuple_create(heap, uint16_t(n_fields + IBUF_REC_FIELD_USER)); /* 1) Space Id */ @@ -2263,7 +2263,7 @@ static void ibuf_delete_recs(const page_id_t page_id) return; dfield_t dfield[IBUF_REC_FIELD_METADATA]; dtuple_t tuple {0,IBUF_REC_FIELD_METADATA,IBUF_REC_FIELD_METADATA, - dfield,0,nullptr + 0,dfield,nullptr #ifdef UNIV_DEBUG ,DATA_TUPLE_MAGIC_N #endif @@ -2459,7 +2459,7 @@ ibuf_merge_space( dfield_t dfield[IBUF_REC_FIELD_METADATA]; dtuple_t tuple {0, IBUF_REC_FIELD_METADATA, - IBUF_REC_FIELD_METADATA,dfield,0,nullptr + IBUF_REC_FIELD_METADATA,0,dfield,nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif @@ -3618,7 +3618,7 @@ ibuf_insert_to_index_page( return DB_CORRUPTION; } - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; page_cur.index = index; page_cur.block = block; @@ -3742,7 +3742,7 @@ ibuf_set_del_mark( page_cur_t page_cur; page_cur.block = block; page_cur.index = index; - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; ut_ad(ibuf_inside(mtr)); ut_ad(dtuple_check_typed(entry)); @@ -3801,7 +3801,7 @@ ibuf_delete( page_cur_t page_cur; page_cur.block = block; page_cur.index = index; - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; ut_ad(ibuf_inside(mtr)); ut_ad(dtuple_check_typed(entry)); @@ -4365,7 +4365,7 @@ void ibuf_delete_for_discarded_space(ulint space) dfield_t dfield[IBUF_REC_FIELD_METADATA]; dtuple_t search_tuple {0,IBUF_REC_FIELD_METADATA, - IBUF_REC_FIELD_METADATA,dfield,0 + IBUF_REC_FIELD_METADATA,0,dfield ,nullptr #ifdef UNIV_DEBUG ,DATA_TUPLE_MAGIC_N diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index e3fd37c069e7c..4663cb5c52648 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -655,14 +655,13 @@ struct btr_path_t { /** Values for the flag documenting the used search method */ enum btr_cur_method { - BTR_CUR_HASH = 1, /*!< successful shortcut using + BTR_CUR_BINARY, /*!< success using the binary search */ +#ifdef BTR_CUR_HASH_ADAPT + BTR_CUR_HASH, /*!< successful shortcut using the hash index */ BTR_CUR_HASH_FAIL, /*!< failure using hash, success using - binary search: the misleading hash - reference is stored in the field - hash_node, and might be necessary to - update */ - BTR_CUR_BINARY, /*!< success using the binary search */ + binary search */ +#endif BTR_CUR_INSERT_TO_IBUF, /*!< performed the intended insert to the insert buffer */ BTR_CUR_DEL_MARK_IBUF, /*!< performed the intended delete @@ -692,7 +691,7 @@ struct btr_cur_t { ulint tree_height; /*!< Tree height if the search is done for a pessimistic insert or update operation */ - ulint up_match; /*!< If the search mode was PAGE_CUR_LE, + uint16_t up_match; /*!< If the search mode was PAGE_CUR_LE, the number of matched fields to the the first user record to the right of the cursor record after search_leaf(); @@ -705,27 +704,26 @@ struct btr_cur_t { record if that record is on a different leaf page! (See the note in row_ins_duplicate_error_in_clust.) */ - ulint up_bytes; /*!< number of matched bytes to the + uint16_t up_bytes; /*!< number of matched bytes to the right at the time cursor positioned; only used internally in searches: not defined after the search */ - ulint low_match; /*!< if search mode was PAGE_CUR_LE, + uint16_t low_match; /*!< if search mode was PAGE_CUR_LE, the number of matched fields to the first user record AT THE CURSOR or to the left of it after search_leaf(); NOT defined for PAGE_CUR_GE or any other search modes; see also the NOTE in up_match! */ - ulint low_bytes; /*!< number of matched bytes to the + uint16_t low_bytes; /*!< number of matched bytes to the left at the time cursor positioned; only used internally in searches: not defined after the search */ - ulint n_fields; /*!< prefix length used in a hash - search if hash_node != NULL */ - ulint n_bytes; /*!< hash prefix bytes if hash_node != - NULL */ - ulint fold; /*!< fold value used in the search if +#ifdef BTR_CUR_HASH_ADAPT + uint32_t n_bytes_fields; /*!< prefix used in a hash search */ + uint32_t fold; /*!< fold value used in the search if flag is BTR_CUR_HASH */ +#endif /* @} */ rtr_info_t* rtr_info; /*!< rtree search info */ btr_cur_t() { memset((void*) this, 0, sizeof *this); } @@ -770,6 +768,16 @@ struct btr_cur_t { #ifdef BTR_CUR_HASH_ADAPT void search_info_update() const noexcept; + + /** Check if a guessed position for a tree cursor is correct. + @param tuple search key + @param mode PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, PAGE_CUR_GE + @param comp nonzero if ROW_FORMAT=REDUNDANT is not being used + @retval true on mismatch or corruption + @retval false on a match; if mode=PAGE_CUR_LE, then up_match,low_match + will be set correctly. */ + bool check_mismatch(const dtuple_t& tuple, page_cur_mode_t mode, ulint comp) + noexcept; #endif }; diff --git a/storage/innobase/include/btr0pcur.inl b/storage/innobase/include/btr0pcur.inl index b827d70dc47bf..4f43972a23673 100644 --- a/storage/innobase/include/btr0pcur.inl +++ b/storage/innobase/include/btr0pcur.inl @@ -59,7 +59,7 @@ btr_pcur_get_up_match( btr_cursor = btr_pcur_get_btr_cur(cursor); - ut_ad(btr_cursor->up_match != ULINT_UNDEFINED); + ut_ad(btr_cursor->up_match != uint16_t(~0U)); return(btr_cursor->up_match); } @@ -80,7 +80,7 @@ btr_pcur_get_low_match( || (cursor->pos_state == BTR_PCUR_IS_POSITIONED)); btr_cursor = btr_pcur_get_btr_cur(cursor); - ut_ad(btr_cursor->low_match != ULINT_UNDEFINED); + ut_ad(btr_cursor->low_match != uint16_t(~0U)); return(btr_cursor->low_match); } diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index 1c18533e23984..aa8a8e50fe44e 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -28,14 +28,14 @@ Created 2/17/1996 Heikki Tuuri #include "dict0dict.h" #ifdef BTR_CUR_HASH_ADAPT -#include "buf0buf.h" +# include "buf0buf.h" -#ifdef UNIV_PFS_RWLOCK +# ifdef UNIV_PFS_RWLOCK extern mysql_pfs_key_t btr_search_latch_key; -#endif /* UNIV_PFS_RWLOCK */ +# endif /* UNIV_PFS_RWLOCK */ -#define btr_search_sys_create() btr_search.create() -#define btr_search_sys_free() btr_search.free() +# define btr_search_sys_create() btr_search.create() +# define btr_search_sys_free() btr_search.free() /** Tries to guess the right search position based on the hash search info of the index. Note that if mode is PAGE_CUR_LE, which is used in inserts, @@ -52,8 +52,8 @@ bool btr_search_guess_on_hash( dict_index_t* index, const dtuple_t* tuple, - ulint mode, - ulint latch_mode, + page_cur_mode_t mode, + btr_latch_mode latch_mode, btr_cur_t* cursor, mtr_t* mtr) noexcept; @@ -61,34 +61,27 @@ btr_search_guess_on_hash( If new_block is already hashed, then any hash index for block is dropped. If new_block is not hashed, and block is hashed, then a new hash index is built to new_block with the same parameters as block. -@param[in,out] new_block destination page -@param[in,out] block source page (subject to deletion later) */ +@param new_block destination page +@param block source page (subject to deletion later) */ void btr_search_move_or_delete_hash_entries(buf_block_t *new_block, buf_block_t *block) noexcept; /** Drop any adaptive hash index entries that point to an index page. -@param[in,out] block block containing index page, s- or x-latched, or an - index page for which we know that - block->buf_fix_count == 0 or it is an index page which - has already been removed from the buf_pool.page_hash - i.e.: it is in state BUF_BLOCK_REMOVE_HASH -@param[in] garbage_collect drop ahi only if the index is marked - as freed */ +@param block latched block containing index page, or a buffer-unfixed + index page or a block in state BUF_BLOCK_REMOVE_HASH +@param garbage_collect drop ahi only if the index is marked as freed */ void btr_search_drop_page_hash_index(buf_block_t* block, - bool garbage_collect) noexcept; + bool garbage_collect) noexcept; /** Drop possible adaptive hash index entries when a page is evicted from the buffer pool or freed in a file, or the index is being dropped. -@param[in] page_id page id */ +@param page_id page identifier of the being-dropped page */ void btr_search_drop_page_hash_when_freed(const page_id_t page_id) noexcept; /** Update the page hash index after a single record is inserted on a page. -@param cursor cursor which was positioned before the inserted record */ -void btr_search_update_hash_node_on_insert(btr_cur_t *cursor) noexcept; - -/** Update the page hash index after a single record is inserted on a page. -@param cursor cursor which was positioned before the inserted record */ -void btr_search_update_hash_on_insert(btr_cur_t *cursor) noexcept; +@param cursor cursor which was positioned before the inserted record +@param reorg whether the page was reorganized */ +void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept; /** Updates the page hash index before a single record is deleted from a page. @param cursor cursor positioned on the to-be-deleted record */ @@ -109,7 +102,9 @@ struct ahi_node; /** The hash index system */ struct btr_sea { - /** the actual value of innodb_adaptive_hash_index */ + /** the actual value of innodb_adaptive_hash_index, protected by + all partition::latch. Note that if buf_block_t::index is not nullptr + while a thread is holding a partition::latch, then also this must hold. */ Atomic_relaxed enabled; /** Disable the adaptive hash search system and empty the index. */ @@ -122,9 +117,9 @@ struct btr_sea /** Partition of the hash table */ struct partition { - /** latch protecting the hash table */ + /** latch protecting table */ alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_spin_lock latch; - /** map of dtuple_fold() or rec_fold() to rec_t* in buf_page_t::frame */ + /** map of CRC-32C of rec prefix to rec_t* in buf_page_t::frame */ hash_table_t table; /** latch protecting blocks, spare; may be acquired while holding latch */ srw_mutex blocks_mutex; @@ -153,22 +148,22 @@ struct btr_sea __attribute__((nonnull)) # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG /** Insert or replace an entry into the hash table. - @param fold hash value of rec + @param fold CRC-32C of rec prefix @param rec B-tree leaf page record @param block the buffer block that contains rec */ - void insert(ulint fold, const rec_t *rec, buf_block_t *block) noexcept; + void insert(uint32_t fold, const rec_t *rec, buf_block_t *block) noexcept; # else /** Insert or replace an entry into the hash table. - @param fold hash value of data + @param fold CRC-32C of rec prefix @param rec B-tree leaf page record */ - void insert(ulint fold, const rec_t *rec) noexcept; + void insert(uint32_t fold, const rec_t *rec) noexcept; # endif /** Delete a pointer to a record if it exists. - @param fold hash value of rec - @param rec B-tree leaf page record + @param fold CRC-32C of rec prefix + @param rec B-tree leaf page record @return whether a record existed and was removed */ - inline bool erase(ulint fold, const rec_t *rec) noexcept; + inline bool erase(uint32_t fold, const rec_t *rec) noexcept; }; /** innodb_adaptive_hash_index_parts */ @@ -200,9 +195,9 @@ extern btr_sea btr_search; # ifdef UNIV_SEARCH_PERF_STAT /** Number of successful adaptive hash index lookups */ -extern ulint btr_search_n_succ; +extern ulint btr_search_n_succ; /** Number of failed adaptive hash index lookups */ -extern ulint btr_search_n_hash_fail; +extern ulint btr_search_n_hash_fail; # endif /* UNIV_SEARCH_PERF_STAT */ #else /* BTR_CUR_HASH_ADAPT */ # define btr_search_sys_create() diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 2d29f00ba7f9d..63165b4130622 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -916,87 +916,35 @@ struct buf_block_t{ x-latch on the block */ /* @} */ #ifdef BTR_CUR_HASH_ADAPT - /** @name Hash search fields (unprotected) - NOTE that these fields are NOT protected by any semaphore! */ - /* @{ */ - - volatile uint16_t n_bytes; /*!< recommended prefix length for hash - search: number of bytes in - an incomplete last field */ - volatile uint16_t n_fields; /*!< recommended prefix length for hash - search: number of full fields */ - uint16_t n_hash_helps; /*!< counter which controls building - of a new hash index for the page */ - volatile bool left_side; /*!< true or false, depending on - whether the leftmost record of several - records with the same prefix should be - indexed in the hash index */ - /* @} */ - - /** @name Hash search fields - These 5 fields may only be modified when: - we are holding the appropriate x-latch in btr_search_latches[], and - one of the following holds: - (1) in_file(), and we are holding lock in any mode, or - (2) !is_read_fixed()&&(state()>=UNFIXED||state()==REMOVE_HASH). - - An exception to this is when we init or create a page - in the buffer pool in buf0buf.cc. - - Another exception for buf_pool_t::clear_hash_index() is that - assigning block->index = NULL (and block->n_pointers = 0) - is allowed whenever all AHI latches are exclusively locked. - - Another exception is that ha_insert_for_fold() may - decrement n_pointers without holding the appropriate latch - in btr_search.parts. Thus, n_pointers must be - protected by atomic memory access. - - This implies that the fields may be read without race - condition whenever any of the following hold: - - the btr_search.parts.latch is being held, or - - state() == NOT_USED || state() == MEMORY, - and holding some latch prevents the state from changing to that. - - Some use of assert_block_ahi_empty() or assert_block_ahi_valid() - is prone to race conditions while buf_pool_t::clear_hash_index() is - executing (the adaptive hash index is being disabled). Such use - is explicitly commented. */ - - /* @{ */ - + /** @name Hash search fields */ + /* @{ */ + /** flag: (true=first, false=last) identical-prefix key is included */ + static constexpr uint32_t LEFT_SIDE= 1U << 31; + + /** AHI parameters: LEFT_SIDE | prefix_bytes << 16 | prefix_fields. + Protected by the btr_sea::partition::latch and + (1) in_file(), and we are holding lock in any mode, or + (2) !is_read_fixed()&&(state()>=UNFIXED||state()==REMOVE_HASH). */ + Atomic_relaxed ahi_left_bytes_fields; + + /** counter which controls building of a new hash index for the page; + may be nonzero even if !index */ + Atomic_relaxed n_hash_helps; # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - Atomic_counter - n_pointers; /*!< used in debugging: the number of - pointers in the adaptive hash index - pointing to this frame */ -# define assert_block_ahi_empty(block) \ - ut_a((block)->n_pointers == 0) -# define assert_block_ahi_empty_on_init(block) do { \ - MEM_MAKE_DEFINED(&(block)->n_pointers, sizeof (block)->n_pointers); \ - assert_block_ahi_empty(block); \ -} while (0) -# define assert_block_ahi_valid(block) \ - ut_a((block)->index || (block)->n_pointers == 0) + /** number of pointers from the btr_sea::partition::table; + !n_pointers == !index */ + Atomic_counter n_pointers; +# define assert_block_ahi_empty(block) ut_a(!(block)->n_pointers) +# define assert_block_ahi_valid(b) ut_a((b)->index || !(b)->n_pointers) # else /* UNIV_AHI_DEBUG || UNIV_DEBUG */ # define assert_block_ahi_empty(block) /* nothing */ # define assert_block_ahi_valid(block) /* nothing */ # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - unsigned curr_n_fields:10;/*!< prefix length for hash indexing: - number of full fields */ - unsigned curr_n_bytes:15;/*!< number of bytes in hash - indexing */ - unsigned curr_left_side:1;/*!< TRUE or FALSE in hash indexing */ - dict_index_t* index; /*!< Index for which the - adaptive hash index has been - created, or NULL if the page - does not exist in the - index. Note that it does not - guarantee that the index is - complete, though: there may - have been hash collisions, - record deletions, etc. */ - /* @} */ + /** index for which the adaptive hash index has been created, + or nullptr if the page does not exist in the index. + Protected by btr_sea::partition::latch. */ + Atomic_relaxed index; + /* @} */ #else /* BTR_CUR_HASH_ADAPT */ # define assert_block_ahi_empty(block) /* nothing */ # define assert_block_ahi_valid(block) /* nothing */ diff --git a/storage/innobase/include/data0data.h b/storage/innobase/include/data0data.h index 6b1a19153503f..f42168c2d6f41 100644 --- a/storage/innobase/include/data0data.h +++ b/storage/innobase/include/data0data.h @@ -193,7 +193,7 @@ dtuple_set_info_bits( Gets number of fields used in record comparisons. @return number of fields used in comparisons in rem0cmp.* */ UNIV_INLINE -ulint +uint16_t dtuple_get_n_fields_cmp( /*====================*/ const dtuple_t* tuple) /*!< in: tuple */ @@ -270,6 +270,7 @@ dtuple_create_with_vcol( /*********************************************************************//** Sets number of fields used in a tuple. Normally this is set in dtuple_create, but if you want later to set it smaller, you can use this. */ +inline void dtuple_set_n_fields( /*================*/ @@ -326,20 +327,6 @@ dtuple_coll_cmp( const dtuple_t* tuple1, const dtuple_t* tuple2) MY_ATTRIBUTE((warn_unused_result)); -/** Fold a prefix given as the number of fields of a tuple. -@param[in] tuple index record -@param[in] n_fields number of complete fields to fold -@param[in] n_bytes number of bytes to fold in the last field -@param[in] index_id index tree ID -@return the folded value */ -UNIV_INLINE -ulint -dtuple_fold( - const dtuple_t* tuple, - ulint n_fields, - ulint n_bytes, - index_id_t tree_id) - MY_ATTRIBUTE((warn_unused_result)); /*******************************************************************//** Sets types of fields binary in a tuple. */ UNIV_INLINE @@ -509,20 +496,20 @@ struct dfield_t{ /** Structure for an SQL data tuple of fields (logical record) */ struct dtuple_t { - ulint info_bits; /*!< info bits of an index record: + byte info_bits; /*!< info bits of an index record: the default is 0; this field is used if an index record is built from a data tuple */ - ulint n_fields; /*!< number of fields in dtuple */ - ulint n_fields_cmp; /*!< number of fields which should + uint16_t n_fields; /*!< number of fields in dtuple */ + uint16_t n_fields_cmp; /*!< number of fields which should be used in comparison services of rem0cmp.*; the index search is performed by comparing only these fields, others are ignored; the default value in dtuple creation is the same value as n_fields */ + uint16_t n_v_fields; /*!< number of virtual fields */ dfield_t* fields; /*!< fields */ - ulint n_v_fields; /*!< number of virtual fields */ dfield_t* v_fields; /*!< fields on virtual column */ #ifdef UNIV_DEBUG ulint magic_n; /*!< magic number, used in @@ -584,7 +571,7 @@ struct dtuple_t { inline void copy_field_types(const dict_index_t &index); }; -inline ulint dtuple_get_n_fields(const dtuple_t* tuple) +inline uint16_t dtuple_get_n_fields(const dtuple_t* tuple) { return tuple->n_fields; } inline dtype_t* dfield_get_type(dfield_t* field) { return &field->type; } inline const dtype_t* dfield_get_type(const dfield_t* field) @@ -618,7 +605,7 @@ inline void dfield_set_ext(dfield_t* field) { field->ext = 1; } /** Gets number of virtual fields in a data tuple. @param[in] tuple dtuple to check @return number of fields */ -inline ulint +inline uint16_t dtuple_get_n_v_fields(const dtuple_t* tuple) { return tuple->n_v_fields; } inline const dfield_t* dtuple_get_nth_field(const dtuple_t* tuple, ulint n) diff --git a/storage/innobase/include/data0data.inl b/storage/innobase/include/data0data.inl index b6c6ace8dc01a..99e1fdb8fa02c 100644 --- a/storage/innobase/include/data0data.inl +++ b/storage/innobase/include/data0data.inl @@ -238,14 +238,14 @@ dtuple_set_info_bits( dtuple_t* tuple, /*!< in: tuple */ ulint info_bits) /*!< in: info bits */ { - tuple->info_bits = info_bits; + tuple->info_bits = byte(info_bits); } /*********************************************************************//** Gets number of fields used in record comparisons. @return number of fields used in comparisons in rem0cmp.* */ UNIV_INLINE -ulint +uint16_t dtuple_get_n_fields_cmp( /*====================*/ const dtuple_t* tuple) /*!< in: tuple */ @@ -264,7 +264,7 @@ dtuple_set_n_fields_cmp( comparisons in rem0cmp.* */ { ut_ad(n_fields_cmp <= tuple->n_fields); - tuple->n_fields_cmp = n_fields_cmp; + tuple->n_fields_cmp = uint16_t(n_fields_cmp); } /** Creates a data tuple from an already allocated chunk of memory. @@ -291,9 +291,9 @@ dtuple_create_from_mem( tuple = (dtuple_t*) buf; tuple->info_bits = 0; - tuple->n_fields = n_fields; - tuple->n_v_fields = n_v_fields; - tuple->n_fields_cmp = n_fields; + tuple->n_fields = uint16_t(n_fields); + tuple->n_v_fields = uint16_t(n_v_fields); + tuple->n_fields_cmp = uint16_t(n_fields); tuple->fields = (dfield_t*) &tuple[1]; if (n_v_fields > 0) { tuple->v_fields = &tuple->fields[n_fields]; @@ -398,6 +398,12 @@ dtuple_create_with_vcol( return(tuple); } +inline void dtuple_set_n_fields(dtuple_t *tuple, ulint n_fields) +{ + tuple->n_fields= uint16_t(n_fields); + tuple->n_fields_cmp= uint16_t(n_fields); +} + /** Copies a data tuple's virtual fields to another. This is a shallow copy; @param[in,out] d_tuple destination tuple @param[in] s_tuple source tuple */ @@ -432,7 +438,7 @@ dtuple_copy( ulint n_fields = dtuple_get_n_fields(tuple); ulint n_v_fields = dtuple_get_n_v_fields(tuple); dtuple_t* new_tuple = dtuple_create_with_vcol( - heap, n_fields, n_v_fields); + heap, tuple->n_fields, tuple->n_v_fields); ulint i; for (i = 0; i < n_fields; i++) { @@ -527,63 +533,6 @@ dtuple_set_types_binary( } } -/** Fold a prefix given as the number of fields of a tuple. -@param[in] tuple index record -@param[in] n_fields number of complete fields to fold -@param[in] n_bytes number of bytes to fold in the last field -@param[in] index_id index tree ID -@return the folded value */ -UNIV_INLINE -ulint -dtuple_fold( - const dtuple_t* tuple, - ulint n_fields, - ulint n_bytes, - index_id_t tree_id) -{ - const dfield_t* field; - ulint i; - const byte* data; - ulint len; - ulint fold; - - ut_ad(tuple); - ut_ad(tuple->magic_n == DATA_TUPLE_MAGIC_N); - ut_ad(dtuple_check_typed(tuple)); - - fold = ut_fold_ull(tree_id); - - for (i = 0; i < n_fields; i++) { - field = dtuple_get_nth_field(tuple, i); - - data = (const byte*) dfield_get_data(field); - len = dfield_get_len(field); - - if (len != UNIV_SQL_NULL) { - fold = ut_fold_ulint_pair(fold, - ut_fold_binary(data, len)); - } - } - - if (n_bytes > 0) { - field = dtuple_get_nth_field(tuple, i); - - data = (const byte*) dfield_get_data(field); - len = dfield_get_len(field); - - if (len != UNIV_SQL_NULL) { - if (len > n_bytes) { - len = n_bytes; - } - - fold = ut_fold_ulint_pair(fold, - ut_fold_binary(data, len)); - } - } - - return(fold); -} - /**********************************************************************//** Writes an SQL null field full of zeros. */ UNIV_INLINE diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 0197a790faa13..656cfed0622a5 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -663,7 +663,7 @@ Gets the number of all non-virtual columns (also system) in a table in the dictionary cache. @return number of columns of a table */ UNIV_INLINE -unsigned +uint16_t dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ @@ -673,7 +673,7 @@ dict_table_get_n_cols( @param[in] table the table to check @return number of virtual columns of a table */ UNIV_INLINE -unsigned +uint16_t dict_table_get_n_v_cols( const dict_table_t* table); diff --git a/storage/innobase/include/dict0dict.inl b/storage/innobase/include/dict0dict.inl index 5c760f1e89d6c..2dabce146a573 100644 --- a/storage/innobase/include/dict0dict.inl +++ b/storage/innobase/include/dict0dict.inl @@ -264,7 +264,7 @@ Gets the number of all non-virtual columns (also system) in a table in the dictionary cache. @return number of non-virtual columns of a table */ UNIV_INLINE -unsigned +uint16_t dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ @@ -277,7 +277,7 @@ dict_table_get_n_cols( @param[in] table the table to check @return number of virtual columns of a table */ UNIV_INLINE -unsigned +uint16_t dict_table_get_n_v_cols( const dict_table_t* table) { diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 7c3c532577d12..e7b8ecc5ff6ef 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1113,7 +1113,7 @@ struct dict_index_t { /** number of consecutive searches which would have succeeded, or did succeed, using the hash index; the range is 0 - .. BTR_SEARCH_BUILD_LIMIT + 5 */ + .. BTR_SEARCH_BUILD_LIMIT */ Atomic_relaxed n_hash_potential{0}; /** whether the last search would have succeeded, or @@ -1122,15 +1122,8 @@ struct dict_index_t { search, and the calculation itself is not always accurate! */ Atomic_relaxed last_hash_succ{false}; - /** recommended full field prefix */ - uint16_t n_fields= 1; - /** recommended number of bytes in an incomplete field */ - uint16_t n_bytes= 0; - - /** whether the leftmost record of several records with the same - prefix should be indexed */ - bool left_side= true; - + /** recommended parameters; @see buf_block_t::left_bytes_fields */ + Atomic_relaxed left_bytes_fields{buf_block_t::LEFT_SIDE | 1}; /** number of buf_block_t::index pointers to this index */ Atomic_counter ref_count{0}; @@ -2598,7 +2591,7 @@ struct dict_table_t { bool is_stats_table() const; /** @return number of unique columns in FTS_DOC_ID index */ - unsigned fts_n_uniq() const { return versioned() ? 2 : 1; } + uint16_t fts_n_uniq() const { return versioned() ? 2 : 1; } /** @return the index for that starts with a specific column */ dict_index_t *get_index(const dict_col_t &col) const; diff --git a/storage/innobase/include/page0cur.h b/storage/innobase/include/page0cur.h index bef8a679ea017..6c3ac21973af0 100644 --- a/storage/innobase/include/page0cur.h +++ b/storage/innobase/include/page0cur.h @@ -231,38 +231,34 @@ page_cur_search_with_match( page_cur_mode_t mode, /*!< in: PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, or PAGE_CUR_GE */ - ulint* iup_matched_fields, + uint16_t* iup_matched_fields, /*!< in/out: already matched fields in upper limit record */ - ulint* ilow_matched_fields, + uint16_t* ilow_matched_fields, /*!< in/out: already matched fields in lower limit record */ page_cur_t* cursor, /*!< in/out: page cursor */ rtr_info_t* rtr_info);/*!< in/out: rtree search stack */ -#ifdef BTR_CUR_HASH_ADAPT -MY_ATTRIBUTE((warn_unused_result)) + /** Search the right position for a page cursor. -@param[in] tuple key to be searched for -@param[in] mode search mode -@param[in,out] iup_matched_fields already matched fields in the -upper limit record -@param[in,out] iup_matched_bytes already matched bytes in the -first partially matched field in the upper limit record -@param[in,out] ilow_matched_fields already matched fields in the -lower limit record -@param[in,out] ilow_matched_bytes already matched bytes in the -first partially matched field in the lower limit record -@param[in,out] cursor page cursor */ -bool -page_cur_search_with_match_bytes( - const dtuple_t* tuple, - page_cur_mode_t mode, - ulint* iup_matched_fields, - ulint* iup_matched_bytes, - ulint* ilow_matched_fields, - ulint* ilow_matched_bytes, - page_cur_t* cursor); -#endif /* BTR_CUR_HASH_ADAPT */ +@param tuple search key +@param mode search mode +@param iup_fields matched fields in the upper limit record +@param ilow_fields matched fields in the low limit record +@param cursor page cursor +@param iup_bytes matched bytes after iup_fields +@param ilow_bytes matched bytes after ilow_fields +@return whether the first partially matched field in the lower limit record, +or the page is corrupted */ +bool page_cur_search_with_match_bytes(const dtuple_t &tuple, + page_cur_mode_t mode, + uint16_t *iup_fields, + uint16_t *ilow_fields, + page_cur_t *cursor, + uint16_t *iup_bytes, + uint16_t *ilow_bytes) + noexcept; + /***********************************************************//** Positions a page cursor on a randomly chosen user record on a page. If there are no user records, sets the cursor on the infimum record. */ diff --git a/storage/innobase/include/page0types.h b/storage/innobase/include/page0types.h index 83fc45cdfc438..4a2d1ccccd758 100644 --- a/storage/innobase/include/page0types.h +++ b/storage/innobase/include/page0types.h @@ -72,11 +72,6 @@ enum page_cur_mode_t { PAGE_CUR_L = 3, PAGE_CUR_LE = 4, -/* PAGE_CUR_LE_OR_EXTENDS = 5,*/ /* This is a search mode used in - "column LIKE 'abc%' ORDER BY column DESC"; - we have to find strings which are <= 'abc' or - which extend it */ - /* These search mode is for search R-tree index. */ PAGE_CUR_CONTAIN = 7, PAGE_CUR_INTERSECT = 8, diff --git a/storage/innobase/include/rem0cmp.h b/storage/innobase/include/rem0cmp.h index 6f2201971d1dc..219ccce86338c 100644 --- a/storage/innobase/include/rem0cmp.h +++ b/storage/innobase/include/rem0cmp.h @@ -43,25 +43,19 @@ cmp_cols_are_equal( ibool check_charsets); /*!< in: whether to check charsets */ /** Compare two data fields. -@param[in] mtype main type -@param[in] prtype precise type -@param[in] data1 data field -@param[in] len1 length of data1 in bytes, or UNIV_SQL_NULL -@param[in] data2 data field -@param[in] len2 length of data2 in bytes, or UNIV_SQL_NULL +@param mtype main type +@param prtype precise type +@param data1 data field +@param len1 length of data1 in bytes, or UNIV_SQL_NULL +@param data2 data field +@param len2 length of data2 in bytes, or UNIV_SQL_NULL @return the comparison result of data1 and data2 @retval 0 if data1 is equal to data2 @retval negative if data1 is less than data2 @retval positive if data1 is greater than data2 */ -int -cmp_data_data( - ulint mtype, - ulint prtype, - const byte* data1, - ulint len1, - const byte* data2, - ulint len2) - MY_ATTRIBUTE((warn_unused_result)); +int cmp_data_data(ulint mtype, ulint prtype, const byte *data1, ulint len1, + const byte *data2, ulint len2) noexcept + MY_ATTRIBUTE((warn_unused_result)); /** Compare two data fields. @param[in] dfield1 data field; must have type field set @@ -150,32 +144,12 @@ cmp_dtuple_rec_with_match_low( const rec_t* rec, const rec_offs* offsets, ulint n_cmp, - ulint* matched_fields) + uint16_t* matched_fields) MY_ATTRIBUTE((nonnull)); #define cmp_dtuple_rec_with_match(tuple,rec,offsets,fields) \ cmp_dtuple_rec_with_match_low( \ tuple,rec,offsets,dtuple_get_n_fields_cmp(tuple),fields) -/** Compare a data tuple to a physical record. -@param[in] dtuple data tuple -@param[in] rec B-tree or R-tree index record -@param[in] index index tree -@param[in] offsets rec_get_offsets(rec) -@param[in,out] matched_fields number of completely matched fields -@param[in,out] matched_bytes number of matched bytes in the first -field that is not matched -@return the comparison result of dtuple and rec -@retval 0 if dtuple is equal to rec -@retval negative if dtuple is less than rec -@retval positive if dtuple is greater than rec */ -int -cmp_dtuple_rec_with_match_bytes( - const dtuple_t* dtuple, - const rec_t* rec, - const dict_index_t* index, - const rec_offs* offsets, - ulint* matched_fields, - ulint* matched_bytes) - MY_ATTRIBUTE((warn_unused_result)); + /** Compare a data tuple to a physical record. @see cmp_dtuple_rec_with_match @param[in] dtuple data tuple diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index 2f038ab349f85..1a2795d4ecc58 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -178,7 +178,7 @@ The following function is used to get the number of fields in an old-style record. @return number of data fields */ UNIV_INLINE -ulint +uint16_t rec_get_n_fields_old( /*=================*/ const rec_t* rec) /*!< in: physical record */ @@ -353,7 +353,7 @@ The following function is used to get the order number of an old-style record in the heap of the index page. @return heap order number */ UNIV_INLINE -ulint +uint16_t rec_get_heap_no_old( /*================*/ const rec_t* rec) /*!< in: physical record */ @@ -363,7 +363,7 @@ The following function is used to get the order number of a new-style record in the heap of the index page. @return heap order number */ UNIV_INLINE -ulint +uint16_t rec_get_heap_no_new( /*================*/ const rec_t* rec) /*!< in: physical record */ diff --git a/storage/innobase/include/rem0rec.inl b/storage/innobase/include/rem0rec.inl index 46c209cbdec43..2767c5aec35dd 100644 --- a/storage/innobase/include/rem0rec.inl +++ b/storage/innobase/include/rem0rec.inl @@ -164,7 +164,7 @@ rec_set_bit_field_1( /******************************************************//** Gets a bit field from within 2 bytes. */ UNIV_INLINE -ulint +uint16_t rec_get_bit_field_2( /*================*/ const rec_t* rec, /*!< in: pointer to record origin */ @@ -174,7 +174,7 @@ rec_get_bit_field_2( { ut_ad(rec); - return((mach_read_from_2(rec - offs) & mask) >> shift); + return uint16_t((mach_read_from_2(rec - offs) & mask) >> shift); } /******************************************************//** @@ -307,18 +307,14 @@ The following function is used to get the number of fields in an old-style record. @return number of data fields */ UNIV_INLINE -ulint +uint16_t rec_get_n_fields_old( /*=================*/ const rec_t* rec) /*!< in: physical record */ { - ulint ret; - - ut_ad(rec); - - ret = rec_get_bit_field_2(rec, REC_OLD_N_FIELDS, - REC_OLD_N_FIELDS_MASK, - REC_OLD_N_FIELDS_SHIFT); + uint16_t ret = rec_get_bit_field_2(rec, REC_OLD_N_FIELDS, + REC_OLD_N_FIELDS_MASK, + REC_OLD_N_FIELDS_SHIFT); ut_ad(ret <= REC_MAX_N_FIELDS); ut_ad(ret > 0); @@ -397,7 +393,7 @@ rec_n_fields_is_sane( /* a record for older SYS_INDEXES table (missing merge_threshold column) is acceptable. */ || (index->table->id == DICT_INDEXES_ID - && n_fields == dtuple_get_n_fields(entry) - 1)); + && n_fields + 1 == dtuple_get_n_fields(entry))); } /******************************************************//** @@ -518,7 +514,7 @@ The following function is used to get the order number of an old-style record in the heap of the index page. @return heap order number */ UNIV_INLINE -ulint +uint16_t rec_get_heap_no_old( /*================*/ const rec_t* rec) /*!< in: physical record */ @@ -532,7 +528,7 @@ The following function is used to get the order number of a new-style record in the heap of the index page. @return heap order number */ UNIV_INLINE -ulint +uint16_t rec_get_heap_no_new( /*================*/ const rec_t* rec) /*!< in: physical record */ diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 57f0b8edfb675..eba7e29c5e830 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -53,49 +53,6 @@ Created 1/20/1994 Heikki Tuuri #define ut_max std::max #define ut_min std::min -/** Calculate the minimum of two pairs. -@param[out] min_hi MSB of the minimum pair -@param[out] min_lo LSB of the minimum pair -@param[in] a_hi MSB of the first pair -@param[in] a_lo LSB of the first pair -@param[in] b_hi MSB of the second pair -@param[in] b_lo LSB of the second pair */ -UNIV_INLINE -void -ut_pair_min( - ulint* min_hi, - ulint* min_lo, - ulint a_hi, - ulint a_lo, - ulint b_hi, - ulint b_lo); -/******************************************************//** -Compares two ulints. -@return 1 if a > b, 0 if a == b, -1 if a < b */ -UNIV_INLINE -int -ut_ulint_cmp( -/*=========*/ - ulint a, /*!< in: ulint */ - ulint b); /*!< in: ulint */ -/** Compare two pairs of integers. -@param[in] a_h more significant part of first pair -@param[in] a_l less significant part of first pair -@param[in] b_h more significant part of second pair -@param[in] b_l less significant part of second pair -@return comparison result of (a_h,a_l) and (b_h,b_l) -@retval -1 if (a_h,a_l) is less than (b_h,b_l) -@retval 0 if (a_h,a_l) is equal to (b_h,b_l) -@retval 1 if (a_h,a_l) is greater than (b_h,b_l) */ -UNIV_INLINE -int -ut_pair_cmp( - ulint a_h, - ulint a_l, - ulint b_h, - ulint b_l) - MY_ATTRIBUTE((warn_unused_result)); - /*************************************************************//** Calculates fast the remainder of n/m when m is a power of two. @param n in: numerator @@ -119,24 +76,6 @@ when m is a power of two. In other words, rounds n up to m * k. template inline T ut_calc_align(T n, T m) { return static_cast(UT_CALC_ALIGN(n, m)); } -/*************************************************************//** -Calculates fast the 2-logarithm of a number, rounded upward to an -integer. -@return logarithm in the base 2, rounded upward */ -UNIV_INLINE -ulint -ut_2_log( -/*=====*/ - ulint n); /*!< in: number */ -/*************************************************************//** -Calculates 2 to power n. -@return 2 to power n */ -UNIV_INLINE -ulint -ut_2_exp( -/*=====*/ - ulint n); /*!< in: number */ - /**********************************************************//** Returns the number of milliseconds since some epoch. The value may wrap around. It should only be used for heuristic @@ -414,7 +353,5 @@ class fatal_or_error : public logger { } // namespace ib -#include "ut0ut.inl" - #endif diff --git a/storage/innobase/include/ut0ut.inl b/storage/innobase/include/ut0ut.inl deleted file mode 100644 index 73feaf82c6a70..0000000000000 --- a/storage/innobase/include/ut0ut.inl +++ /dev/null @@ -1,143 +0,0 @@ -/***************************************************************************** - -Copyright (c) 1994, 2015, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************************//** -@file include/ut0ut.ic -Various utilities - -Created 5/30/1994 Heikki Tuuri -*******************************************************************/ - -#include - -/** Calculate the minimum of two pairs. -@param[out] min_hi MSB of the minimum pair -@param[out] min_lo LSB of the minimum pair -@param[in] a_hi MSB of the first pair -@param[in] a_lo LSB of the first pair -@param[in] b_hi MSB of the second pair -@param[in] b_lo LSB of the second pair */ -UNIV_INLINE -void -ut_pair_min( - ulint* min_hi, - ulint* min_lo, - ulint a_hi, - ulint a_lo, - ulint b_hi, - ulint b_lo) -{ - if (a_hi == b_hi) { - *min_hi = a_hi; - *min_lo = std::min(a_lo, b_lo); - } else if (a_hi < b_hi) { - *min_hi = a_hi; - *min_lo = a_lo; - } else { - *min_hi = b_hi; - *min_lo = b_lo; - } -} - -/******************************************************//** -Compares two ulints. -@return 1 if a > b, 0 if a == b, -1 if a < b */ -UNIV_INLINE -int -ut_ulint_cmp( -/*=========*/ - ulint a, /*!< in: ulint */ - ulint b) /*!< in: ulint */ -{ - if (a < b) { - return(-1); - } else if (a == b) { - return(0); - } else { - return(1); - } -} - -/** Compare two pairs of integers. -@param[in] a_h more significant part of first pair -@param[in] a_l less significant part of first pair -@param[in] b_h more significant part of second pair -@param[in] b_l less significant part of second pair -@return comparison result of (a_h,a_l) and (b_h,b_l) -@retval -1 if (a_h,a_l) is less than (b_h,b_l) -@retval 0 if (a_h,a_l) is equal to (b_h,b_l) -@retval 1 if (a_h,a_l) is greater than (b_h,b_l) */ -UNIV_INLINE -int -ut_pair_cmp( - ulint a_h, - ulint a_l, - ulint b_h, - ulint b_l) -{ - if (a_h < b_h) { - return(-1); - } - if (a_h > b_h) { - return(1); - } - return(ut_ulint_cmp(a_l, b_l)); -} - -/*************************************************************//** -Calculates fast the 2-logarithm of a number, rounded upward to an -integer. -@return logarithm in the base 2, rounded upward */ -UNIV_INLINE -ulint -ut_2_log( -/*=====*/ - ulint n) /*!< in: number != 0 */ -{ - ulint res; - - res = 0; - - ut_ad(n > 0); - - n = n - 1; - - for (;;) { - n = n / 2; - - if (n == 0) { - break; - } - - res++; - } - - return(res + 1); -} - -/*************************************************************//** -Calculates 2 to power n. -@return 2 to power n */ -UNIV_INLINE -ulint -ut_2_exp( -/*=====*/ - ulint n) /*!< in: number */ -{ - return((ulint) 1 << n); -} diff --git a/storage/innobase/log/log0sync.cc b/storage/innobase/log/log0sync.cc index 0de283ca2bb85..f6ca440efa8b9 100644 --- a/storage/innobase/log/log0sync.cc +++ b/storage/innobase/log/log0sync.cc @@ -68,6 +68,7 @@ Note that if write operation is very fast, a) or b) can be fine as alternative. #include #endif +#include #include #include #include diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 8a9fa9c060ddb..498489a50a33a 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -923,7 +923,8 @@ void mtr_t::page_lock_upgrade(const buf_block_t &block) (MTR_MEMO_PAGE_SX_FIX | MTR_MEMO_PAGE_X_FIX)); #ifdef BTR_CUR_HASH_ADAPT - ut_ad(!block.index || !block.index->freed()); + ut_d(if (dict_index_t *index= block.index)) + ut_ad(!index->freed()); #endif /* BTR_CUR_HASH_ADAPT */ } diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 0721eb97c56b6..592521a5da1b0 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -33,243 +33,732 @@ Created 10/4/1994 Heikki Tuuri #include "rem0cmp.h" #include "gis0rtree.h" -#include - #ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_SEARCH_PERF_STAT -static ulint page_cur_short_succ; -# endif /* UNIV_SEARCH_PERF_STAT */ +/** Get the pad character code point for a type. +@param type +@return pad character code point +@retval ULINT_UNDEFINED if no padding is specified */ +static ulint cmp_get_pad_char(const dtype_t &type) noexcept +{ + switch (type.mtype) { + default: + break; + case DATA_FIXBINARY: + case DATA_BINARY: + if (dtype_get_charset_coll(type.prtype) == + DATA_MYSQL_BINARY_CHARSET_COLL) + /* Starting from 5.0.18, we do not pad VARBINARY or BINARY columns. */ + break; + /* Fall through */ + case DATA_CHAR: + case DATA_VARCHAR: + case DATA_MYSQL: + case DATA_VARMYSQL: + /* Space is the padding character for all char and binary + strings, and starting from 5.0.3, also for TEXT strings. */ + return 0x20; + case DATA_BLOB: + if (!(type.prtype & DATA_BINARY_TYPE)) + return 0x20; + } -/** Try a search shortcut based on the last insert. -@param[in] block index page -@param[in] index index tree -@param[in] tuple search key -@param[in,out] iup_matched_fields already matched fields in the -upper limit record -@param[in,out] ilow_matched_fields already matched fields in the -lower limit record -@param[out] cursor page cursor -@return true on success */ -UNIV_INLINE -bool -page_cur_try_search_shortcut( - const buf_block_t* block, - const dict_index_t* index, - const dtuple_t* tuple, - ulint* iup_matched_fields, - ulint* ilow_matched_fields, - page_cur_t* cursor) + /* No padding specified */ + return ULINT_UNDEFINED; +} + +/** Compare a data tuple to a physical record. +@param page B-tree index leaf page +@param rec PAGE_LAST_INSERT record +@param index index B-tree +@param tuple search key +@param match matched fields << 16 | bytes +@param comp nonzero if ROW_FORMAT=REDUNDANT is not being used +@return the comparison result of dtuple and rec +@retval 0 if dtuple is equal to rec +@retval negative if dtuple is less than rec +@retval positive if dtuple is greater than rec */ +static int cmp_dtuple_rec_bytes(const page_t *page, + const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, int *match, ulint comp) + noexcept { - const rec_t* rec; - const rec_t* next_rec; - ulint low_match; - ulint up_match; - ibool success = FALSE; - const page_t* page = buf_block_get_frame(block); - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - rec_offs_init(offsets_); + ut_ad(dtuple_check_typed(&tuple)); + ut_ad(page_is_leaf(page)); + ut_ad(!(REC_INFO_MIN_REC_FLAG & dtuple_get_info_bits(&tuple))); + ut_ad(!!comp == index.table->not_redundant()); + ut_ad(!index.is_ibuf()); - ut_ad(dtuple_check_typed(tuple)); - ut_ad(page_is_leaf(page)); + if (UNIV_UNLIKELY(REC_INFO_MIN_REC_FLAG & rec_get_info_bits(rec, comp))) + { + ut_ad(page_rec_is_first(rec, page)); + ut_ad(!page_has_prev(page)); + ut_ad(rec_is_metadata(rec, index)); + return 1; + } - rec = page_header_get_ptr(page, PAGE_LAST_INSERT); - offsets = rec_get_offsets(rec, index, offsets, index->n_core_fields, - dtuple_get_n_fields(tuple), &heap); + ulint cur_field= *match >> 16; + ulint cur_bytes= uint16_t(*match); + ulint n_cmp= dtuple_get_n_fields_cmp(&tuple); + int ret= 0; - ut_ad(rec); - ut_ad(page_rec_is_user_rec(rec)); + ut_ad(n_cmp <= dtuple_get_n_fields(&tuple)); + ut_ad(cur_field <= n_cmp); + ut_ad(cur_field + !!cur_bytes <= + (index.is_primary() ? index.db_roll_ptr() : index.n_core_fields) || + index.is_ibuf()); - low_match = up_match = std::min(*ilow_matched_fields, - *iup_matched_fields); + if (UNIV_LIKELY(comp != 0)) + { + const unsigned n_core_null_bytes= index.n_core_null_bytes; + const byte *nulls= rec - REC_N_NEW_EXTRA_BYTES; + const byte *lens= --nulls - n_core_null_bytes; + byte null_mask= 1; + + size_t i= 0; + const dict_field_t *field= index.fields; + const dict_field_t *const end= field + tuple.n_fields_cmp; + const byte *f= rec; + do + { + const dict_col_t *col= field->col; + if (col->is_nullable()) + { + const int is_null{*nulls & null_mask}; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic push +# if __GNUC__ < 12 || defined WITH_UBSAN +# pragma GCC diagnostic ignored "-Wconversion" +# endif +#endif + null_mask<<= 1; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic pop +#endif + if (UNIV_UNLIKELY(!null_mask)) + null_mask= 1, nulls--; + if (is_null) + { + if (i < cur_field || tuple.fields[i].len == UNIV_SQL_NULL) + continue; + cur_bytes= 0; + ret= 1; + break; + } + } - if (cmp_dtuple_rec_with_match(tuple, rec, offsets, &low_match) < 0) { - goto exit_func; - } + size_t len= field->fixed_len; - if (!(next_rec = page_rec_get_next_const(rec))) { - goto exit_func; - } + if (!len) + { + len= *lens--; + if (UNIV_UNLIKELY(len & 0x80) && DATA_BIG_COL(col)) + { + len<<= 8; + len|= *lens--; + ut_ad(!(len & 0x4000)); + len&= 0x3fff; + } + } - if (!page_rec_is_supremum(next_rec)) { - offsets = rec_get_offsets(next_rec, index, offsets, - index->n_core_fields, - dtuple_get_n_fields(tuple), &heap); + if (i >= cur_field) + { + const dfield_t *const df= dtuple_get_nth_field(&tuple, i); + ut_ad(!dfield_is_ext(df)); + if (df->len == UNIV_SQL_NULL) + { + ut_ad(cur_bytes == 0); + less: + ret= -1; + goto non_redundant_order_resolved; + } - if (cmp_dtuple_rec_with_match(tuple, next_rec, offsets, - &up_match) >= 0) { - goto exit_func; - } + switch (df->type.mtype) { + case DATA_FIXBINARY: + case DATA_BINARY: + case DATA_INT: + case DATA_SYS_CHILD: + case DATA_SYS: + break; + case DATA_BLOB: + if (df->type.prtype & DATA_BINARY_TYPE) + break; + /* fall through */ + default: + cur_bytes= 0; + ret= cmp_data_data(df->type.mtype, df->type.prtype, + static_cast(df->data), df->len, + f, len); + if (ret) + goto non_redundant_order_resolved; + goto next_field; + } - *iup_matched_fields = up_match; - } + /* Set the pointers at the current byte */ + const byte *rec_b_ptr= f + cur_bytes; + const byte *dtuple_b_ptr= + static_cast(df->data) + cur_bytes; + /* Compare then the fields */ + for (const ulint pad= cmp_get_pad_char(df->type);; cur_bytes++) + { + const bool eod= df->len <= cur_bytes; + ulint rec_byte= pad, dtuple_byte= pad; + + if (len > cur_bytes) + rec_byte= *rec_b_ptr++; + else if (eod) + break; + else if (rec_byte == ULINT_UNDEFINED) + { + greater: + ret= 1; + goto non_redundant_order_resolved; + } + + if (!eod) + dtuple_byte= *dtuple_b_ptr++; + else if (dtuple_byte == ULINT_UNDEFINED) + goto less; + + if (dtuple_byte == rec_byte); + else if (dtuple_byte < rec_byte) + goto less; + else + goto greater; + } - page_cur_position(rec, block, cursor); + cur_bytes= 0; + } - *ilow_matched_fields = low_match; + next_field: + f+= len; + } + while (i++, ++field < end); -#ifdef UNIV_SEARCH_PERF_STAT - page_cur_short_succ++; -#endif - success = TRUE; -exit_func: - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - return(success); + ut_ad(cur_bytes == 0); + non_redundant_order_resolved: + ut_ad(i >= cur_field); + cur_field= i; + } + else + { + for (; cur_field < n_cmp; cur_field++) + { + const dfield_t *df= dtuple_get_nth_field(&tuple, cur_field); + ut_ad(!dfield_is_ext(df)); + size_t len; + const byte *rec_b_ptr= rec_get_nth_field_old(rec, cur_field, &len); + /* If we have matched yet 0 bytes, it may be that one or + both the fields are SQL null, or the record or dtuple may be + the predefined minimum record. */ + if (df->len == UNIV_SQL_NULL) + { + ut_ad(cur_bytes == 0); + if (len == UNIV_SQL_NULL) + continue; + redundant_less: + ret= -1; + goto order_resolved; + } + else if (len == UNIV_SQL_NULL) + { + ut_ad(cur_bytes == 0); + /* We define the SQL null to be the smallest possible value */ + redundant_greater: + ret= 1; + goto order_resolved; + } + + switch (df->type.mtype) { + case DATA_FIXBINARY: + case DATA_BINARY: + case DATA_INT: + case DATA_SYS_CHILD: + case DATA_SYS: + break; + case DATA_BLOB: + if (df->type.prtype & DATA_BINARY_TYPE) + break; + /* fall through */ + default: + ret= cmp_data_data(df->type.mtype, df->type.prtype, + static_cast(df->data), df->len, + rec_b_ptr, len); + cur_bytes= 0; + if (!ret) + continue; + goto order_resolved; + } + + /* Set the pointers at the current byte */ + rec_b_ptr+= cur_bytes; + const byte *dtuple_b_ptr= static_cast(df->data) + + cur_bytes; + /* Compare then the fields */ + for (const ulint pad= cmp_get_pad_char(df->type);; cur_bytes++) + { + const bool eod= df->len <= cur_bytes; + ulint rec_byte= pad, dtuple_byte= pad; + + if (len > cur_bytes) + rec_byte= *rec_b_ptr++; + else if (eod) + break; + else if (rec_byte == ULINT_UNDEFINED) + goto redundant_greater; + + if (!eod) + dtuple_byte= *dtuple_b_ptr++; + else if (dtuple_byte == ULINT_UNDEFINED) + goto redundant_less; + + if (dtuple_byte == rec_byte); + else if (dtuple_byte < rec_byte) + goto redundant_less; + else if (dtuple_byte > rec_byte) + goto redundant_greater; + } + + cur_bytes= 0; + } + + ut_ad(cur_bytes == 0); + } + +order_resolved: + *match= int(cur_field << 16 | cur_bytes); + return ret; } /** Try a search shortcut based on the last insert. -@param[in] block index page -@param[in] index index tree -@param[in] tuple search key -@param[in,out] iup_matched_fields already matched fields in the -upper limit record -@param[in,out] iup_matched_bytes already matched bytes in the -first partially matched field in the upper limit record -@param[in,out] ilow_matched_fields already matched fields in the -lower limit record -@param[in,out] ilow_matched_bytes already matched bytes in the -first partially matched field in the lower limit record -@param[out] cursor page cursor +@param page B-tree index leaf page +@param rec PAGE_LAST_INSERT record +@param index index B-tree +@param tuple search key +@param iup_fields matched fields in the upper limit record +@param ilow_fields matched fields in the low limit record +@param iup_bytes matched bytes after iup_fields +@param ilow_bytes matched bytes after ilow_fields @return true on success */ -UNIV_INLINE -bool -page_cur_try_search_shortcut_bytes( - const buf_block_t* block, - const dict_index_t* index, - const dtuple_t* tuple, - ulint* iup_matched_fields, - ulint* iup_matched_bytes, - ulint* ilow_matched_fields, - ulint* ilow_matched_bytes, - page_cur_t* cursor) +static bool page_cur_try_search_shortcut_bytes(const page_t *page, + const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, + uint16_t *iup_fields, + uint16_t *ilow_fields, + uint16_t *iup_bytes, + uint16_t *ilow_bytes) noexcept { - const rec_t* rec; - const rec_t* next_rec; - ulint low_match; - ulint low_bytes; - ulint up_match; - ulint up_bytes; - ibool success = FALSE; - const page_t* page = buf_block_get_frame(block); - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - rec_offs_init(offsets_); + ut_ad(page_rec_is_user_rec(rec)); + int low= int(*ilow_fields << 16 | *ilow_bytes); + int up= int(*iup_fields << 16 | *iup_bytes); + up= low= std::min(low, up); + const auto comp= page_is_comp(page); + if (cmp_dtuple_rec_bytes(page, rec, index, tuple, &low, comp) < 0) + return false; + const rec_t *next; + if (UNIV_LIKELY(comp != 0)) + { + if (!(next= page_rec_next_get(page, rec))) + return false; + if (next != page + PAGE_NEW_SUPREMUM) + { + cmp_up: + if (cmp_dtuple_rec_bytes(page, rec, index, tuple, &up, comp) >= 0) + return false; + *iup_fields= uint16_t(up >> 16); + *iup_bytes= uint16_t(up); + } + } + else + { + if (!(next= page_rec_next_get(page, rec))) + return false; + if (next != page + PAGE_OLD_SUPREMUM) + goto cmp_up; + } - ut_ad(dtuple_check_typed(tuple)); - ut_ad(page_is_leaf(page)); + *ilow_fields= uint16_t(low >> 16); + *ilow_bytes= uint16_t(low); + return true; +} - rec = page_header_get_ptr(page, PAGE_LAST_INSERT); - offsets = rec_get_offsets(rec, index, offsets, index->n_core_fields, - dtuple_get_n_fields(tuple), &heap); +/** Search the right position for a page cursor. +@param tuple search key +@param mode search mode +@param iup_fields matched fields in the upper limit record +@param ilow_fields matched fields in the low limit record +@param cursor page cursor +@param iup_bytes matched bytes after iup_fields +@param ilow_bytes matched bytes after ilow_fields +@return whether the first partially matched field in the lower limit record, +or the page is corrupted */ +bool page_cur_search_with_match_bytes(const dtuple_t &tuple, + page_cur_mode_t mode, + uint16_t *iup_fields, + uint16_t *ilow_fields, + page_cur_t *cursor, + uint16_t *iup_bytes, + uint16_t *ilow_bytes) + noexcept +{ + ut_ad(dtuple_validate(&tuple)); + ut_ad(!(tuple.info_bits & REC_INFO_MIN_REC_FLAG)); + ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE || + mode == PAGE_CUR_G || mode == PAGE_CUR_GE); + const dict_index_t &index= *cursor->index; + const buf_block_t *const block= cursor->block; + const page_t *const page= block->page.frame; + ut_ad(page_is_leaf(page)); + ut_d(page_check_dir(page)); +#ifdef UNIV_ZIP_DEBUG + if (const page_zip_des_t *page_zip= buf_block_get_page_zip(block)) + ut_a(page_zip_validate(page_zip, page, &index)); +#endif /* UNIV_ZIP_DEBUG */ + const auto comp= page_is_comp(page); - ut_ad(rec); - ut_ad(page_rec_is_user_rec(rec)); - if (ut_pair_cmp(*ilow_matched_fields, *ilow_matched_bytes, - *iup_matched_fields, *iup_matched_bytes) < 0) { - up_match = low_match = *ilow_matched_fields; - up_bytes = low_bytes = *ilow_matched_bytes; - } else { - up_match = low_match = *iup_matched_fields; - up_bytes = low_bytes = *iup_matched_bytes; - } + if (mode != PAGE_CUR_LE || page_get_direction(page) != PAGE_RIGHT); + else if (uint16_t last= page_header_get_offs(page, PAGE_LAST_INSERT)) + { + const rec_t *rec= page + last; + if (page_header_get_field(page, PAGE_N_DIRECTION) > 3 && + page_cur_try_search_shortcut_bytes(page, rec, index, tuple, + iup_fields, ilow_fields, + iup_bytes, ilow_bytes)) + { + page_cur_position(rec, block, cursor); + return false; + } + } - if (cmp_dtuple_rec_with_match_bytes( - tuple, rec, index, offsets, &low_match, &low_bytes) < 0) { - goto exit_func; - } + /* If mode PAGE_CUR_G is specified, we are trying to position the + cursor to answer a query of the form "tuple < X", where tuple is the + input parameter, and X denotes an arbitrary physical record on the + page. We want to position the cursor on the first X which satisfies + the condition. */ + int up_cmp= int(*iup_fields << 16 | *iup_bytes); + int low_cmp= int(*ilow_fields << 16 | *ilow_bytes); + + /* Perform binary search. First the search is done through the page + directory, after that as a linear search in the list of records + owned by the upper limit directory slot. */ + size_t low= 0, up= ulint{page_dir_get_n_slots(page)} - 1; + const rec_t *mid_rec; + + /* Perform binary search until the lower and upper limit directory + slots come to the distance 1 of each other */ + while (up - low > 1) + { + const size_t mid= (low + up) / 2; + mid_rec= page_dir_slot_get_rec_validate(page_dir_get_nth_slot(page, mid)); + if (UNIV_UNLIKELY(!mid_rec)) + return true; + int cur= std::min(low_cmp, up_cmp); + int cmp= cmp_dtuple_rec_bytes(page, mid_rec, index, tuple, &cur, comp); + if (cmp > 0) + low_slot_match: + low= mid, low_cmp= cur; + else if (cmp) + up_slot_match: + up= mid, up_cmp= cur; + else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE) + goto low_slot_match; + else + goto up_slot_match; + } - if (!(next_rec = page_rec_get_next_const(rec))) { - goto exit_func; - } + const rec_t *up_rec= + page_dir_slot_get_rec_validate(page_dir_get_nth_slot(page, up));; + const rec_t *low_rec= + page_dir_slot_get_rec_validate(page_dir_get_nth_slot(page, low)); + if (UNIV_UNLIKELY(!low_rec || !up_rec)) + return true; - if (!page_rec_is_supremum(next_rec)) { - offsets = rec_get_offsets(next_rec, index, offsets, - index->n_core_fields, - dtuple_get_n_fields(tuple), &heap); + /* Perform linear search until the upper and lower records come to + distance 1 of each other. */ - if (cmp_dtuple_rec_with_match_bytes( - tuple, next_rec, index, offsets, - &up_match, &up_bytes) - >= 0) { - goto exit_func; - } + for (;;) + { + mid_rec= comp + ? page_rec_next_get(page, low_rec) + : page_rec_next_get(page, low_rec); + if (!mid_rec) + return true; + if (mid_rec == up_rec) + break; - *iup_matched_fields = up_match; - *iup_matched_bytes = up_bytes; - } + int cur= std::min(low_cmp, up_cmp); + int cmp; + + if (UNIV_UNLIKELY(rec_get_info_bits(mid_rec, comp) & + REC_INFO_MIN_REC_FLAG)) + { + ut_ad(!page_has_prev(page_align(mid_rec))); + ut_ad(rec_is_metadata(mid_rec, index)); + goto low_rec_match; + } - page_cur_position(rec, block, cursor); + cmp= cmp_dtuple_rec_bytes(page, mid_rec, index, tuple, &cur, comp); - *ilow_matched_fields = low_match; - *ilow_matched_bytes = low_bytes; + if (cmp > 0) + low_rec_match: + low_rec= mid_rec, low_cmp= cur; + else if (cmp) + up_rec_match: + up_rec= mid_rec, up_cmp= cur; + else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE) + goto low_rec_match; + else + goto up_rec_match; + } + + page_cur_position(mode <= PAGE_CUR_GE ? up_rec : low_rec, block, cursor); + *iup_fields= uint16_t(up_cmp >> 16), *iup_bytes= uint16_t(up_cmp); + *ilow_fields= uint16_t(low_cmp >> 16), *ilow_bytes= uint16_t(low_cmp); + return false; +} -#ifdef UNIV_SEARCH_PERF_STAT - page_cur_short_succ++; +static int cmp_dtuple_rec_leaf(const dtuple_t &dtuple, const rec_t *rec, + const dict_index_t &index, + uint16_t *matched_fields, ulint comp) noexcept +{ + ut_ad(dtuple_check_typed(&dtuple)); + ut_ad(page_rec_is_leaf(rec)); + ut_ad(!!comp == index.table->not_redundant()); + ulint cur_field= *matched_fields; + ut_ad(dtuple.n_fields_cmp > 0); + ut_ad(dtuple.n_fields_cmp <= index.n_core_fields || index.is_ibuf()); + ut_ad(cur_field <= dtuple.n_fields_cmp); + int ret= 0; + + if (UNIV_LIKELY(comp != 0)) + { + const unsigned n_core_null_bytes= index.n_core_null_bytes; + const byte *nulls= rec - REC_N_NEW_EXTRA_BYTES; + const byte *lens= --nulls - n_core_null_bytes; + byte null_mask= 1; + + size_t i= 0; + const dict_field_t *field= index.fields; + const dict_field_t *const end= field + dtuple.n_fields_cmp; + const byte *f= rec; + do + { + const dict_col_t *col= field->col; + if (col->is_nullable()) + { + const int is_null{*nulls & null_mask}; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic push +# if __GNUC__ < 12 || defined WITH_UBSAN +# pragma GCC diagnostic ignored "-Wconversion" +# endif #endif - success = TRUE; -exit_func: - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - return(success); + null_mask<<= 1; +#if defined __GNUC__ && !defined __clang__ +# pragma GCC diagnostic pop +#endif + if (UNIV_UNLIKELY(!null_mask)) + null_mask= 1, nulls--; + if (is_null) + { + if (i < cur_field || dtuple.fields[i].len == UNIV_SQL_NULL) + continue; + ret= 1; + break; + } + } + + size_t len= field->fixed_len; + + if (!len) + { + len= *lens--; + if (UNIV_UNLIKELY(len & 0x80) && DATA_BIG_COL(col)) + { + len<<= 8; + len|= *lens--; + ut_ad(!(len & 0x4000)); + len&= 0x3fff; + } + } + + if (i >= cur_field) + { + const dfield_t *df= dtuple_get_nth_field(&dtuple, i); + ut_ad(!dfield_is_ext(df)); + ret= cmp_data_data(df->type.mtype, df->type.prtype, + static_cast(df->data), df->len, + f, len); + if (ret) + break; + } + + f+= len; + } + while (i++, ++field < end); + ut_ad(i >= cur_field); + *matched_fields= uint16_t(i); + } + else + { + for (; cur_field < dtuple.n_fields_cmp; cur_field++) + { + const dfield_t *df= dtuple_get_nth_field(&dtuple, cur_field); + ut_ad(!dfield_is_ext(df)); + size_t len; + const byte *f= rec_get_nth_field_old(rec, cur_field, &len); + ret= cmp_data_data(df->type.mtype, df->type.prtype, + static_cast(df->data), df->len, f, len); + if (ret) + break; + } + *matched_fields= uint16_t(cur_field); + } + return ret; } -#endif /* BTR_CUR_HASH_ADAPT */ -#ifdef PAGE_CUR_LE_OR_EXTENDS -/****************************************************************//** -Checks if the nth field in a record is a character type field which extends -the nth field in tuple, i.e., the field is longer or equal in length and has -common first characters. -@return TRUE if rec field extends tuple field */ -static -ibool -page_cur_rec_field_extends( -/*=======================*/ - const dtuple_t* tuple, /*!< in: data tuple */ - const rec_t* rec, /*!< in: record */ - const rec_offs* offsets,/*!< in: array returned by rec_get_offsets() */ - ulint n) /*!< in: compare nth field */ +bool btr_cur_t::check_mismatch(const dtuple_t& tuple, page_cur_mode_t mode, + ulint comp) noexcept { - const dtype_t* type; - const dfield_t* dfield; - const byte* rec_f; - ulint rec_f_len; - - ut_ad(rec_offs_validate(rec, NULL, offsets)); - dfield = dtuple_get_nth_field(tuple, n); - - type = dfield_get_type(dfield); - - rec_f = rec_get_nth_field(rec, offsets, n, &rec_f_len); - - if (type->mtype == DATA_VARCHAR - || type->mtype == DATA_CHAR - || type->mtype == DATA_FIXBINARY - || type->mtype == DATA_BINARY - || type->mtype == DATA_BLOB - || DATA_GEOMETRY_MTYPE(type->mtype) - || type->mtype == DATA_VARMYSQL - || type->mtype == DATA_MYSQL) { - - if (dfield_get_len(dfield) != UNIV_SQL_NULL - && rec_f_len != UNIV_SQL_NULL - && rec_f_len >= dfield_get_len(dfield) - && !cmp_data_data(type->mtype, type->prtype, - dfield_get_data(dfield), - dfield_get_len(dfield), - rec_f, dfield_get_len(dfield))) { - - return(TRUE); - } - } + const rec_t *rec= page_cur.rec; + uint16_t match= 0; + int cmp= cmp_dtuple_rec_leaf(tuple, rec, *index(), &match, comp); + + switch (mode) { + const rec_t *other; + const page_t *page; + case PAGE_CUR_GE: + if (cmp > 0) + return true; + up_match= match; + if (match >= dict_index_get_n_unique_in_tree(index())) + return false; + goto check_ge; + case PAGE_CUR_G: + if (cmp >= 0) + return true; + check_ge: + match= 0; + if (!(other= page_rec_get_prev_const(rec))) + return true; + page= page_cur.block->page.frame; + if (uintptr_t(other - page) == + (comp ? PAGE_NEW_INFIMUM : PAGE_OLD_INFIMUM)) + return page_has_prev(page); + if (UNIV_LIKELY(comp != 0)) + switch (rec_get_status(other)) { + case REC_STATUS_INSTANT: + case REC_STATUS_ORDINARY: + break; + default: + return true; + } + cmp= cmp_dtuple_rec_leaf(tuple, other, *index(), &match, comp); + return (mode == PAGE_CUR_GE) ? cmp <= 0 : cmp < 0; + case PAGE_CUR_LE: + if (cmp < 0) + return true; + low_match= match; + goto check_le; + case PAGE_CUR_L: + if (cmp <= 0) + return true; + check_le: + match= 0; + ut_ad(!page_rec_is_supremum(rec)); + page= page_cur.block->page.frame; + if (UNIV_LIKELY(comp != 0)) + { + other= page_rec_next_get(page, rec); + if (!other) + return true; + if (other - page == PAGE_NEW_SUPREMUM) + { + le_supremum: + if (page_has_next(page)) + return true; + up_match= 0; + return false; + } + switch (rec_get_status(other)) { + case REC_STATUS_INSTANT: + case REC_STATUS_ORDINARY: + break; + default: + return true; + } + } + else + { + other= page_rec_next_get(page, rec); + if (!other) + return true; + if (other - page == PAGE_OLD_SUPREMUM) + goto le_supremum; + } + cmp= cmp_dtuple_rec_leaf(tuple, other, *index(), &match, comp); + if (mode != PAGE_CUR_LE) + return cmp > 0; + up_match= match; + return cmp >= 0; + default: + ut_ad("invalid mode" == 0); + return true; + } +} + +/** Try a search shortcut based on the last insert. +@param page index page +@param rec PAGE_LAST_INSERT record +@param index index tree +@param tuple search key +@param iup matched fields in the upper limit record +@param ilow matched fields in the lower limit record +@return record +@return nullptr if the tuple was not found */ +static bool page_cur_try_search_shortcut(const page_t *page, const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, + uint16_t *iup, uint16_t *ilow) + noexcept +{ + ut_ad(dtuple_check_typed(&tuple)); + ut_ad(page_is_leaf(page)); + auto comp= page_is_comp(page); + ut_ad(page_rec_is_user_rec(rec)); + + uint16_t low= std::min(*ilow, *iup), up= low; + + if (cmp_dtuple_rec_leaf(tuple, rec, index, &low, comp) < 0) + return false; + + if (comp) + { + rec= page_rec_next_get(page, rec); + if (!rec) + return false; + if (rec != page + PAGE_NEW_SUPREMUM) + { + compare_next: + if (cmp_dtuple_rec_leaf(tuple, rec, index, &up, comp) >= 0) + return false; + *iup= up; + } + } + else + { + rec= page_rec_next_get(page, rec); + if (!rec) + return false; + if (rec != page + PAGE_OLD_SUPREMUM) + goto compare_next; + } - return(FALSE); + *ilow= low; + return true; } -#endif /* PAGE_CUR_LE_OR_EXTENDS */ +#endif /* BTR_CUR_HASH_ADAPT */ /****************************************************************//** Searches the right position for a page cursor. */ @@ -280,10 +769,10 @@ page_cur_search_with_match( page_cur_mode_t mode, /*!< in: PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, or PAGE_CUR_GE */ - ulint* iup_matched_fields, + uint16_t* iup_matched_fields, /*!< in/out: already matched fields in upper limit record */ - ulint* ilow_matched_fields, + uint16_t* ilow_matched_fields, /*!< in/out: already matched fields in lower limit record */ page_cur_t* cursor, /*!< out: page cursor */ @@ -296,9 +785,9 @@ page_cur_search_with_match( const rec_t* up_rec; const rec_t* low_rec; const rec_t* mid_rec; - ulint up_matched_fields; - ulint low_matched_fields; - ulint cur_matched_fields; + uint16_t up_matched_fields; + uint16_t low_matched_fields; + uint16_t cur_matched_fields; int cmp; const dict_index_t* const index = cursor->index; const buf_block_t* const block = cursor->block; @@ -311,17 +800,9 @@ page_cur_search_with_match( rec_offs_init(offsets_); ut_ad(dtuple_validate(tuple)); -#ifdef UNIV_DEBUG -# ifdef PAGE_CUR_DBG - if (mode != PAGE_CUR_DBG) -# endif /* PAGE_CUR_DBG */ -# ifdef PAGE_CUR_LE_OR_EXTENDS - if (mode != PAGE_CUR_LE_OR_EXTENDS) -# endif /* PAGE_CUR_LE_OR_EXTENDS */ - ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE - || mode == PAGE_CUR_G || mode == PAGE_CUR_GE - || dict_index_is_spatial(index)); -#endif /* UNIV_DEBUG */ + ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE + || mode == PAGE_CUR_G || mode == PAGE_CUR_GE + || index->is_spatial()); page = buf_block_get_frame(block); #ifdef UNIV_ZIP_DEBUG ut_a(!page_zip || page_zip_validate(page_zip, page, index)); @@ -330,45 +811,34 @@ page_cur_search_with_match( ut_d(page_check_dir(page)); const ulint n_core = page_is_leaf(page) ? index->n_core_fields : 0; -#ifdef BTR_CUR_HASH_ADAPT - if (n_core - && page_get_direction(page) == PAGE_RIGHT - && page_header_get_offs(page, PAGE_LAST_INSERT) - && mode == PAGE_CUR_LE - && !index->is_spatial() - && page_header_get_field(page, PAGE_N_DIRECTION) > 3 - && page_cur_try_search_shortcut( - block, index, tuple, - iup_matched_fields, ilow_matched_fields, cursor)) { - return false; - } -# ifdef PAGE_CUR_DBG - if (mode == PAGE_CUR_DBG) { - mode = PAGE_CUR_LE; - } -# endif -#endif /* BTR_CUR_HASH_ADAPT */ - /* If the mode is for R-tree indexes, use the special MBR related compare functions */ - if (index->is_spatial() && mode > PAGE_CUR_LE) { + if (index->is_spatial()) { /* For leaf level insert, we still use the traditional compare function for now */ - if (mode == PAGE_CUR_RTREE_INSERT && n_core) { + if (n_core && mode == PAGE_CUR_RTREE_INSERT) { mode = PAGE_CUR_LE; - } else { + } else if (mode > PAGE_CUR_LE) { return rtr_cur_search_with_match( block, (dict_index_t*)index, tuple, mode, cursor, rtr_info); } +#ifdef BTR_CUR_HASH_ADAPT + } else if (!n_core || mode != PAGE_CUR_LE || !page_is_leaf(page) + || page_get_direction(page) != PAGE_RIGHT) { + } else if (uint16_t last = + page_header_get_offs(page, PAGE_LAST_INSERT)) { + if (page_header_get_field(page, PAGE_N_DIRECTION) > 3 + && page_cur_try_search_shortcut(page, page + last, *index, + *tuple, + iup_matched_fields, + ilow_matched_fields)) { + page_cur_position(page + last, block, cursor); + return false; + } +#endif /* BTR_CUR_HASH_ADAPT */ } - /* The following flag does not work for non-latin1 char sets because - cmp_full_field does not tell how many bytes matched */ -#ifdef PAGE_CUR_LE_OR_EXTENDS - ut_a(mode != PAGE_CUR_LE_OR_EXTENDS); -#endif /* PAGE_CUR_LE_OR_EXTENDS */ - /* If mode PAGE_CUR_G is specified, we are trying to position the cursor to answer a query of the form "tuple < X", where tuple is the input parameter, and X denotes an arbitrary physical record on @@ -412,24 +882,11 @@ page_cur_search_with_match( low_matched_fields = cur_matched_fields; } else if (cmp) { -#ifdef PAGE_CUR_LE_OR_EXTENDS - if (mode == PAGE_CUR_LE_OR_EXTENDS - && page_cur_rec_field_extends( - tuple, mid_rec, offsets, - cur_matched_fields)) { - - goto low_slot_match; - } -#endif /* PAGE_CUR_LE_OR_EXTENDS */ up_slot_match: up = mid; up_matched_fields = cur_matched_fields; - } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE -#ifdef PAGE_CUR_LE_OR_EXTENDS - || mode == PAGE_CUR_LE_OR_EXTENDS -#endif /* PAGE_CUR_LE_OR_EXTENDS */ - ) { + } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE) { goto low_slot_match; } else { @@ -478,38 +935,18 @@ page_cur_search_with_match( low_matched_fields = cur_matched_fields; } else if (cmp) { -#ifdef PAGE_CUR_LE_OR_EXTENDS - if (mode == PAGE_CUR_LE_OR_EXTENDS - && page_cur_rec_field_extends( - tuple, mid_rec, offsets, - cur_matched_fields)) { - - goto low_rec_match; - } -#endif /* PAGE_CUR_LE_OR_EXTENDS */ up_rec_match: up_rec = mid_rec; up_matched_fields = cur_matched_fields; - } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE -#ifdef PAGE_CUR_LE_OR_EXTENDS - || mode == PAGE_CUR_LE_OR_EXTENDS -#endif /* PAGE_CUR_LE_OR_EXTENDS */ - ) { + } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE) { if (!cmp && !cur_matched_fields) { -#ifdef UNIV_DEBUG - mtr_t mtr; - mtr_start(&mtr); - /* We got a match, but cur_matched_fields is 0, it must have REC_INFO_MIN_REC_FLAG */ - ulint rec_info = rec_get_info_bits(mid_rec, - rec_offs_comp(offsets)); - ut_ad(rec_info & REC_INFO_MIN_REC_FLAG); + ut_ad(rec_get_info_bits(mid_rec, + rec_offs_comp(offsets)) + & REC_INFO_MIN_REC_FLAG); ut_ad(!page_has_prev(page)); - mtr_commit(&mtr); -#endif - - cur_matched_fields = dtuple_get_n_fields_cmp(tuple); + cur_matched_fields = tuple->n_fields_cmp; } goto low_rec_match; @@ -519,11 +956,8 @@ page_cur_search_with_match( } } - if (mode <= PAGE_CUR_GE) { - page_cur_position(up_rec, block, cursor); - } else { - page_cur_position(low_rec, block, cursor); - } + page_cur_position(mode <= PAGE_CUR_GE ? up_rec : low_rec, block, + cursor); *iup_matched_fields = up_matched_fields; *ilow_matched_fields = low_matched_fields; @@ -534,270 +968,6 @@ page_cur_search_with_match( return false; } -#ifdef BTR_CUR_HASH_ADAPT -/** Search the right position for a page cursor. -@param[in] block buffer block -@param[in] index index tree -@param[in] tuple key to be searched for -@param[in] mode search mode -@param[in,out] iup_matched_fields already matched fields in the -upper limit record -@param[in,out] iup_matched_bytes already matched bytes in the -first partially matched field in the upper limit record -@param[in,out] ilow_matched_fields already matched fields in the -lower limit record -@param[in,out] ilow_matched_bytes already matched bytes in the -first partially matched field in the lower limit record -@param[out] cursor page cursor */ -bool -page_cur_search_with_match_bytes( - const dtuple_t* tuple, - page_cur_mode_t mode, - ulint* iup_matched_fields, - ulint* iup_matched_bytes, - ulint* ilow_matched_fields, - ulint* ilow_matched_bytes, - page_cur_t* cursor) -{ - ulint up; - ulint low; - const page_t* page; - const rec_t* up_rec; - const rec_t* low_rec; - const rec_t* mid_rec; - ulint up_matched_fields; - ulint up_matched_bytes; - ulint low_matched_fields; - ulint low_matched_bytes; - ulint cur_matched_fields; - ulint cur_matched_bytes; - int cmp; - const dict_index_t* const index = cursor->index; - const buf_block_t* const block = cursor->block; -#ifdef UNIV_ZIP_DEBUG - const page_zip_des_t* page_zip = buf_block_get_page_zip(block); -#endif /* UNIV_ZIP_DEBUG */ - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - rec_offs_init(offsets_); - - ut_ad(dtuple_validate(tuple)); - ut_ad(!(tuple->info_bits & REC_INFO_MIN_REC_FLAG)); -#ifdef UNIV_DEBUG -# ifdef PAGE_CUR_DBG - if (mode != PAGE_CUR_DBG) -# endif /* PAGE_CUR_DBG */ -# ifdef PAGE_CUR_LE_OR_EXTENDS - if (mode != PAGE_CUR_LE_OR_EXTENDS) -# endif /* PAGE_CUR_LE_OR_EXTENDS */ - ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE - || mode == PAGE_CUR_G || mode == PAGE_CUR_GE); -#endif /* UNIV_DEBUG */ - page = buf_block_get_frame(block); -#ifdef UNIV_ZIP_DEBUG - ut_a(!page_zip || page_zip_validate(page_zip, page, index)); -#endif /* UNIV_ZIP_DEBUG */ - - ut_d(page_check_dir(page)); - -#ifdef BTR_CUR_HASH_ADAPT - if (page_is_leaf(page) - && page_get_direction(page) == PAGE_RIGHT - && page_header_get_offs(page, PAGE_LAST_INSERT) - && mode == PAGE_CUR_LE - && page_header_get_field(page, PAGE_N_DIRECTION) > 3 - && page_cur_try_search_shortcut_bytes( - block, index, tuple, - iup_matched_fields, iup_matched_bytes, - ilow_matched_fields, ilow_matched_bytes, - cursor)) { - return false; - } -# ifdef PAGE_CUR_DBG - if (mode == PAGE_CUR_DBG) { - mode = PAGE_CUR_LE; - } -# endif -#endif /* BTR_CUR_HASH_ADAPT */ - - /* The following flag does not work for non-latin1 char sets because - cmp_full_field does not tell how many bytes matched */ -#ifdef PAGE_CUR_LE_OR_EXTENDS - ut_a(mode != PAGE_CUR_LE_OR_EXTENDS); -#endif /* PAGE_CUR_LE_OR_EXTENDS */ - - /* If mode PAGE_CUR_G is specified, we are trying to position the - cursor to answer a query of the form "tuple < X", where tuple is - the input parameter, and X denotes an arbitrary physical record on - the page. We want to position the cursor on the first X which - satisfies the condition. */ - - up_matched_fields = *iup_matched_fields; - up_matched_bytes = *iup_matched_bytes; - low_matched_fields = *ilow_matched_fields; - low_matched_bytes = *ilow_matched_bytes; - - /* Perform binary search. First the search is done through the page - directory, after that as a linear search in the list of records - owned by the upper limit directory slot. */ - - low = 0; - up = ulint(page_dir_get_n_slots(page)) - 1; - - /* Perform binary search until the lower and upper limit directory - slots come to the distance 1 of each other */ - const ulint n_core = page_is_leaf(page) ? index->n_core_fields : 0; - - while (up - low > 1) { - const ulint mid = (low + up) / 2; - mid_rec = page_dir_slot_get_rec_validate( - page_dir_get_nth_slot(page, mid)); - if (UNIV_UNLIKELY(!mid_rec)) { - goto corrupted; - } - - ut_pair_min(&cur_matched_fields, &cur_matched_bytes, - low_matched_fields, low_matched_bytes, - up_matched_fields, up_matched_bytes); - - offsets = rec_get_offsets( - mid_rec, index, offsets_, n_core, - dtuple_get_n_fields_cmp(tuple), &heap); - - cmp = cmp_dtuple_rec_with_match_bytes( - tuple, mid_rec, index, offsets, - &cur_matched_fields, &cur_matched_bytes); - - if (cmp > 0) { -low_slot_match: - low = mid; - low_matched_fields = cur_matched_fields; - low_matched_bytes = cur_matched_bytes; - - } else if (cmp) { -#ifdef PAGE_CUR_LE_OR_EXTENDS - if (mode == PAGE_CUR_LE_OR_EXTENDS - && page_cur_rec_field_extends( - tuple, mid_rec, offsets, - cur_matched_fields)) { - - goto low_slot_match; - } -#endif /* PAGE_CUR_LE_OR_EXTENDS */ -up_slot_match: - up = mid; - up_matched_fields = cur_matched_fields; - up_matched_bytes = cur_matched_bytes; - - } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE -#ifdef PAGE_CUR_LE_OR_EXTENDS - || mode == PAGE_CUR_LE_OR_EXTENDS -#endif /* PAGE_CUR_LE_OR_EXTENDS */ - ) { - goto low_slot_match; - } else { - - goto up_slot_match; - } - } - - low_rec = page_dir_slot_get_rec_validate( - page_dir_get_nth_slot(page, low)); - up_rec = page_dir_slot_get_rec_validate( - page_dir_get_nth_slot(page, up)); - if (UNIV_UNLIKELY(!low_rec || !up_rec)) { -corrupted: - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - return true; - } - - /* Perform linear search until the upper and lower records come to - distance 1 of each other. */ - - for (;;) { - if (const rec_t* next = page_rec_get_next_const(low_rec)) { - if (next == up_rec) { - break; - } - mid_rec = next; - } else { - goto corrupted; - } - ut_pair_min(&cur_matched_fields, &cur_matched_bytes, - low_matched_fields, low_matched_bytes, - up_matched_fields, up_matched_bytes); - - if (UNIV_UNLIKELY(rec_get_info_bits( - mid_rec, - dict_table_is_comp(index->table)) - & REC_INFO_MIN_REC_FLAG)) { - ut_ad(!page_has_prev(page_align(mid_rec))); - ut_ad(!page_rec_is_leaf(mid_rec) - || rec_is_metadata(mid_rec, *index)); - cmp = 1; - goto low_rec_match; - } - - offsets = rec_get_offsets( - mid_rec, index, offsets_, n_core, - dtuple_get_n_fields_cmp(tuple), &heap); - - cmp = cmp_dtuple_rec_with_match_bytes( - tuple, mid_rec, index, offsets, - &cur_matched_fields, &cur_matched_bytes); - - if (cmp > 0) { -low_rec_match: - low_rec = mid_rec; - low_matched_fields = cur_matched_fields; - low_matched_bytes = cur_matched_bytes; - - } else if (cmp) { -#ifdef PAGE_CUR_LE_OR_EXTENDS - if (mode == PAGE_CUR_LE_OR_EXTENDS - && page_cur_rec_field_extends( - tuple, mid_rec, offsets, - cur_matched_fields)) { - - goto low_rec_match; - } -#endif /* PAGE_CUR_LE_OR_EXTENDS */ -up_rec_match: - up_rec = mid_rec; - up_matched_fields = cur_matched_fields; - up_matched_bytes = cur_matched_bytes; - } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE -#ifdef PAGE_CUR_LE_OR_EXTENDS - || mode == PAGE_CUR_LE_OR_EXTENDS -#endif /* PAGE_CUR_LE_OR_EXTENDS */ - ) { - goto low_rec_match; - } else { - - goto up_rec_match; - } - } - - if (mode <= PAGE_CUR_GE) { - page_cur_position(up_rec, block, cursor); - } else { - page_cur_position(low_rec, block, cursor); - } - - *iup_matched_fields = up_matched_fields; - *iup_matched_bytes = up_matched_bytes; - *ilow_matched_fields = low_matched_fields; - *ilow_matched_bytes = low_matched_bytes; - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - return false; -} -#endif /* BTR_CUR_HASH_ADAPT */ - /***********************************************************//** Positions a page cursor on a randomly chosen user record on a page. If there are no user records, sets the cursor on the infimum record. */ diff --git a/storage/innobase/rem/rem0cmp.cc b/storage/innobase/rem/rem0cmp.cc index 536ea224d1815..619fbb1b7b5f5 100644 --- a/storage/innobase/rem/rem0cmp.cc +++ b/storage/innobase/rem/rem0cmp.cc @@ -189,19 +189,8 @@ cmp_decimal(const byte* a, ulint a_length, const byte* b, ulint b_length) return(swap_flag); } -/** Compare two data fields. -@param mtype main type -@param prtype precise type -@param data1 data field -@param len1 length of data1 in bytes, or UNIV_SQL_NULL -@param data2 data field -@param len2 length of data2 in bytes, or UNIV_SQL_NULL -@return the comparison result of data1 and data2 -@retval 0 if data1 is equal to data2 -@retval negative if data1 is less than data2 -@retval positive if data1 is greater than data2 */ -static int cmp_data(ulint mtype, ulint prtype, const byte *data1, ulint len1, - const byte *data2, ulint len2) +int cmp_data_data(ulint mtype, ulint prtype, const byte *data1, ulint len1, + const byte *data2, ulint len2) noexcept { ut_ad(len1 != UNIV_SQL_DEFAULT); ut_ad(len2 != UNIV_SQL_DEFAULT); @@ -308,29 +297,6 @@ static int cmp_data(ulint mtype, ulint prtype, const byte *data1, ulint len1, return len1 > len2 ? 1 : len2 > len1 ? -1 : 0; } -/** Compare two data fields. -@param[in] mtype main type -@param[in] prtype precise type -@param[in] data1 data field -@param[in] len1 length of data1 in bytes, or UNIV_SQL_NULL -@param[in] data2 data field -@param[in] len2 length of data2 in bytes, or UNIV_SQL_NULL -@return the comparison result of data1 and data2 -@retval 0 if data1 is equal to data2 -@retval negative if data1 is less than data2 -@retval positive if data1 is greater than data2 */ -int -cmp_data_data( - ulint mtype, - ulint prtype, - const byte* data1, - ulint len1, - const byte* data2, - ulint len2) -{ - return(cmp_data(mtype, prtype, data1, len1, data2, len2)); -} - /** Compare a data tuple to a physical record. @param[in] dtuple data tuple @param[in] rec B-tree record @@ -347,15 +313,14 @@ cmp_dtuple_rec_with_match_low( const rec_t* rec, const rec_offs* offsets, ulint n_cmp, - ulint* matched_fields) + uint16_t* matched_fields) { - ulint cur_field; /* current field number */ - int ret; /* return value */ + int ret = 0; /* return value */ ut_ad(dtuple_check_typed(dtuple)); ut_ad(rec_offs_validate(rec, NULL, offsets)); - cur_field = *matched_fields; + auto cur_field = *matched_fields; ut_ad(n_cmp > 0); ut_ad(n_cmp <= dtuple_get_n_fields(dtuple)); @@ -368,11 +333,9 @@ cmp_dtuple_rec_with_match_low( ulint tup_info = dtuple_get_info_bits(dtuple); if (UNIV_UNLIKELY(rec_info & REC_INFO_MIN_REC_FLAG)) { - ret = !(tup_info & REC_INFO_MIN_REC_FLAG); - goto order_resolved; + return !(tup_info & REC_INFO_MIN_REC_FLAG); } else if (UNIV_UNLIKELY(tup_info & REC_INFO_MIN_REC_FLAG)) { - ret = -1; - goto order_resolved; + return -1; } } @@ -407,229 +370,15 @@ cmp_dtuple_rec_with_match_low( ut_ad(!dfield_is_ext(dtuple_field)); - ret = cmp_data(type->mtype, type->prtype, - dtuple_b_ptr, dtuple_f_len, - rec_b_ptr, rec_f_len); + ret = cmp_data_data(type->mtype, type->prtype, + dtuple_b_ptr, dtuple_f_len, + rec_b_ptr, rec_f_len); if (ret) { - goto order_resolved; - } - } - - ret = 0; /* If we ran out of fields, dtuple was equal to rec - up to the common fields */ -order_resolved: - *matched_fields = cur_field; - return(ret); -} - -/** Get the pad character code point for a type. -@param[in] type -@return pad character code point -@retval ULINT_UNDEFINED if no padding is specified */ -UNIV_INLINE -ulint -cmp_get_pad_char( - const dtype_t* type) -{ - switch (type->mtype) { - case DATA_FIXBINARY: - case DATA_BINARY: - if (dtype_get_charset_coll(type->prtype) - == DATA_MYSQL_BINARY_CHARSET_COLL) { - /* Starting from 5.0.18, do not pad - VARBINARY or BINARY columns. */ - return(ULINT_UNDEFINED); - } - /* Fall through */ - case DATA_CHAR: - case DATA_VARCHAR: - case DATA_MYSQL: - case DATA_VARMYSQL: - /* Space is the padding character for all char and binary - strings, and starting from 5.0.3, also for TEXT strings. */ - return(0x20); - case DATA_GEOMETRY: - /* DATA_GEOMETRY is binary data, not ASCII-based. */ - return(ULINT_UNDEFINED); - case DATA_BLOB: - if (!(type->prtype & DATA_BINARY_TYPE)) { - return(0x20); - } - /* Fall through */ - default: - /* No padding specified */ - return(ULINT_UNDEFINED); - } -} - -/** Compare a data tuple to a physical record. -@param[in] dtuple data tuple -@param[in] rec B-tree or R-tree index record -@param[in] index index tree -@param[in] offsets rec_get_offsets(rec) -@param[in,out] matched_fields number of completely matched fields -@param[in,out] matched_bytes number of matched bytes in the first -field that is not matched -@return the comparison result of dtuple and rec -@retval 0 if dtuple is equal to rec -@retval negative if dtuple is less than rec -@retval positive if dtuple is greater than rec */ -int -cmp_dtuple_rec_with_match_bytes( - const dtuple_t* dtuple, - const rec_t* rec, - const dict_index_t* index, - const rec_offs* offsets, - ulint* matched_fields, - ulint* matched_bytes) -{ - ut_ad(dtuple_check_typed(dtuple)); - ut_ad(rec_offs_validate(rec, index, offsets)); - ut_ad(!(REC_INFO_MIN_REC_FLAG - & dtuple_get_info_bits(dtuple))); - - if (UNIV_UNLIKELY(REC_INFO_MIN_REC_FLAG - & rec_get_info_bits(rec, rec_offs_comp(offsets)))) { - ut_ad(page_rec_is_first(rec, page_align(rec))); - ut_ad(!page_has_prev(page_align(rec))); - ut_ad(rec_is_metadata(rec, *index)); - return 1; - } - - ulint cur_field = *matched_fields; - ulint cur_bytes = *matched_bytes; - ulint n_cmp = dtuple_get_n_fields_cmp(dtuple); - int ret; - - ut_ad(n_cmp <= dtuple_get_n_fields(dtuple)); - ut_ad(cur_field <= n_cmp); - ut_ad(cur_field + (cur_bytes > 0) <= rec_offs_n_fields(offsets)); - - /* Match fields in a loop; stop if we run out of fields in dtuple - or find an externally stored field */ - - while (cur_field < n_cmp) { - const dfield_t* dfield = dtuple_get_nth_field( - dtuple, cur_field); - const dtype_t* type = dfield_get_type(dfield); - ulint dtuple_f_len = dfield_get_len(dfield); - const byte* dtuple_b_ptr; - const byte* rec_b_ptr; - ulint rec_f_len; - - dtuple_b_ptr = static_cast( - dfield_get_data(dfield)); - - ut_ad(!rec_offs_nth_default(offsets, cur_field)); - rec_b_ptr = rec_get_nth_field(rec, offsets, - cur_field, &rec_f_len); - ut_ad(!rec_offs_nth_extern(offsets, cur_field)); - - /* If we have matched yet 0 bytes, it may be that one or - both the fields are SQL null, or the record or dtuple may be - the predefined minimum record. */ - if (cur_bytes == 0) { - if (dtuple_f_len == UNIV_SQL_NULL) { - if (rec_f_len == UNIV_SQL_NULL) { - - goto next_field; - } - - ret = -1; - goto order_resolved; - } else if (rec_f_len == UNIV_SQL_NULL) { - /* We define the SQL null to be the - smallest possible value of a field - in the alphabetical order */ - - ret = 1; - goto order_resolved; - } - } - - switch (type->mtype) { - case DATA_FIXBINARY: - case DATA_BINARY: - case DATA_INT: - case DATA_SYS_CHILD: - case DATA_SYS: break; - case DATA_BLOB: - if (type->prtype & DATA_BINARY_TYPE) { - break; - } - /* fall through */ - default: - ret = cmp_data(type->mtype, type->prtype, - dtuple_b_ptr, dtuple_f_len, - rec_b_ptr, rec_f_len); - - if (!ret) { - goto next_field; - } - - cur_bytes = 0; - goto order_resolved; - } - - /* Set the pointers at the current byte */ - - rec_b_ptr += cur_bytes; - dtuple_b_ptr += cur_bytes; - /* Compare then the fields */ - - for (const ulint pad = cmp_get_pad_char(type);; - cur_bytes++) { - ulint rec_byte = pad; - ulint dtuple_byte = pad; - - if (rec_f_len <= cur_bytes) { - if (dtuple_f_len <= cur_bytes) { - - goto next_field; - } - - if (rec_byte == ULINT_UNDEFINED) { - ret = 1; - - goto order_resolved; - } - } else { - rec_byte = *rec_b_ptr++; - } - - if (dtuple_f_len <= cur_bytes) { - if (dtuple_byte == ULINT_UNDEFINED) { - ret = -1; - - goto order_resolved; - } - } else { - dtuple_byte = *dtuple_b_ptr++; - } - - if (dtuple_byte < rec_byte) { - ret = -1; - goto order_resolved; - } else if (dtuple_byte > rec_byte) { - ret = 1; - goto order_resolved; - } } - -next_field: - cur_field++; - cur_bytes = 0; } - ut_ad(cur_bytes == 0); - - ret = 0; /* If we ran out of fields, dtuple was equal to rec - up to the common fields */ -order_resolved: *matched_fields = cur_field; - *matched_bytes = cur_bytes; - return(ret); } @@ -649,11 +398,8 @@ cmp_dtuple_rec( const rec_t* rec, const rec_offs* offsets) { - ulint matched_fields = 0; - - ut_ad(rec_offs_validate(rec, NULL, offsets)); - return(cmp_dtuple_rec_with_match(dtuple, rec, offsets, - &matched_fields)); + uint16_t matched_fields= 0; + return cmp_dtuple_rec_with_match(dtuple, rec, offsets, &matched_fields); } /**************************************************************//** @@ -667,11 +413,10 @@ cmp_dtuple_is_prefix_of_rec( const rec_t* rec, /*!< in: physical record */ const rec_offs* offsets)/*!< in: array returned by rec_get_offsets() */ { - ulint n_fields; - ulint matched_fields = 0; + uint16_t matched_fields = 0; ut_ad(rec_offs_validate(rec, NULL, offsets)); - n_fields = dtuple_get_n_fields(dtuple); + uint16_t n_fields = dtuple_get_n_fields(dtuple); if (n_fields > rec_offs_n_fields(offsets)) { ut_ad(0); @@ -710,8 +455,8 @@ cmp_rec_rec_simple_field( rec1_b_ptr = rec_get_nth_field(rec1, offsets1, n, &rec1_f_len); rec2_b_ptr = rec_get_nth_field(rec2, offsets2, n, &rec2_f_len); - return(cmp_data(col->mtype, col->prtype, - rec1_b_ptr, rec1_f_len, rec2_b_ptr, rec2_f_len)); + return cmp_data_data(col->mtype, col->prtype, rec1_b_ptr, rec1_f_len, + rec2_b_ptr, rec2_f_len); } /** Compare two physical records that contain the same number of columns, @@ -914,9 +659,9 @@ cmp_rec_rec( goto order_resolved; } - ret = cmp_data(mtype, prtype, - rec1_b_ptr, rec1_f_len, - rec2_b_ptr, rec2_f_len); + ret = cmp_data_data(mtype, prtype, + rec1_b_ptr, rec1_f_len, + rec2_b_ptr, rec2_f_len); if (ret) { goto order_resolved; } @@ -949,7 +694,7 @@ test_cmp_data_data(ulint len) ut_chrono_t ch(__func__); for (i = 1000000; i > 0; i--) { - i += cmp_data(DATA_INT, 0, zeros, len, zeros, len); + i += cmp_data_data(DATA_INT, 0, zeros, len, zeros, len); } } diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index da3db84440b49..98f838865ae27 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1980,15 +1980,13 @@ row_ins_dupl_error_with_rec( dict_index_t* index, /*!< in: index */ const rec_offs* offsets)/*!< in: rec_get_offsets(rec, index) */ { - ulint matched_fields; - ulint n_unique; ulint i; ut_ad(rec_offs_validate(rec, index, offsets)); - n_unique = dict_index_get_n_unique(index); + const auto n_unique = dict_index_get_n_unique(index); - matched_fields = 0; + uint16_t matched_fields = 0; cmp_dtuple_rec_with_match(entry, rec, offsets, &matched_fields); @@ -2085,7 +2083,6 @@ row_ins_scan_sec_index_for_duplicate( mem_heap_t* offsets_heap) /*!< in/out: memory heap that can be emptied */ { - ulint n_unique; int cmp; ulint n_fields_cmp; btr_pcur_t pcur; @@ -2097,7 +2094,7 @@ row_ins_scan_sec_index_for_duplicate( ut_ad(!index->lock.have_any()); - n_unique = dict_index_get_n_unique(index); + const auto n_unique = dict_index_get_n_unique(index); /* If the secondary index is unique, but one of the fields in the n_unique first fields is NULL, a unique key violation cannot occur, @@ -2225,7 +2222,7 @@ row_ins_duplicate_online( const rec_t* rec, /*!< in: clustered index record */ rec_offs* offsets)/*!< in/out: rec_get_offsets(rec) */ { - ulint fields = 0; + uint16_t fields = 0; /* During rebuild, there should not be any delete-marked rows in the new table. */ @@ -3224,7 +3221,7 @@ row_ins_clust_index_entry( : index->table->is_temporary() ? BTR_NO_LOCKING_FLAG : 0; #endif /* WITH_WSREP */ - const ulint orig_n_fields = entry->n_fields; + const auto orig_n_fields = entry->n_fields; /* For intermediate table during copy alter table, skip the undo log and record lock checking for diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 6f3425aab17e3..c8a43641099fb 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -744,7 +744,7 @@ row_log_table_low_redundant( ulint avail_size; mem_heap_t* heap = NULL; dtuple_t* tuple; - const ulint n_fields = rec_get_n_fields_old(rec); + const auto n_fields = rec_get_n_fields_old(rec); ut_ad(index->n_fields >= n_fields); ut_ad(index->n_fields == n_fields || index->is_instant()); @@ -1701,22 +1701,7 @@ row_log_table_apply_delete_low( if (error) { goto err_exit; } -#ifdef UNIV_DEBUG - switch (btr_pcur_get_btr_cur(pcur)->flag) { - case BTR_CUR_DELETE_REF: - case BTR_CUR_DEL_MARK_IBUF: - case BTR_CUR_DELETE_IBUF: - case BTR_CUR_INSERT_TO_IBUF: - /* We did not request buffering. */ - break; - case BTR_CUR_HASH: - case BTR_CUR_HASH_FAIL: - case BTR_CUR_BINARY: - goto flag_ok; - } - ut_ad(0); -flag_ok: -#endif /* UNIV_DEBUG */ + ut_ad(pcur->btr_cur.flag == BTR_CUR_BINARY); if (page_rec_is_infimum(btr_pcur_get_rec(pcur)) || btr_pcur_get_low_match(pcur) < index->n_uniq) { @@ -1785,22 +1770,8 @@ row_log_table_apply_delete( if (err != DB_SUCCESS) { goto all_done; } -#ifdef UNIV_DEBUG - switch (btr_pcur_get_btr_cur(&pcur)->flag) { - case BTR_CUR_DELETE_REF: - case BTR_CUR_DEL_MARK_IBUF: - case BTR_CUR_DELETE_IBUF: - case BTR_CUR_INSERT_TO_IBUF: - /* We did not request buffering. */ - break; - case BTR_CUR_HASH: - case BTR_CUR_HASH_FAIL: - case BTR_CUR_BINARY: - goto flag_ok; - } - ut_ad(0); -flag_ok: -#endif /* UNIV_DEBUG */ + + ut_ad(btr_pcur_get_btr_cur(&pcur)->flag == BTR_CUR_BINARY); if (page_rec_is_infimum(btr_pcur_get_rec(&pcur)) || btr_pcur_get_low_match(&pcur) < index->n_uniq) { @@ -1934,19 +1905,8 @@ row_log_table_apply_update( return error; } -#ifdef UNIV_DEBUG - switch (btr_pcur_get_btr_cur(&pcur)->flag) { - case BTR_CUR_DELETE_REF: - case BTR_CUR_DEL_MARK_IBUF: - case BTR_CUR_DELETE_IBUF: - case BTR_CUR_INSERT_TO_IBUF: - ut_ad(0);/* We did not request buffering. */ - case BTR_CUR_HASH: - case BTR_CUR_HASH_FAIL: - case BTR_CUR_BINARY: - break; - } -#endif /* UNIV_DEBUG */ + + ut_ad(btr_pcur_get_btr_cur(&pcur)->flag == BTR_CUR_BINARY); ut_ad(!page_rec_is_infimum(btr_pcur_get_rec(&pcur)) && btr_pcur_get_low_match(&pcur) >= index->n_uniq); diff --git a/storage/innobase/row/row0row.cc b/storage/innobase/row/row0row.cc index 29aa31a337407..dee88bd8fdb50 100644 --- a/storage/innobase/row/row0row.cc +++ b/storage/innobase/row/row0row.cc @@ -772,7 +772,7 @@ row_rec_to_index_entry_impl( ut_ad(info_bits == 0); ut_ad(!pad); } - dtuple_t* entry = dtuple_create(heap, rec_len); + dtuple_t* entry = dtuple_create(heap, uint16_t(rec_len)); dfield_t* dfield = entry->fields; dtuple_set_n_fields_cmp(entry, @@ -869,7 +869,7 @@ row_rec_to_index_entry_impl( } if (mblob == 2) { - ulint n_fields = ulint(dfield - entry->fields); + uint16_t n_fields = uint16_t(dfield - entry->fields); ut_ad(entry->n_fields >= n_fields); entry->n_fields = n_fields; } @@ -1300,8 +1300,10 @@ row_search_index_entry( case BTR_CUR_INSERT_TO_IBUF: return(ROW_BUFFERED); +#ifdef BTR_CUR_HASH_ADAPT case BTR_CUR_HASH: case BTR_CUR_HASH_FAIL: +#endif case BTR_CUR_BINARY: break; } diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 2a174ecd1a4a3..0d63f659da2f0 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -2613,9 +2613,9 @@ row_sel_convert_mysql_key_to_innobase( key_end = key_ptr + key_len; - /* Permit us to access any field in the tuple (ULINT_MAX): */ + /* Permit us to access any field in the tuple: */ - dtuple_set_n_fields(tuple, ULINT_MAX); + ut_d(dtuple_set_n_fields(tuple, uint16_t(~0))); dfield = dtuple_get_nth_field(tuple, 0); field = dict_index_get_nth_field(index, 0); @@ -2782,7 +2782,7 @@ row_sel_convert_mysql_key_to_innobase( /* We set the length of tuple to n_fields: we assume that the memory area allocated for it is big enough (usually bigger than n_fields). */ - dtuple_set_n_fields(tuple, n_fields); + dtuple_set_n_fields(tuple, uint16_t(n_fields)); } /**************************************************************//** @@ -3455,7 +3455,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( page_cur_t page_cursor; page_cursor.block = block; page_cursor.index = sec_index; - ulint up_match = 0, low_match = 0; + uint16_t up_match = 0, low_match = 0; ut_ad(!page_cur_search_with_match(tuple, PAGE_CUR_LE, &up_match, &low_match, @@ -4173,8 +4173,7 @@ row_sel_fill_vrow( offsets = rec_get_offsets(rec, index, offsets, index->n_core_fields, ULINT_UNDEFINED, &heap); - *vrow = dtuple_create_with_vcol( - heap, 0, dict_table_get_n_v_cols(index->table)); + *vrow = dtuple_create_with_vcol(heap, 0, index->table->n_v_cols); /* Initialize all virtual row's mtype to DATA_MISSING */ dtuple_init_v_fld(*vrow); @@ -6005,7 +6004,6 @@ row_count_rtree_recs( mem_heap_t* heap; dtuple_t* entry; dtuple_t* search_entry = prebuilt->search_tuple; - ulint entry_len; ulint i; byte* buf; @@ -6016,10 +6014,9 @@ row_count_rtree_recs( heap = mem_heap_create(256); /* Build a search tuple. */ - entry_len = dict_index_get_n_fields(index); - entry = dtuple_create(heap, entry_len); + entry = dtuple_create(heap, index->n_fields); - for (i = 0; i < entry_len; i++) { + for (i = 0; i < index->n_fields; i++) { const dict_field_t* ind_field = dict_index_get_nth_field(index, i); const dict_col_t* col @@ -6797,9 +6794,8 @@ dberr_t row_check_index(row_prebuilt_t *prebuilt, ulint *n_rows) if (prev_entry) { - ulint matched_fields= 0; - int cmp= cmp_dtuple_rec_with_match(prev_entry, rec, offsets, - &matched_fields); + uint16_t matched= 0; + int cmp= cmp_dtuple_rec_with_match(prev_entry, rec, offsets, &matched); const char* msg; if (UNIV_LIKELY(cmp < 0)); @@ -6812,7 +6808,7 @@ dberr_t row_check_index(row_prebuilt_t *prebuilt, ulint *n_rows) << ": " << *prev_entry << ", " << rec_offsets_print(rec, offsets); } - else if (index->is_unique() && matched_fields >= + else if (index->is_unique() && matched >= dict_index_get_n_ordering_defined_by_user(index)) { /* NULL values in unique indexes are considered not to be duplicates */ diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 511948f2d53bd..bd40ff759ef14 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -522,7 +522,7 @@ row_vers_build_cur_vrow_low( const rec_t* version; rec_t* prev_version; mem_heap_t* heap = NULL; - ulint num_v = dict_table_get_n_v_cols(index->table); + const auto num_v = dict_table_get_n_v_cols(index->table); const dfield_t* field; ulint i; bool all_filled = false; diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index e04e8a51f07d0..4c1bbcefd84d6 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -45,8 +45,7 @@ Created 3/26/1996 Heikki Tuuri const dtuple_t trx_undo_metadata = { /* This also works for REC_INFO_METADATA_ALTER, because the delete-mark (REC_INFO_DELETED_FLAG) is ignored when searching. */ - REC_INFO_METADATA_ADD, 0, 0, - NULL, 0, NULL + REC_INFO_METADATA_ADD, 0, 0, 0, nullptr, nullptr #ifdef UNIV_DEBUG , DATA_TUPLE_MAGIC_N #endif /* UNIV_DEBUG */ @@ -580,7 +579,7 @@ trx_undo_rec_get_row_ref( { ut_ad(index->is_primary()); - const ulint ref_len = dict_index_get_n_unique(index); + const uint16_t ref_len = dict_index_get_n_unique(index); dtuple_t* tuple = dtuple_create(heap, ref_len); *ref = tuple; diff --git a/storage/innobase/unittest/CMakeLists.txt b/storage/innobase/unittest/CMakeLists.txt index 9330d231f0686..0e561e0ccab3f 100644 --- a/storage/innobase/unittest/CMakeLists.txt +++ b/storage/innobase/unittest/CMakeLists.txt @@ -25,6 +25,12 @@ ADD_EXECUTABLE(innodb_fts-t innodb_fts-t.cc) TARGET_LINK_LIBRARIES(innodb_fts-t mysys mytap) ADD_DEPENDENCIES(innodb_fts-t GenError) MY_ADD_TEST(innodb_fts) +IF (WITH_INNODB_AHI) + ADD_EXECUTABLE(innodb_ahi-t innodb_ahi-t.cc) + TARGET_LINK_LIBRARIES(innodb_ahi-t mysys mytap) + ADD_DEPENDENCIES(innodb_ahi-t GenError) + MY_ADD_TEST(innodb_ahi) +ENDIF() # See explanation in innobase/CmakeLists.txt IF(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64|powerpc64|s390x") ADD_COMPILE_FLAGS( diff --git a/storage/innobase/unittest/innodb_ahi-t.cc b/storage/innobase/unittest/innodb_ahi-t.cc new file mode 100644 index 0000000000000..487182683afbe --- /dev/null +++ b/storage/innobase/unittest/innodb_ahi-t.cc @@ -0,0 +1,220 @@ +#include "tap.h" + +#define SUX_LOCK_GENERIC +#define NO_ELISION +#define thd_kill_level(thd) 0 +#define srv0mon_h +#define MONITOR_INC(x) +#define MONITOR_INC_VALUE(x,y) +#include "../btr/btr0sea.cc" +const size_t alloc_max_retries= 0; +const byte zero[16384]= { '\0', }; +const byte *field_ref_zero= zero; +ulint srv_buf_pool_curr_size, srv_buf_pool_old_size, srv_buf_pool_size; +ulong srv_page_size_shift= 14, srv_page_size= 1 << 14; +dict_sys_t dict_sys; +buf_pool_t buf_pool; +buf_pool_t::chunk_t::map *buf_pool_t::chunk_t::map_reg; +buf_pool_t::chunk_t::map *buf_pool_t::chunk_t::map_ref; + +void buf_pool_t::free_block(buf_block_t*) noexcept {} +void dict_mem_table_free(dict_table_t*) {} +void dict_mem_index_free(dict_index_t*) {} +buf_block_t *buf_LRU_get_free_block(bool) { return nullptr; } +ibool dtuple_check_typed(const dtuple_t*) { return true; } +bool btr_cur_t::check_mismatch(const dtuple_t&,page_cur_mode_t,ulint) noexcept +{ return false; } +buf_block_t *buf_page_get_gen(const page_id_t, ulint, rw_lock_type_t, + buf_block_t*,ulint,mtr_t*,dberr_t*,bool) noexcept +{ return nullptr; } +bool buf_page_make_young_if_needed(buf_page_t*) { return false; } + +mtr_t::mtr_t()= default; +mtr_t::~mtr_t()= default; +void mtr_t::start() {} +void mtr_t::commit() {} +void mtr_t::rollback_to_savepoint(ulint, ulint) {} +void small_vector_base::grow_by_1(void *, size_t) {} + +void sql_print_error(const char*, ...) {} +ulint ut_find_prime(ulint n) { return n; } +void mem_heap_block_free(mem_block_info_t*, mem_block_info_t*){} +namespace ib { error::~error() {} fatal_or_error::~fatal_or_error() {} } +std::ostream &operator<<(std::ostream &out, const page_id_t) { return out; } + +#ifdef UNIV_DEBUG +byte data_error; +bool srw_lock_debug::have_wr() const noexcept { return false; } +void srw_lock_debug::rd_unlock() noexcept {} +void srw_lock_debug::rd_lock(SRW_LOCK_ARGS(const char*,unsigned)) noexcept {} +#endif + +void page_hash_latch::read_lock_wait() noexcept {} +# ifndef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +template<> void pthread_mutex_wrapper::wr_wait() noexcept {} +# endif +template<> void srw_lock_::rd_wait() noexcept {} +template<> void srw_lock_::wr_wait() noexcept {} +template void ssux_lock_impl::wake() noexcept {} +template void srw_mutex_impl::wake() noexcept {} + +#ifdef UNIV_PFS_MEMORY +PSI_memory_key ut_new_get_key_by_file(uint32_t){ return PSI_NOT_INSTRUMENTED; } +PSI_memory_key mem_key_other, mem_key_std; +#endif + +#ifdef UNIV_PFS_RWLOCK +template +void srw_lock_impl::psi_wr_lock(const char*, unsigned) noexcept {} +template +void srw_lock_impl::psi_rd_lock(const char*, unsigned) noexcept {} + +void dict_sys_t::unlock() noexcept {} +void dict_sys_t::freeze(const char *, unsigned) noexcept {} +void dict_sys_t::unfreeze() noexcept {} +#endif /* UNIV_PFS_RWLOCK */ + +void ut_dbg_assertion_failed(const char *e, const char *file, unsigned line) +{ + fprintf(stderr, "%s:%u: Assertion %s failed\n", file, line, e ? e : ""); + abort(); +} + +int main(int, char **argv) +{ + MY_INIT(*argv); + plan(42); + + btr_search.create(); + btr_search.free(); + + dfield_t fields[2]= {{nullptr,0,0,UNIV_SQL_NULL,{0,DATA_VARCHAR,3,1,1}}, + {(char*)"42",0,0,2,{0,DATA_CHAR,2,1,1}}}; + dtuple_t tuple2{0,2,2,0,fields,nullptr, ut_d(DATA_TUPLE_MAGIC_N) }; + dict_col_t cols[]={{}, {}, {DATA_NOT_NULL,DATA_CHAR,2,1,1,1,0,0,{nullptr,0}}}; + dict_field_t ifields[3]= {{}, {}, {}}; + dict_table_t table{}; + dict_index_t index{}; + index.table= &table; + index.n_uniq= 3; + index.n_nullable= 3; + index.n_fields= 3; + index.n_core_fields= 3; + index.n_core_null_bytes= 1; + index.fields= ifields; + + ifields[0].col= &cols[0]; + ifields[1].col= &cols[2]; + ifields[2].col= &cols[2]; + ifields[1].fixed_len= 2; + ifields[2].fixed_len= 2; + + constexpr uint32_t crc42= 0x2e7d3dcb, crc3z42= 0x9a6e3c2c, + crc2z= 0xf16177d2, crc3z= 0x6064a37a; + + { + btr_cur_t cursor; + cursor.page_cur.index= &index; + cursor.n_bytes_fields= 2; + + ok(dtuple_fold(&tuple2, &cursor) == crc42, "dtuple_fold(NULL,'42')"); + table.flags= DICT_TF_COMPACT; + ok(dtuple_fold(&tuple2, &cursor) == crc42, "dtuple_fold(NULL,'42')"); + fields[0].type.mtype= DATA_CHAR; + ok(dtuple_fold(&tuple2, &cursor) == crc42, "dtuple_fold(NULL,'42')"); + table.flags= 0; + ok(dtuple_fold(&tuple2, &cursor) == crc3z42, "dtuple_fold('\\0\\0\\0','42')"); + fields[0].type.mtype= DATA_VARCHAR; + + cursor.n_bytes_fields= 1; + ok(dtuple_fold(&tuple2, &cursor) == 0, "dtuple_fold(NULL)"); + table.flags= DICT_TF_COMPACT; + ok(dtuple_fold(&tuple2, &cursor) == 0, "dtuple_fold(NULL)"); + fields[0].type.mtype= DATA_CHAR; + ok(dtuple_fold(&tuple2, &cursor) == 0, "dtuple_fold(NULL)"); + table.flags= 0; + ok(dtuple_fold(&tuple2, &cursor) == crc3z, "dtuple_fold('\\0\\0\\0')"); + fields[0].type.mtype= DATA_VARCHAR; + + cursor.n_bytes_fields= 2 << 16; + ok(dtuple_fold(&tuple2, &cursor) == 0, "dtuple_fold(NULL)"); + table.flags= DICT_TF_COMPACT; + ok(dtuple_fold(&tuple2, &cursor) == 0, "dtuple_fold(NULL)"); + fields[0].type.mtype= DATA_CHAR; + ok(dtuple_fold(&tuple2, &cursor) == 0, "dtuple_fold(NULL)"); + table.flags= 0; + ok(dtuple_fold(&tuple2, &cursor) == crc2z, "dtuple_fold('\\0\\0')"); + fields[0].type.mtype= DATA_VARCHAR; + } + + byte *page= static_cast(aligned_malloc(16384, 16384)); + memset_aligned<16384>(page, 0, 16384); + byte *rec= &page[256]; + page[PAGE_HEADER + PAGE_HEAP_TOP]= 1; + page[PAGE_HEADER + PAGE_HEAP_TOP + 1]= 4 + 2; + const byte r1_varchar[]= {2,0x80,0,0,0,2<<1|1,0,0, '4','2'}; + const byte r2_varchar[]= {0,2,0x80,0,0,0,0,2<<1,0,0, '4','2'}; + const byte r1_var3[]= {2,0x80,0x80,0,0,0,3<<1|1,0,0, '4','2'}; + const byte r2_var3[]= {0,2,0x80,0,0x80,0,0,0,0,3<<1,0,0, '4','2'}; + const byte r1_char[]={2+3,0x83,0,0,0,2<<1|1,0,0, 0,0,0,'4','2'}; + const byte r2_char[]= {0,2+3,0x80,3,0,0,0,2<<1,0,0, 0,0,0,'4','2'}; + const byte c[]= { 0,1,0,0,0,0,0, '4','2'}; + const byte c3[]= { 0,3,0,0,0,0,0, '4','2'}; + + memcpy(rec - sizeof r1_varchar + 2, r1_varchar, sizeof r1_varchar); + ok(rec_fold(rec, index, 2, false) == crc42, "rec_fold(NULL, '42')"); + ok(rec_fold(rec, index, 1, false) == 0, "rec_fold(NULL)"); + ok(rec_fold(rec, index, 2 << 16, false) == 0, "rec_fold(NULL)"); + memcpy(rec - sizeof r2_varchar + 2, r2_varchar, sizeof r2_varchar); + ok(rec_fold(rec, index, 2, false) == crc42, "rec_fold(NULL, '42')"); + ok(rec_fold(rec, index, 1, false) == 0, "rec_fold(NULL)"); + ok(rec_fold(rec, index, 2 << 16, false) == 0, "rec_fold(NULL)"); + + memcpy(rec - sizeof r1_var3 + 2, r1_var3, sizeof r1_var3); + ok(rec_fold(rec, index, 3, false) == crc42, "rec_fold(NULL, NULL, '42')"); + ok(rec_fold(rec, index, 2, false) == 0, "rec_fold(NULL,NULL)"); + ok(rec_fold(rec, index, 1 | 2 << 16, false) == 0, "rec_fold(NULL,NULL)"); + memcpy(rec - sizeof r2_var3 + 2, r2_var3, sizeof r2_var3); + ok(rec_fold(rec, index, 3, false) == crc42, "rec_fold(NULL, NULL, '42')"); + ok(rec_fold(rec, index, 2, false) == 0, "rec_fold(NULL,NULL)"); + ok(rec_fold(rec, index, 1 | 2 << 16, false) == 0, "rec_fold(NULL,NULL)"); + + fields[0].type.mtype= DATA_CHAR; + memcpy(rec - sizeof r1_char + 3 + 2, r1_char, sizeof r1_char); + ok(rec_fold(rec, index, 2, false) == crc3z42, "rec_fold('\\0\\0\\0', '42')"); + ok(rec_fold(rec, index, 1, false) == crc3z, "rec_fold('\\0\\0\\0')"); + ok(rec_fold(rec, index, 2 << 16, false) == crc2z, "rec_fold('\\0\\0')"); + memcpy(rec - sizeof r2_char + 3 + 2, r2_char, sizeof r2_char); + ok(rec_fold(rec, index, 2, false) == crc3z42, "rec_fold('\\0\\0\\0', '42')"); + ok(rec_fold(rec, index, 1, false) == crc3z, "rec_fold('\\0\\0\\0')"); + ok(rec_fold(rec, index, 2 << 16, false) == crc2z, "rec_fold('\\0\\0')"); + + page[PAGE_HEADER + PAGE_N_HEAP]= 0x80; + table.flags= DICT_TF_COMPACT; + memcpy(rec - sizeof c + 2, c, sizeof c); + ok(rec_fold(rec, index, 2, true) == crc42, "rec_fold(NULL, '42')"); + ok(rec_fold(rec, index, 1, true) == 0, "rec_fold(NULL)"); + ok(rec_fold(rec, index, 2 << 16, true) == 0, "rec_fold(NULL)"); + fields[0].type.mtype= DATA_VARCHAR; + ok(rec_fold(rec, index, 2, true) == crc42, "rec_fold(NULL, '42')"); + ok(rec_fold(rec, index, 1, true) == 0, "rec_fold(NULL)"); + ok(rec_fold(rec, index, 2 << 16, true) == 0, "rec_fold(NULL)"); + + memcpy(rec - sizeof c3 + 2, c3, sizeof c3); + fields[0].type.mtype= DATA_CHAR; + + ifields[1].col= &cols[1]; + ifields[1].fixed_len= 0; + + ok(rec_fold(rec, index, 3, true) == crc42, "rec_fold(NULL, NULL, '42')"); + ok(rec_fold(rec, index, 2, true) == 0, "rec_fold(NULL, NULL)"); + ok(rec_fold(rec, index, 1 | 2 << 16, true) == 0, "rec_fold(NULL, NULL)"); + fields[0].type.mtype= DATA_VARCHAR; + ok(rec_fold(rec, index, 3, true) == crc42, "rec_fold(NULL, NULL, '42')"); + ok(rec_fold(rec, index, 2, true) == 0, "rec_fold(NULL, NULL)"); + ok(rec_fold(rec, index, 1 | 2 << 16, true) == 0, "rec_fold(NULL, NULL)"); + aligned_free(page); + + my_end(MY_CHECK_ERROR); + return exit_status(); +} From 0e06773e9dc6f8680cf7414ccefa206229d9abd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:02:23 +0200 Subject: [PATCH 03/10] MDEV-35049: Privatize ut_fold_ulint_pair() Now that ut_fold_ulint_pair() and ut_fold_binary() are no longer needed for anything else than compatibility with old InnoDB data files that may use innodb_checksum_algorithm=innodb, let us move the code to a single compilation unit. Reviewed by: Vladislav Lesin --- extra/innochecksum.cc | 6 -- storage/innobase/CMakeLists.txt | 1 - storage/innobase/buf/buf0checksum.cc | 71 +++++++++++++------ storage/innobase/include/buf0checksum.h | 18 +++-- storage/innobase/include/ut0rnd.h | 31 +------- storage/innobase/include/ut0rnd.inl | 94 ------------------------- storage/innobase/log/log0recv.cc | 1 + 7 files changed, 61 insertions(+), 161 deletions(-) delete mode 100644 storage/innobase/include/ut0rnd.inl diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 664f1baf64f5c..e5e85cc706510 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -53,12 +53,6 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ #include -#ifdef UNIV_NONINL -# include "fsp0fsp.inl" -# include "mach0data.inl" -# include "ut0rnd.inl" -#endif - #ifndef PRIuMAX #define PRIuMAX "llu" #endif diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index a067bce2678ad..44348dedd2f1e 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -368,7 +368,6 @@ SET(INNOBASE_SOURCES include/ut0pool.h include/ut0rbt.h include/ut0rnd.h - include/ut0rnd.inl include/ut0sort.h include/ut0stage.h include/ut0ut.h diff --git a/storage/innobase/buf/buf0checksum.cc b/storage/innobase/buf/buf0checksum.cc index c9818fa600f20..68ef0680dd0c6 100644 --- a/storage/innobase/buf/buf0checksum.cc +++ b/storage/innobase/buf/buf0checksum.cc @@ -27,10 +27,9 @@ Created Aug 11, 2011 Vasil Dimov #include "buf0checksum.h" #include "fil0fil.h" #include "ut0crc32.h" -#include "ut0rnd.h" #ifndef UNIV_INNOCHECKSUM -#include "srv0srv.h" +# include "srv0srv.h" #endif /* !UNIV_INNOCHECKSUM */ /** Calculate the CRC32 checksum of a page. The value is stored to the page @@ -39,7 +38,7 @@ the file. Note that we must be careful to calculate the same value on all architectures. @param[in] page buffer page (srv_page_size bytes) @return CRC-32C */ -uint32_t buf_calc_page_crc32(const byte* page) +uint32_t buf_calc_page_crc32(const byte *page) noexcept { /* Note: innodb_checksum_algorithm=crc32 could and should have included the entire page in the checksum, and CRC-32 values @@ -55,13 +54,60 @@ uint32_t buf_calc_page_crc32(const byte* page) } #ifndef UNIV_INNOCHECKSUM +static inline ulint ut_fold_ulint_pair(ulint n1, ulint n2) +{ + return ((((n1 ^ n2 ^ 1653893711) << 8) + n1) ^ 1463735687) + n2; +} + +/** Fold a binary string, similar to innodb_checksum_algorithm=innodb. +@return folded value */ +ulint ut_fold_binary(const byte *str, size_t len) noexcept +{ + ulint fold= 0; + for (const byte *const str_end= str + (len & 0xFFFFFFF8); str < str_end; + str+= 8) + { + fold= ut_fold_ulint_pair(fold, str[0]); + fold= ut_fold_ulint_pair(fold, str[1]); + fold= ut_fold_ulint_pair(fold, str[2]); + fold= ut_fold_ulint_pair(fold, str[3]); + fold= ut_fold_ulint_pair(fold, str[4]); + fold= ut_fold_ulint_pair(fold, str[5]); + fold= ut_fold_ulint_pair(fold, str[6]); + fold= ut_fold_ulint_pair(fold, str[7]); + } + + switch (len & 0x7) { + case 7: + fold= ut_fold_ulint_pair(fold, *str++); + /* fall through */ + case 6: + fold= ut_fold_ulint_pair(fold, *str++); + /* fall through */ + case 5: + fold= ut_fold_ulint_pair(fold, *str++); + /* fall through */ + case 4: + fold= ut_fold_ulint_pair(fold, *str++); + /* fall through */ + case 3: + fold= ut_fold_ulint_pair(fold, *str++); + /* fall through */ + case 2: + fold= ut_fold_ulint_pair(fold, *str++); + /* fall through */ + case 1: + fold= ut_fold_ulint_pair(fold, *str++); + } + return fold; +} + /** Calculate a checksum which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on 32-bit and 64-bit architectures. @param[in] page file page (srv_page_size bytes) @return checksum */ -uint32_t -buf_calc_page_new_checksum(const byte* page) +uint32_t buf_calc_page_new_checksum(const byte *page) noexcept { ulint checksum; @@ -81,19 +127,4 @@ buf_calc_page_new_checksum(const byte* page) - FIL_PAGE_END_LSN_OLD_CHKSUM); return(static_cast(checksum)); } - -/** In MySQL before 4.0.14 or 4.1.1 there was an InnoDB bug that -the checksum only looked at the first few bytes of the page. -This calculates that old checksum. -NOTE: we must first store the new formula checksum to -FIL_PAGE_SPACE_OR_CHKSUM before calculating and storing this old checksum -because this takes that field as an input! -@param[in] page file page (srv_page_size bytes) -@return checksum */ -uint32_t -buf_calc_page_old_checksum(const byte* page) -{ - return(static_cast - (ut_fold_binary(page, FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION))); -} #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/buf0checksum.h b/storage/innobase/include/buf0checksum.h index d9f0317781299..a6455304a1240 100644 --- a/storage/innobase/include/buf0checksum.h +++ b/storage/innobase/include/buf0checksum.h @@ -33,25 +33,23 @@ the file. Note that we must be careful to calculate the same value on all architectures. @param[in] page buffer page (srv_page_size bytes) @return CRC-32C */ -uint32_t buf_calc_page_crc32(const byte* page); +uint32_t buf_calc_page_crc32(const byte *page) noexcept; #ifndef UNIV_INNOCHECKSUM +/** Fold a binary string, similar to innodb_checksum_algorithm=innodb. +@return folded value */ +ulint ut_fold_binary(const byte *str, size_t len) noexcept; + /** Calculate a checksum which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on 32-bit and 64-bit architectures. @param[in] page file page (srv_page_size bytes) @return checksum */ uint32_t -buf_calc_page_new_checksum(const byte* page); +buf_calc_page_new_checksum(const byte *page) noexcept; /** In MySQL before 4.0.14 or 4.1.1 there was an InnoDB bug that the checksum only looked at the first few bytes of the page. -This calculates that old checksum. -NOTE: we must first store the new formula checksum to -FIL_PAGE_SPACE_OR_CHKSUM before calculating and storing this old checksum -because this takes that field as an input! -@param[in] page file page (srv_page_size bytes) -@return checksum */ -uint32_t -buf_calc_page_old_checksum(const byte* page); +This calculates that old checksum. */ +# define buf_calc_page_old_checksum(page) uint32_t(ut_fold_binary(page, 26)) #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/ut0rnd.h b/storage/innobase/include/ut0rnd.h index 5cba6ee534f65..90c69a0464d99 100644 --- a/storage/innobase/include/ut0rnd.h +++ b/storage/innobase/include/ut0rnd.h @@ -24,13 +24,11 @@ Random numbers and hashing Created 1/20/1994 Heikki Tuuri ***********************************************************************/ -#ifndef ut0rnd_h -#define ut0rnd_h +#pragma once #include "ut0byte.h" #include -#ifndef UNIV_INNOCHECKSUM /** Seed value of ut_rnd_gen() */ extern std::atomic ut_rnd_current; @@ -87,30 +85,3 @@ ut_find_prime( /*==========*/ ulint n) /*!< in: positive number > 100 */ MY_ATTRIBUTE((const)); - -#endif /* !UNIV_INNOCHECKSUM */ - -/*************************************************************//** -Folds a pair of ulints. -@return folded value */ -UNIV_INLINE -ulint -ut_fold_ulint_pair( -/*===============*/ - ulint n1, /*!< in: ulint */ - ulint n2) /*!< in: ulint */ - MY_ATTRIBUTE((const)); -/*************************************************************//** -Folds a binary string. -@return folded value */ -UNIV_INLINE -ulint -ut_fold_binary( -/*===========*/ - const byte* str, /*!< in: string of bytes */ - ulint len) /*!< in: length */ - MY_ATTRIBUTE((pure)); - -#include "ut0rnd.inl" - -#endif diff --git a/storage/innobase/include/ut0rnd.inl b/storage/innobase/include/ut0rnd.inl deleted file mode 100644 index c9f7373dd1be4..0000000000000 --- a/storage/innobase/include/ut0rnd.inl +++ /dev/null @@ -1,94 +0,0 @@ -/***************************************************************************** - -Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************************//** -@file include/ut0rnd.ic -Random numbers and hashing - -Created 5/30/1994 Heikki Tuuri -*******************************************************************/ - -#define UT_HASH_RANDOM_MASK 1463735687 -#define UT_HASH_RANDOM_MASK2 1653893711 - -/*************************************************************//** -Folds a pair of ulints. -@return folded value */ -UNIV_INLINE -ulint -ut_fold_ulint_pair( -/*===============*/ - ulint n1, /*!< in: ulint */ - ulint n2) /*!< in: ulint */ -{ - return(((((n1 ^ n2 ^ UT_HASH_RANDOM_MASK2) << 8) + n1) - ^ UT_HASH_RANDOM_MASK) + n2); -} - -/*************************************************************//** -Folds a binary string. -@return folded value */ -UNIV_INLINE -ulint -ut_fold_binary( -/*===========*/ - const byte* str, /*!< in: string of bytes */ - ulint len) /*!< in: length */ -{ - ulint fold = 0; - const byte* str_end = str + (len & 0xFFFFFFF8); - - ut_ad(str || !len); - - while (str < str_end) { - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - } - - switch (len & 0x7) { - case 7: - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - /* fall through */ - case 6: - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - /* fall through */ - case 5: - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - /* fall through */ - case 4: - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - /* fall through */ - case 3: - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - /* fall through */ - case 2: - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - /* fall through */ - case 1: - fold = ut_fold_ulint_pair(fold, (ulint)(*str++)); - } - - return(fold); -} diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 96c964e6de71c..a571e0f7d6a86 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -41,6 +41,7 @@ Created 9/20/1997 Heikki Tuuri #include "buf0buf.h" #include "buf0dblwr.h" #include "buf0flu.h" +#include "buf0checksum.h" #include "mtr0mtr.h" #include "mtr0log.h" #include "page0page.h" From 94e9a704f4ec37dc6d471bf3f3532273cec7d74a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:07:58 +0200 Subject: [PATCH 04/10] MDEV-35049: Fix bogus BTR_CUR_HASH_FAIL on PAGE_CUR_LE btr_cur_t::search_leaf(): Do not attempt to use the adaptive hash index for PAGE_CUR_G or PAGE_CUR_L, because those modes expect an inequal result, and the adaptive hash index can only deliver equal results. btr_cur_t::check_mismatch(): Only handle PAGE_CUR_LE and PAGE_CUR_GE. For PAGE_CUR_LE (bool ge=false), qualify a full match for the last record of a page that is not at the end of the index. Previously, an adaptive hash index lookup would fail when the record is at the end of an index page but not at the end of the index. This would lead to unnecessary rebuild of the adaptive hash index in read-only workloads. Reviewed by: Vladislav Lesin --- storage/innobase/btr/btr0cur.cc | 4 +- storage/innobase/btr/btr0sea.cc | 6 +- storage/innobase/include/btr0cur.h | 5 +- storage/innobase/include/btr0sea.h | 4 +- storage/innobase/page/page0cur.cc | 156 +++++++++++++--------- storage/innobase/unittest/innodb_ahi-t.cc | 2 +- 6 files changed, 102 insertions(+), 75 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index a023ccdfb6aa2..86978450a02af 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1172,10 +1172,12 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, # endif if (latch_mode > BTR_MODIFY_LEAF) /* The adaptive hash index cannot be useful for these searches. */; + else if (mode != PAGE_CUR_LE && mode != PAGE_CUR_GE) + ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_G); /* We do a dirty read of btr_search.enabled below, and btr_search_guess_on_hash() will have to check it again. */ else if (!btr_search.enabled); - else if (btr_search_guess_on_hash(index(), tuple, mode, + else if (btr_search_guess_on_hash(index(), tuple, mode != PAGE_CUR_LE, latch_mode, this, mtr)) { /* Search using the hash index succeeded */ diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 7ae1fa399fc5b..45418d0fdbcc5 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1033,7 +1033,7 @@ and the function returns TRUE, then cursor->up_match and cursor->low_match both have sensible values. @param[in,out] index index @param[in] tuple logical record -@param[in] mode PAGE_CUR_L, .... +@param[in] ge false=PAGE_CUR_LE, true=PAGE_CUR_GE @param[in] latch_mode BTR_SEARCH_LEAF, ... @param[out] cursor tree cursor @param[in] mtr mini-transaction @@ -1043,7 +1043,7 @@ bool btr_search_guess_on_hash( dict_index_t* index, const dtuple_t* tuple, - page_cur_mode_t mode, + bool ge, btr_latch_mode latch_mode, btr_cur_t* cursor, mtr_t* mtr) noexcept @@ -1180,7 +1180,7 @@ btr_search_guess_on_hash( /* Check the validity of the guess within the page */ if (index_id != btr_page_get_index_id(block->page.frame) || - cursor->check_mismatch(*tuple, mode, comp)) + cursor->check_mismatch(*tuple, ge, comp)) goto mismatch; uint8_t n_hash_potential= index->search_info.n_hash_potential; diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 4663cb5c52648..524cacddc83fa 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -771,13 +771,12 @@ struct btr_cur_t { /** Check if a guessed position for a tree cursor is correct. @param tuple search key - @param mode PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, PAGE_CUR_GE + @param ge false=PAGE_CUR_LE, true=PAGE_CUR_GE @param comp nonzero if ROW_FORMAT=REDUNDANT is not being used @retval true on mismatch or corruption @retval false on a match; if mode=PAGE_CUR_LE, then up_match,low_match will be set correctly. */ - bool check_mismatch(const dtuple_t& tuple, page_cur_mode_t mode, ulint comp) - noexcept; + bool check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) noexcept; #endif }; diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index aa8a8e50fe44e..e975a53ae8cc9 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -43,7 +43,7 @@ and the function returns TRUE, then cursor->up_match and cursor->low_match both have sensible values. @param[in,out] index index @param[in] tuple logical record -@param[in] mode PAGE_CUR_L, .... +@param[in] ge false=PAGE_CUR_LE, true=PAGE_CUR_GE @param[in] latch_mode BTR_SEARCH_LEAF, ... @param[out] cursor tree cursor @param[in] mtr mini-transaction @@ -52,7 +52,7 @@ bool btr_search_guess_on_hash( dict_index_t* index, const dtuple_t* tuple, - page_cur_mode_t mode, + bool ge, btr_latch_mode latch_mode, btr_cur_t* cursor, mtr_t* mtr) noexcept; diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 592521a5da1b0..b5585fa02f279 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -32,6 +32,9 @@ Created 10/4/1994 Heikki Tuuri #include "log0recv.h" #include "rem0cmp.h" #include "gis0rtree.h" +#ifdef UNIV_DEBUG +# include "trx0roll.h" +#endif #ifdef BTR_CUR_HASH_ADAPT /** Get the pad character code point for a type. @@ -518,24 +521,63 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, return false; } +/** Compare a data tuple to a physical record. +@param page B-tree index page +@param rec B-tree index record +@param index index B-tree +@param tuple search key +@param match matched fields << 16 | bytes +@param comp nonzero if ROW_FORMAT=REDUNDANT is not being used +@return the comparison result of dtuple and rec +@retval 0 if dtuple is equal to rec +@retval negative if dtuple is less than rec +@retval positive if dtuple is greater than rec */ static int cmp_dtuple_rec_leaf(const dtuple_t &dtuple, const rec_t *rec, const dict_index_t &index, uint16_t *matched_fields, ulint comp) noexcept { ut_ad(dtuple_check_typed(&dtuple)); - ut_ad(page_rec_is_leaf(rec)); ut_ad(!!comp == index.table->not_redundant()); ulint cur_field= *matched_fields; ut_ad(dtuple.n_fields_cmp > 0); ut_ad(dtuple.n_fields_cmp <= index.n_core_fields || index.is_ibuf()); ut_ad(cur_field <= dtuple.n_fields_cmp); + ut_ad(page_rec_is_leaf(rec)); + ut_ad(!(rec_get_info_bits(rec, comp) & REC_INFO_MIN_REC_FLAG) || + index.is_instant() || + (index.is_primary() && trx_roll_crash_recv_trx && + !trx_rollback_is_active)); + ut_ad(!(dtuple.info_bits & REC_INFO_MIN_REC_FLAG) || + index.is_instant() || + (index.is_primary() && trx_roll_crash_recv_trx && + !trx_rollback_is_active)); int ret= 0; + if (dtuple.info_bits & REC_INFO_MIN_REC_FLAG) + { + *matched_fields= 0; + return -!(rec_get_info_bits(rec, comp) & REC_INFO_MIN_REC_FLAG); + } + else if (rec_get_info_bits(rec, comp) & REC_INFO_MIN_REC_FLAG) + { + *matched_fields= 0; + return 1; + } + if (UNIV_LIKELY(comp != 0)) { - const unsigned n_core_null_bytes= index.n_core_null_bytes; const byte *nulls= rec - REC_N_NEW_EXTRA_BYTES; - const byte *lens= --nulls - n_core_null_bytes; + const byte *lens; + if (rec_get_status(rec) == REC_STATUS_INSTANT) + { + ulint n_fields= index.n_core_fields + rec_get_n_add_field(nulls) + 1; + ut_ad(n_fields <= index.n_fields); + const ulint n_nullable= index.get_n_nullable(n_fields); + ut_ad(n_nullable <= index.n_nullable); + lens= --nulls - UT_BITS_IN_BYTES(n_nullable); + } + else + lens= --nulls - index.n_core_null_bytes; byte null_mask= 1; size_t i= 0; @@ -618,70 +660,39 @@ static int cmp_dtuple_rec_leaf(const dtuple_t &dtuple, const rec_t *rec, return ret; } -bool btr_cur_t::check_mismatch(const dtuple_t& tuple, page_cur_mode_t mode, - ulint comp) noexcept +bool btr_cur_t::check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) + noexcept { + ut_ad(page_is_leaf(page_cur.block->page.frame)); + ut_ad(page_rec_is_user_rec(page_cur.rec)); + const rec_t *rec= page_cur.rec; uint16_t match= 0; int cmp= cmp_dtuple_rec_leaf(tuple, rec, *index(), &match, comp); + const auto uniq= dict_index_get_n_unique_in_tree(index()); + ut_ad(match <= uniq); + ut_ad(match <= tuple.n_fields_cmp); + ut_ad(match < uniq || !cmp); - switch (mode) { - const rec_t *other; - const page_t *page; - case PAGE_CUR_GE: - if (cmp > 0) - return true; - up_match= match; - if (match >= dict_index_get_n_unique_in_tree(index())) - return false; - goto check_ge; - case PAGE_CUR_G: - if (cmp >= 0) - return true; - check_ge: - match= 0; - if (!(other= page_rec_get_prev_const(rec))) - return true; - page= page_cur.block->page.frame; - if (uintptr_t(other - page) == - (comp ? PAGE_NEW_INFIMUM : PAGE_OLD_INFIMUM)) - return page_has_prev(page); - if (UNIV_LIKELY(comp != 0)) - switch (rec_get_status(other)) { - case REC_STATUS_INSTANT: - case REC_STATUS_ORDINARY: - break; - default: - return true; - } - cmp= cmp_dtuple_rec_leaf(tuple, other, *index(), &match, comp); - return (mode == PAGE_CUR_GE) ? cmp <= 0 : cmp < 0; - case PAGE_CUR_LE: + const page_t *const page= page_cur.block->page.frame; + + if (UNIV_LIKELY(!ge)) + { if (cmp < 0) return true; low_match= match; - goto check_le; - case PAGE_CUR_L: - if (cmp <= 0) - return true; - check_le: - match= 0; - ut_ad(!page_rec_is_supremum(rec)); - page= page_cur.block->page.frame; + up_match= 0; if (UNIV_LIKELY(comp != 0)) { - other= page_rec_next_get(page, rec); - if (!other) + rec= page_rec_next_get(page, rec); + if (!rec) return true; - if (other - page == PAGE_NEW_SUPREMUM) - { + if (uintptr_t(rec - page) == PAGE_NEW_SUPREMUM) le_supremum: - if (page_has_next(page)) - return true; - up_match= 0; - return false; - } - switch (rec_get_status(other)) { + /* If we matched the full key at the end of a page (but not the index), + the adaptive hash index was successful. */ + return page_has_next(page) && match < uniq; + switch (rec_get_status(rec)) { case REC_STATUS_INSTANT: case REC_STATUS_ORDINARY: break; @@ -691,20 +702,35 @@ bool btr_cur_t::check_mismatch(const dtuple_t& tuple, page_cur_mode_t mode, } else { - other= page_rec_next_get(page, rec); - if (!other) + rec= page_rec_next_get(page, rec); + if (!rec) return true; - if (other - page == PAGE_OLD_SUPREMUM) + if (uintptr_t(rec - page) == PAGE_OLD_SUPREMUM) goto le_supremum; } - cmp= cmp_dtuple_rec_leaf(tuple, other, *index(), &match, comp); - if (mode != PAGE_CUR_LE) - return cmp > 0; + return cmp_dtuple_rec_leaf(tuple, rec, *index(), &up_match, comp) >= 0; + } + else + { + if (cmp > 0) + return true; up_match= match; - return cmp >= 0; - default: - ut_ad("invalid mode" == 0); - return true; + if (match >= uniq) + return false; + match= 0; + if (!(rec= page_rec_get_prev_const(rec))) + return true; + if (uintptr_t(rec - page) == (comp ? PAGE_NEW_INFIMUM : PAGE_OLD_INFIMUM)) + return page_has_prev(page); + if (UNIV_LIKELY(comp != 0)) + switch (rec_get_status(rec)) { + case REC_STATUS_INSTANT: + case REC_STATUS_ORDINARY: + break; + default: + return true; + } + return cmp_dtuple_rec_leaf(tuple, rec, *index(), &match, comp) <= 0; } } diff --git a/storage/innobase/unittest/innodb_ahi-t.cc b/storage/innobase/unittest/innodb_ahi-t.cc index 487182683afbe..5e6779e5ec235 100644 --- a/storage/innobase/unittest/innodb_ahi-t.cc +++ b/storage/innobase/unittest/innodb_ahi-t.cc @@ -22,7 +22,7 @@ void dict_mem_table_free(dict_table_t*) {} void dict_mem_index_free(dict_index_t*) {} buf_block_t *buf_LRU_get_free_block(bool) { return nullptr; } ibool dtuple_check_typed(const dtuple_t*) { return true; } -bool btr_cur_t::check_mismatch(const dtuple_t&,page_cur_mode_t,ulint) noexcept +bool btr_cur_t::check_mismatch(const dtuple_t&,bool,ulint) noexcept { return false; } buf_block_t *buf_page_get_gen(const page_id_t, ulint, rw_lock_type_t, buf_block_t*,ulint,mtr_t*,dberr_t*,bool) noexcept From cbe54823d2202593f772b4830eba17e57cd39ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:08:10 +0200 Subject: [PATCH 05/10] MDEV-35049: Fix bogus BTR_CUR_HASH_FAIL on contention btr_search_guess_on_hash(): Only set BTR_CUR_HASH_FAIL on actual mismatch. If the page latch cannot be acquired, the hash search might very well have succeeded. Do not count that as a failure, that is, do not unnecessarily invoke btr_search_update_hash_ref() after a normal search. Set cursor->flag=BTR_CUR_HASH_ABORT if the current parameters of the adaptive hash index are not suitable for the search and a call to btr_cur_t::search_info_update() might help. btr_cur_t::search_leaf(): Do not invoke search_info_update() if btr_search_guess_on_hash() failed due to contention. btr_cur_t::pessimistic_search_leaf(): Do not invoke search_info_update() on the change buffer tree. Preivously, this condition was being checked inside search_info_update(). --- storage/innobase/btr/btr0cur.cc | 22 ++++++++----------- storage/innobase/btr/btr0sea.cc | 35 ++++++++++++++++++++---------- storage/innobase/include/btr0cur.h | 1 + storage/innobase/row/row0row.cc | 1 + 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 86978450a02af..73e2b6a542b70 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1535,18 +1535,14 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, goto need_opposite_intention; #ifdef BTR_CUR_HASH_ADAPT - /* We do a dirty read of btr_search.enabled here. We will recheck in - btr_search_build_page_hash_index() before building a page hash - index, while holding search latch. */ - if (!btr_search.enabled); - else if (tuple->info_bits & REC_INFO_MIN_REC_FLAG) - /* This may be a search tuple for btr_pcur_t::restore_position(). */ - ut_ad(tuple->is_metadata() || - (tuple->is_metadata(tuple->info_bits ^ REC_STATUS_INSTANT))); - else if (index()->table->is_temporary()); - else if (!rec_is_metadata(page_cur.rec, *index()) && - index()->search_info.hash_analysis_useful()) - search_info_update(); + if (flag != BTR_CUR_BINARY) + { + ut_ad(!(tuple->info_bits & REC_INFO_MIN_REC_FLAG)); + ut_ad(!index()->table->is_temporary()); + if (!rec_is_metadata(page_cur.rec, *index()) && + index()->search_info.hash_analysis_useful()) + search_info_update(); + } #endif /* BTR_CUR_HASH_ADAPT */ goto func_exit; @@ -1790,7 +1786,7 @@ dberr_t btr_cur_t::pessimistic_search_leaf(const dtuple_t *tuple, /* This may be a search tuple for btr_pcur_t::restore_position(). */ ut_ad(tuple->is_metadata() || (tuple->is_metadata(tuple->info_bits ^ REC_STATUS_INSTANT))); - else if (index()->table->is_temporary()); + else if (index()->is_ibuf() || index()->table->is_temporary()); else if (!rec_is_metadata(page_cur.rec, *index()) && index()->search_info.hash_analysis_useful()) search_info_update(); diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 45418d0fdbcc5..9298ca01d95df 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -502,14 +502,12 @@ static void btr_search_update_hash_ref(const btr_cur_t &cursor, @retval 0 if the adaptive hash index should not be rebuilt */ static uint32_t btr_search_info_update_hash(const btr_cur_t &cursor) noexcept { - ut_ad(cursor.flag != BTR_CUR_HASH); + ut_ad(cursor.flag == BTR_CUR_HASH_FAIL || + cursor.flag == BTR_CUR_HASH_ABORT || + cursor.flag == BTR_CUR_BINARY); dict_index_t *const index= cursor.index(); - - if (index->is_ibuf()) - /* Too many deletes are performed on the change buffer */ - return 0; - + ut_ad(!index->is_ibuf()); const uint16_t n_uniq{dict_index_get_n_unique_in_tree(index)}; dict_index_t::ahi &info= index->search_info; @@ -1051,11 +1049,20 @@ btr_search_guess_on_hash( ut_ad(mtr->is_active()); ut_ad(index->is_btree() || index->is_ibuf()); ut_ad(latch_mode == BTR_SEARCH_LEAF || latch_mode == BTR_MODIFY_LEAF); + ut_ad(cursor->flag == BTR_CUR_BINARY); + + if ((tuple->info_bits & REC_INFO_MIN_REC_FLAG)) + return false; - if ((tuple->info_bits & REC_INFO_MIN_REC_FLAG) || - !index->search_info.last_hash_succ || + if (!index->search_info.last_hash_succ || !index->search_info.n_hash_potential) + { + ahi_unusable: + if (!index->is_ibuf() && !index->table->is_temporary() && + btr_search.enabled) + cursor->flag= BTR_CUR_HASH_ABORT; return false; + } ut_ad(index->is_btree()); ut_ad(!index->table->is_temporary()); @@ -1067,7 +1074,7 @@ btr_search_guess_on_hash( ~buf_block_t::LEFT_SIDE; if (dtuple_get_n_fields(tuple) < btr_search_get_n_fields(cursor)) - return false; + goto ahi_unusable; const index_id_t index_id= index->id; @@ -1076,7 +1083,6 @@ btr_search_guess_on_hash( #endif const uint32_t fold= dtuple_fold(tuple, cursor); cursor->fold= fold; - cursor->flag= BTR_CUR_HASH; btr_sea::partition &part= btr_search.get_part(*index); part.latch.rd_lock(SRW_LOCK_CALL); @@ -1086,8 +1092,6 @@ btr_search_guess_on_hash( ahi_release_and_fail: part.latch.rd_unlock(); fail: - cursor->flag= BTR_CUR_HASH_FAIL; - #ifdef UNIV_SEARCH_PERF_STAT ++index->search_info.n_hash_fail; if (index->search_info.n_hash_succ > 0) @@ -1102,7 +1106,10 @@ btr_search_guess_on_hash( { return node->fold == fold; }); if (!node) + { + cursor->flag= BTR_CUR_HASH_FAIL; goto ahi_release_and_fail; + } const rec_t *rec= node->rec; buf_block_t *block= buf_pool.block_from_ahi(rec); @@ -1133,6 +1140,7 @@ btr_search_guess_on_hash( block->page.lock.s_unlock(); else block->page.lock.x_unlock(); + cursor->flag= BTR_CUR_HASH_FAIL; goto ahi_release_and_fail; } @@ -1143,6 +1151,7 @@ btr_search_guess_on_hash( if (index != block_index && index_id == block_index->id) { ut_a(block_index->freed()); + cursor->flag= BTR_CUR_HASH_FAIL; goto block_and_ahi_release_and_fail; } @@ -1175,6 +1184,7 @@ btr_search_guess_on_hash( default: mismatch: mtr->release_last_page(); + cursor->flag= BTR_CUR_HASH_FAIL; goto fail; } @@ -1189,6 +1199,7 @@ btr_search_guess_on_hash( index->search_info.n_hash_potential= n_hash_potential; index->search_info.last_hash_succ= true; + cursor->flag= BTR_CUR_HASH; #ifdef UNIV_SEARCH_PERF_STAT btr_search_n_succ++; diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 524cacddc83fa..1a68ff2b6c9f1 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -659,6 +659,7 @@ enum btr_cur_method { #ifdef BTR_CUR_HASH_ADAPT BTR_CUR_HASH, /*!< successful shortcut using the hash index */ + BTR_CUR_HASH_ABORT, /*!< the hash index could not be used */ BTR_CUR_HASH_FAIL, /*!< failure using hash, success using binary search */ #endif diff --git a/storage/innobase/row/row0row.cc b/storage/innobase/row/row0row.cc index dee88bd8fdb50..4d3a0bfa11bfb 100644 --- a/storage/innobase/row/row0row.cc +++ b/storage/innobase/row/row0row.cc @@ -1302,6 +1302,7 @@ row_search_index_entry( #ifdef BTR_CUR_HASH_ADAPT case BTR_CUR_HASH: + case BTR_CUR_HASH_ABORT: case BTR_CUR_HASH_FAIL: #endif case BTR_CUR_BINARY: From 2b57676de63e5a535eb89f64b9dd3da0f4135785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:08:26 +0200 Subject: [PATCH 06/10] MDEV-35049: Fix bogus rebuild on BTR_CUR_HASH_FAIL btr_search_info_update_hash(): Do nothing if the record is positioned on the page supremum or infimum pseudo-record. The adaptive hash index can only include user records. This deficiency would cause the adaptive hash index parameters to change between hashing a prefix of 1 field or a prefix of 1 byte. Reviewed by: Vladislav Lesin --- storage/innobase/btr/btr0sea.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 9298ca01d95df..f252b38655984 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -508,18 +508,27 @@ static uint32_t btr_search_info_update_hash(const btr_cur_t &cursor) noexcept dict_index_t *const index= cursor.index(); ut_ad(!index->is_ibuf()); - const uint16_t n_uniq{dict_index_get_n_unique_in_tree(index)}; - dict_index_t::ahi &info= index->search_info; - - uint32_t left_bytes_fields{info.left_bytes_fields}; - uint8_t n_hash_potential= info.n_hash_potential; buf_block_t *const block= cursor.page_cur.block; ut_ad(block->page.lock.have_any()); ut_d(const uint32_t state= block->page.state()); ut_ad(state >= buf_page_t::UNFIXED); ut_ad(!block->page.is_read_fixed(state)); + + switch (uintptr_t(btr_cur_get_rec(&cursor) - block->page.frame)) { + case PAGE_OLD_INFIMUM: + case PAGE_OLD_SUPREMUM: + case PAGE_NEW_INFIMUM: + case PAGE_NEW_SUPREMUM: + /* The adaptive hash index only includes user records. */ + return 0; + } + const dict_index_t *const block_index= block->index; uint16_t n_hash_helps{block->n_hash_helps}; + const uint16_t n_uniq{dict_index_get_n_unique_in_tree(index)}; + dict_index_t::ahi &info= index->search_info; + uint32_t left_bytes_fields{info.left_bytes_fields}; + uint8_t n_hash_potential= info.n_hash_potential; uint32_t ret; if (!n_hash_potential) From f3e3219a314937bce8ad71d3d8c38ec2c9195190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:08:52 +0200 Subject: [PATCH 07/10] MDEV-35049: Improve btr_search_drop_page_hash_index() btr_search_drop_page_hash_index(): Replace the Boolean parameter with const dict_index_t *not_garbage. If buf_block_t::index points to that, there is no need to acquire btr_sea::partition::latch. The old parameter bool garbage_collect=false is equivalent to the parameter not_garbage=nullptr. The parameter garbage_collect=true will be replaced either with the actual index that is associated with the buffer page, or with a bogus pointer not_garbage=-1 to indicate that any lazily entries for a freed index need to be removed. buf_page_get_low(), buf_page_get_gen(), mtr_t::page_lock(), mtr_t::upgrade_buffer_fix(): Do not invoke btr_search_drop_page_hash_index(). Our caller will have to do it when appropriate. buf_page_create_low(): Keep invoking btr_search_drop_page_hash_index(). This is the normal way of lazily dropping the adaptive hash index after a DDL operation such as DROP INDEX operation. btr_block_get(), btr_root_block_get(), btr_root_adjust_on_import(), btr_read_autoinc_with_fallback(), btr_cur_instant_init_low(), btr_cur_t::search_leaf(), btr_cur_t::pessimistic_search_leaf(), btr_pcur_optimistic_latch_leaves(), dict_stats_analyze_index_below_cur(): Invoke btr_search_drop_page_hash_index(block, index) for pages that may be leaf pages. No adaptive hash index may have been created on anything else than a B-tree leaf page. btr_cur_search_to_nth_level(): Do not invoke btr_search_drop_page_hash_index(), because we are only accessing non-leaf pages and the adaptive hash index may only have been created on leaf pages. btr_page_alloc_for_ibuf() and many other callers of buf_page_get_gen() or similar functions do not invoke btr_search_drop_page_hash_index(), because the adaptive hash index is never created on such pages. If a page in the tablespace was freed as part of a DDL operation and reused for something else, then buf_page_create_low() will take care of dropping the adaptive hash index before the freed page will be modified. It is notable that while the flst_ functions may access pages that are related to allocating B-tree index pages (the BTR_SEG_TOP and BTR_SEG_LEAF linked from the index root page), those pages themselves can never be stored in the adaptive hash index. Therefore, it is not necessary to invoke btr_search_drop_page_hash_index() on them. Reviewed by: Vladislav Lesin --- storage/innobase/btr/btr0btr.cc | 51 ++++++++++++++++---------- storage/innobase/btr/btr0cur.cc | 17 +++++++-- storage/innobase/btr/btr0defragment.cc | 2 +- storage/innobase/btr/btr0pcur.cc | 4 +- storage/innobase/btr/btr0sea.cc | 39 ++++++++++---------- storage/innobase/buf/buf0buf.cc | 10 ++--- storage/innobase/buf/buf0lru.cc | 2 +- storage/innobase/dict/dict0stats.cc | 2 + storage/innobase/fil/fil0crypt.cc | 5 +++ storage/innobase/include/btr0sea.h | 8 ++-- storage/innobase/include/mtr0mtr.h | 7 ++-- storage/innobase/mtr/mtr0mtr.cc | 20 +++------- storage/innobase/page/page0zip.cc | 2 +- 13 files changed, 95 insertions(+), 74 deletions(-) diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index c6fba3422f77b..645735773f87d 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -242,6 +242,7 @@ buf_block_t *btr_block_get(const dict_index_t &index, uint32_t page, if (UNIV_LIKELY(block != nullptr)) { + btr_search_drop_page_hash_index(block, &index); if (!!page_is_comp(block->page.frame) != index.table->not_redundant() || btr_page_get_index_id(block->page.frame) != index.id || !fil_page_index_page_check(block->page.frame) || @@ -295,6 +296,7 @@ btr_root_block_get( if (UNIV_LIKELY(block != nullptr)) { + btr_search_drop_page_hash_index(block, index); if (!!page_is_comp(block->page.frame) != index->table->not_redundant() || btr_page_get_index_id(block->page.frame) != index->id || @@ -402,6 +404,7 @@ btr_root_adjust_on_import( goto func_exit; } + btr_search_drop_page_hash_index(block, index); page = buf_block_get_frame(block); page_zip = buf_block_get_page_zip(block); @@ -568,6 +571,7 @@ btr_page_alloc_for_ibuf( 0, RW_X_LATCH, nullptr, BUF_GET, mtr, err); if (new_block) { + btr_search_drop_page_hash_index(new_block, index); buf_page_make_young_if_needed(&new_block->page); *err= flst_remove(root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, new_block, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST_NODE, @@ -956,7 +960,7 @@ static void btr_free_root(buf_block_t *block, const fil_space_t &space, MTR_MEMO_PAGE_SX_FIX)); ut_ad(mtr->is_named_space(&space)); - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); if (btr_root_fseg_validate(PAGE_HEADER + PAGE_BTR_SEG_TOP, *block, space)) { @@ -984,15 +988,18 @@ buf_block_t *btr_free_root_check(const page_id_t page_id, ulint zip_size, buf_block_t *block= buf_page_get_gen(page_id, zip_size, RW_X_LATCH, nullptr, BUF_GET_POSSIBLY_FREED, mtr); - if (!block); - else if (fil_page_index_page_check(block->page.frame) && - index_id == btr_page_get_index_id(block->page.frame)) - /* This should be a root page. It should not be possible to - reassign the same index_id for some other index in the - tablespace. */ - ut_ad(!page_has_siblings(block->page.frame)); - else - block= nullptr; + if (block) + { + btr_search_drop_page_hash_index(block,reinterpret_cast(-1)); + if (fil_page_index_page_check(block->page.frame) && + index_id == btr_page_get_index_id(block->page.frame)) + /* This should be a root page. It should not be possible to + reassign the same index_id for some other index in the + tablespace. */ + ut_ad(!page_has_siblings(block->page.frame)); + else + block= nullptr; + } return block; } @@ -1234,7 +1241,7 @@ dberr_t dict_index_t::clear(que_thr_t *thr) ,any_ahi_pages() #endif ); - btr_search_drop_page_hash_index(root_block, false); + btr_search_drop_page_hash_index(root_block, nullptr); #ifdef BTR_CUR_HASH_ADAPT ut_ad(!any_ahi_pages()); #endif @@ -1347,12 +1354,13 @@ uint64_t btr_read_autoinc_with_fallback(const dict_table_t *table, uint64_t autoinc= 0; mtr_t mtr; mtr.start(); + const dict_index_t *const first_index= dict_table_get_first_index(table); if (buf_block_t *block= - buf_page_get(page_id_t(table->space_id, - dict_table_get_first_index(table)->page), + buf_page_get(page_id_t(table->space_id, first_index->page), table->space->zip_size(), RW_SX_LATCH, &mtr)) { + btr_search_drop_page_hash_index(block, first_index); autoinc= page_get_autoinc(block->page.frame); if (autoinc > 0 && autoinc <= max && mysql_version >= 100210); @@ -1405,6 +1413,9 @@ btr_write_autoinc(dict_index_t* index, ib_uint64_t autoinc, bool reset) if (buf_block_t *root= buf_page_get(page_id_t(space->id, index->page), space->zip_size(), RW_SX_LATCH, &mtr)) { +#ifdef BTR_CUR_HASH_ADAPT + ut_d(if (dict_index_t *ri= root->index)) ut_ad(ri == index); +#endif /* BTR_CUR_HASH_ADAPT */ buf_page_make_young_if_needed(&root->page); mtr.set_named_space(space); page_set_autoinc(root, autoinc, &mtr, reset); @@ -1435,7 +1446,7 @@ static dberr_t btr_page_reorganize_low(page_cur_t *cursor, mtr_t *mtr) if (UNIV_UNLIKELY(pos == ULINT_UNDEFINED)) return DB_CORRUPTION; - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); buf_block_t *old= buf_block_alloc(); /* Copy the old page to temporary space */ @@ -1765,7 +1776,7 @@ btr_page_empty( || page_zip_validate(page_zip, block->page.frame, index)); #endif /* UNIV_ZIP_DEBUG */ - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); /* Recreate the page: note that global data on page (possible segment headers, next page-field, etc.) is preserved intact */ @@ -3624,7 +3635,7 @@ btr_lift_page_up( mem_heap_free(heap); } - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); /* Make the father empty */ btr_page_empty(father_block, father_page_zip, index, page_level, mtr); @@ -3950,7 +3961,7 @@ btr_compress( goto err_exit; } - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); /* Remove the page from the level list */ err = btr_level_list_remove(*block, *index, mtr); @@ -4053,7 +4064,7 @@ btr_compress( goto err_exit; } - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); if (merge_page_zip && left_page_no == FIL_NULL) { @@ -4253,7 +4264,7 @@ btr_discard_only_page_on_level( ut_ad(fil_page_index_page_check(page)); ut_ad(block->page.id().space() == index->table->space->id); ut_ad(mtr->memo_contains_flagged(block, MTR_MEMO_PAGE_X_FIX)); - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); cursor.page_cur.index = index; cursor.page_cur.block = block; @@ -4450,7 +4461,7 @@ btr_discard_page( return DB_CORRUPTION; } - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); if (dict_index_is_spatial(index)) { rtr_node_ptr_delete(&parent_cursor, mtr); diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 73e2b6a542b70..a6b06d882736a 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -364,6 +364,8 @@ static dberr_t btr_cur_instant_init_low(dict_index_t* index, mtr_t* mtr) goto incompatible; } + btr_search_drop_page_hash_index(block, index); + if (fil_page_get_type(block->page.frame) != FIL_PAGE_TYPE_BLOB || mach_read_from_4(&block->page.frame [FIL_PAGE_DATA @@ -1337,6 +1339,8 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, goto search_loop; } + btr_search_drop_page_hash_index(block, index()); + if (!!page_is_comp(block->page.frame) != index()->table->not_redundant() || btr_page_get_index_id(block->page.frame) != index()->id || fil_page_get_type(block->page.frame) == FIL_PAGE_RTREE || @@ -1452,9 +1456,7 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, mtr->rollback_to_savepoint(savepoint, savepoint + 1); reached_index_root_and_leaf: ut_ad(rw_latch == RW_X_LATCH); -#ifdef BTR_CUR_HASH_ADAPT - btr_search_drop_page_hash_index(block, true); -#endif + btr_search_drop_page_hash_index(block, index()); if (page_cur_search_with_match(tuple, mode, &up_match, &low_match, &page_cur, nullptr)) goto corrupted; @@ -1752,6 +1754,7 @@ dberr_t btr_cur_t::pessimistic_search_leaf(const dtuple_t *tuple, const page_cur_mode_t page_mode{btr_cur_nonleaf_mode(mode)}; mtr->page_lock(block, RW_X_LATCH); + btr_search_drop_page_hash_index(block, index()); up_match= 0; up_bytes= 0; @@ -1821,6 +1824,8 @@ dberr_t btr_cur_t::pessimistic_search_leaf(const dtuple_t *tuple, goto func_exit; } + btr_search_drop_page_hash_index(block, index()); + if (!!page_is_comp(block->page.frame) != index()->table->not_redundant() || btr_page_get_index_id(block->page.frame) != index()->id || fil_page_get_type(block->page.frame) == FIL_PAGE_RTREE || @@ -1916,7 +1921,10 @@ dberr_t btr_cur_search_to_nth_level(ulint level, goto func_exit; } else + { + btr_search_drop_page_hash_index(block, index); btr_cur_nonleaf_make_young(&block->page); + } #ifdef UNIV_ZIP_DEBUG if (const page_zip_des_t *page_zip= buf_block_get_page_zip(block)) @@ -3531,6 +3539,9 @@ static void btr_cur_trim_alter_metadata(dtuple_t* entry, mtr.commit(); return; } + + btr_search_drop_page_hash_index(block, index); + ut_ad(fil_page_get_type(block->page.frame) == FIL_PAGE_TYPE_BLOB); ut_ad(mach_read_from_4(&block->page.frame [FIL_PAGE_DATA + BTR_BLOB_HDR_NEXT_PAGE_NO]) diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 642db0e9f1c57..fa0fcf0ce5672 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -479,7 +479,7 @@ btr_defragment_merge_pages( free it. */ lock_update_merge_left(*to_block, orig_pred, from_block->page.id()); - btr_search_drop_page_hash_index(from_block, false); + btr_search_drop_page_hash_index(from_block, nullptr); if (btr_level_list_remove(*from_block, *index, mtr) != DB_SUCCESS || btr_cur_node_ptr_delete(&parent, mtr) != DB_SUCCESS diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 529f463a70342..d85669f054ff6 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -26,8 +26,8 @@ Created 2/23/1996 Heikki Tuuri #include "btr0pcur.h" #include "buf0rea.h" +#include "btr0sea.h" #include "rem0cmp.h" -#include "trx0trx.h" #include "ibuf0ibuf.h" /**************************************************************//** @@ -271,11 +271,13 @@ static bool btr_pcur_optimistic_latch_leaves(btr_pcur_t *pcur, memcmp_aligned<2>(block->page.frame + PAGE_HEADER + PAGE_INDEX_ID, prev->page.frame + PAGE_HEADER + PAGE_INDEX_ID, 8)) goto fail; + btr_search_drop_page_hash_index(prev, pcur->index()); } else prev= nullptr; mtr->upgrade_buffer_fix(savepoint, mode); + btr_search_drop_page_hash_index(block, pcur->index()); if (UNIV_UNLIKELY(block->modify_clock != modify_clock) || UNIV_UNLIKELY(block->page.is_freed()) || diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index f252b38655984..e5dd218514ea5 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -635,7 +635,7 @@ static uint32_t btr_search_info_update_hash(const btr_cur_t &cursor) noexcept else if (UNIV_UNLIKELY(block_index != index)) { ut_ad(block_index->id == index->id); - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); } else if (cursor.flag == BTR_CUR_HASH_FAIL) btr_search_update_hash_ref(cursor, block, left_bytes_fields); @@ -1224,15 +1224,15 @@ static constexpr size_t REC_FOLD_IN_STACK= 128; /** Drop any adaptive hash index entries that point to an index page. @param block latched block containing index page, or a buffer-unfixed index page or a block in state BUF_BLOCK_REMOVE_HASH -@param garbage_collect drop ahi only if the index is marked as freed +@param not_garbage drop only if the index is set and NOT this @param folds work area for REC_FOLD_IN_STACK rec_fold() values */ static void btr_search_drop_page_hash_index(buf_block_t *block, - bool garbage_collect, + const dict_index_t *not_garbage, uint32_t *folds) noexcept { retry: dict_index_t *index= block->index; - if (!index) + if (!index || index == not_garbage) return; ut_d(const auto state= block->page.state()); @@ -1272,8 +1272,12 @@ static void btr_search_drop_page_hash_index(buf_block_t *block, goto retry; } } - else if (garbage_collect) + else if (not_garbage != nullptr) + { + ut_ad(!index || index == not_garbage || + not_garbage == reinterpret_cast(-1)); goto unlock_and_return; + } assert_block_ahi_valid(block); @@ -1399,10 +1403,10 @@ static void btr_search_drop_page_hash_index(buf_block_t *block, } void btr_search_drop_page_hash_index(buf_block_t *block, - bool garbage_collect) noexcept + const dict_index_t *not_garbage) noexcept { uint32_t folds[REC_FOLD_IN_STACK]; - btr_search_drop_page_hash_index(block, garbage_collect, folds); + btr_search_drop_page_hash_index(block, not_garbage, folds); } void btr_search_drop_page_hash_when_freed(const page_id_t page_id) noexcept @@ -1416,14 +1420,11 @@ void btr_search_drop_page_hash_when_freed(const page_id_t page_id) noexcept if (buf_block_t *block= buf_page_get_gen(page_id, 0, RW_X_LATCH, nullptr, BUF_PEEK_IF_IN_POOL, &mtr)) { - if (IF_DBUG(dict_index_t *index=,) block->index) - { - /* In all our callers, the table handle should be open, or we - should be in the process of dropping the table (preventing - eviction). */ - DBUG_ASSERT(index->table->get_ref_count() || dict_sys.locked()); - btr_search_drop_page_hash_index(block, false); - } + /* In all our callers, the table handle should be open, or we + should be in the process of dropping the table (preventing eviction). */ + ut_d(if (dict_index_t *i= block->index)) + ut_ad(i->table->get_ref_count() || dict_sys.locked()); + btr_search_drop_page_hash_index(block, nullptr); } mtr.commit(); @@ -1471,7 +1472,7 @@ static void btr_search_build_page_hash_index(dict_index_t *index, struct{uint32_t fold;uint32_t offset;} fr[REC_FOLD_IN_STACK / 2]; if (rebuild) - btr_search_drop_page_hash_index(block, false, &fr[0].fold); + btr_search_drop_page_hash_index(block, nullptr, &fr[0].fold); const uint32_t n_bytes_fields{left_bytes_fields & ~buf_block_t::LEFT_SIDE}; @@ -1619,7 +1620,7 @@ void btr_search_move_or_delete_hash_entries(buf_block_t *new_block, { ut_ad(!index || index == new_block_index); drop_exit: - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); return; } @@ -1666,7 +1667,7 @@ void btr_search_update_hash_on_delete(btr_cur_t *cursor) noexcept if (UNIV_UNLIKELY(index != cursor->index())) { - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); return; } @@ -1722,7 +1723,7 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept { ut_ad(index->id == cursor->index()->id); drop: - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); return; } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ce929527b2125..b08fc0de2bc0a 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2189,7 +2189,7 @@ void buf_page_free(fil_space_t *space, uint32_t page, mtr_t *mtr) ibuf_merge_or_delete_for_page(nullptr, page_id, block->page.zip_size()); #ifdef BTR_CUR_HASH_ADAPT if (block->index) - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); #endif /* BTR_CUR_HASH_ADAPT */ block->page.set_freed(block->page.state()); mtr->memo_push(block, MTR_MEMO_PAGE_X_MODIFY); @@ -2992,9 +2992,6 @@ buf_page_get_low( mtr->lock_register(mtr->get_savepoint() - 1, MTR_MEMO_BUF_FIX); goto corrupted; } -#ifdef BTR_CUR_HASH_ADAPT - btr_search_drop_page_hash_index(block, true); -#endif /* BTR_CUR_HASH_ADAPT */ ut_ad(page_id_t(page_get_space_id(block->page.frame), page_get_page_no(block->page.frame)) == page_id); @@ -3097,8 +3094,7 @@ buf_page_get_gen( mtr->memo_push(block, mtr_memo_type_t(rw_latch)); return block; } - mtr->page_lock(block, rw_latch); - return block; + return mtr->page_lock(block, rw_latch); } TRANSACTIONAL_TARGET @@ -3365,7 +3361,7 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, #ifdef BTR_CUR_HASH_ADAPT if (drop_hash_entry) btr_search_drop_page_hash_index(reinterpret_cast(bpage), - false); + nullptr); #endif /* BTR_CUR_HASH_ADAPT */ if (ibuf_exist && !recv_recovery_is_on()) diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index fd67cf11c8afd..350b1f92f48d4 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -949,7 +949,7 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) order to avoid bogus Valgrind or MSAN warnings.*/ MEM_MAKE_DEFINED(block->page.frame, srv_page_size); - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); MEM_UNDEFINED(block->page.frame, srv_page_size); mysql_mutex_lock(&buf_pool.mutex); } diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 0bd899ccd3205..024e18b1c7db0 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -33,6 +33,7 @@ Created Jan 06, 2010 Vasil Dimov #include #include "log.h" #include "btr0btr.h" +#include "btr0sea.h" #include "que0que.h" #include "scope.h" #include "debug_sync.h" @@ -1933,6 +1934,7 @@ dict_stats_analyze_index_below_cur( goto func_exit; } + btr_search_drop_page_hash_index(block, index); page = block->page.frame; if (page_is_leaf(page)) { diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index bfaaaafc3595d..09cbf6544d148 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -32,6 +32,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #else #include "buf0flu.h" #include "buf0dblwr.h" +#include "btr0sea.h" #include "srv0srv.h" #include "srv0start.h" #include "mtr0mtr.h" @@ -1673,6 +1674,8 @@ fil_crypt_get_page_throttle( BUF_PEEK_IF_IN_POOL, mtr); if (block != NULL) { /* page was in buffer pool */ + btr_search_drop_page_hash_index( + block, reinterpret_cast(-1)); state->crypt_stat.pages_read_from_cache++; return block; } @@ -1756,6 +1759,8 @@ fil_crypt_rotate_page( if (buf_block_t* block = fil_crypt_get_page_throttle(state, offset, &mtr, &sleeptime_ms)) { + btr_search_drop_page_hash_index( + block, reinterpret_cast(-1)); bool modified = false; byte* frame = buf_block_get_frame(block); const lsn_t block_lsn = mach_read_from_8(FIL_PAGE_LSN + frame); diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index e975a53ae8cc9..f2cd84838df2a 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -69,9 +69,9 @@ void btr_search_move_or_delete_hash_entries(buf_block_t *new_block, /** Drop any adaptive hash index entries that point to an index page. @param block latched block containing index page, or a buffer-unfixed index page or a block in state BUF_BLOCK_REMOVE_HASH -@param garbage_collect drop ahi only if the index is marked as freed */ -void btr_search_drop_page_hash_index(buf_block_t* block, - bool garbage_collect) noexcept; +@param not_garbage drop only if the index is set and NOT this */ +void btr_search_drop_page_hash_index(buf_block_t *block, + const dict_index_t *not_garbage) noexcept; /** Drop possible adaptive hash index entries when a page is evicted from the buffer pool or freed in a file, or the index is being dropped. @@ -202,7 +202,7 @@ extern ulint btr_search_n_hash_fail; #else /* BTR_CUR_HASH_ADAPT */ # define btr_search_sys_create() # define btr_search_sys_free() -# define btr_search_drop_page_hash_index(block, garbage_collect) +# define btr_search_drop_page_hash_index(block, not_garbage) # define btr_search_move_or_delete_hash_entries(new_block, block) # define btr_search_update_hash_on_insert(cursor, ahi_latch) # define btr_search_update_hash_on_delete(cursor) diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 3828a31cdb518..612985889fd77 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -325,13 +325,14 @@ struct mtr_t { /** Latch a buffer pool block. @param block block to be latched - @param rw_latch RW_S_LATCH, RW_SX_LATCH, RW_X_LATCH, RW_NO_LATCH */ - void page_lock(buf_block_t *block, ulint rw_latch); + @param rw_latch RW_S_LATCH, RW_SX_LATCH, RW_X_LATCH, RW_NO_LATCH + @return block */ + buf_block_t *page_lock(buf_block_t *block, ulint rw_latch) noexcept; /** Acquire a latch on a buffer-fixed buffer pool block. @param savepoint savepoint location of the buffer-fixed block @param rw_latch latch to acquire */ - void upgrade_buffer_fix(ulint savepoint, rw_lock_type_t rw_latch); + void upgrade_buffer_fix(ulint savepoint, rw_lock_type_t rw_latch) noexcept; /** Register a change to the page latch state. */ void lock_register(ulint savepoint, mtr_memo_type_t type) diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 498489a50a33a..aed6e6dc73d24 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -928,10 +928,7 @@ void mtr_t::page_lock_upgrade(const buf_block_t &block) #endif /* BTR_CUR_HASH_ADAPT */ } -/** Latch a buffer pool block. -@param block block to be latched -@param rw_latch RW_S_LATCH, RW_SX_LATCH, RW_X_LATCH, RW_NO_LATCH */ -void mtr_t::page_lock(buf_block_t *block, ulint rw_latch) +buf_block_t *mtr_t::page_lock(buf_block_t *block, ulint rw_latch) noexcept { mtr_memo_type_t fix_type; ut_d(const auto state= block->page.state()); @@ -957,23 +954,21 @@ void mtr_t::page_lock(buf_block_t *block, ulint rw_latch) { block->unfix(); page_lock_upgrade(*block); - return; + return block; } ut_ad(!block->page.is_io_fixed()); } -#ifdef BTR_CUR_HASH_ADAPT - btr_search_drop_page_hash_index(block, true); -#endif - done: ut_ad(state < buf_page_t::UNFIXED || page_id_t(page_get_space_id(block->page.frame), page_get_page_no(block->page.frame)) == block->page.id()); memo_push(block, fix_type); + return block; } void mtr_t::upgrade_buffer_fix(ulint savepoint, rw_lock_type_t rw_latch) + noexcept { ut_ad(is_active()); mtr_memo_slot_t &slot= m_memo[savepoint]; @@ -1003,9 +998,6 @@ void mtr_t::upgrade_buffer_fix(ulint savepoint, rw_lock_type_t rw_latch) ut_ad(!block->page.is_io_fixed()); } -#ifdef BTR_CUR_HASH_ADAPT - btr_search_drop_page_hash_index(block, true); -#endif ut_ad(page_id_t(page_get_space_id(block->page.frame), page_get_page_no(block->page.frame)) == block->page.id()); } @@ -1235,7 +1227,7 @@ void mtr_t::free(const fil_space_t &space, uint32_t offset) } } else if (slot.type & (MTR_MEMO_PAGE_X_FIX | MTR_MEMO_PAGE_SX_FIX) && - block->page.id() == id) + block->page.id() == id) { ut_ad(!block->page.is_freed()); ut_ad(!freed); @@ -1258,7 +1250,7 @@ void mtr_t::free(const fil_space_t &space, uint32_t offset) } #ifdef BTR_CUR_HASH_ADAPT if (block->index) - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); #endif /* BTR_CUR_HASH_ADAPT */ block->page.set_freed(block->page.state()); } diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 0448f24a42c0c..19d112cb0d6c4 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4406,7 +4406,7 @@ page_zip_reorganize( mtr_log_t log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE); temp_block = buf_block_alloc(); - btr_search_drop_page_hash_index(block, false); + btr_search_drop_page_hash_index(block, nullptr); temp_page = temp_block->page.frame; /* Copy the old page to temporary space */ From 817e80e87159e09558be1c97703c6f2685ea3db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:17:27 +0200 Subject: [PATCH 08/10] MDEV-35049: Avoid building AHI beyond unique field prefix During a workload, an adaptive hash index had been built on UNIQUE INDEX(ID) on SYS_TABLES, and during a DROP TABLE operation the adaptive hash index would be widened to cover also the PRIMARY KEY(NAME) field that the index includes: (ID,NAME). Such an adaptive hash index is unlikely to satisfy (m)any queries. Let us limit the AHI prefix to the unique fields. Reviewed by: Vladislav Lesin --- storage/innobase/btr/btr0sea.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index e5dd218514ea5..1314eb197c094 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -525,7 +525,8 @@ static uint32_t btr_search_info_update_hash(const btr_cur_t &cursor) noexcept const dict_index_t *const block_index= block->index; uint16_t n_hash_helps{block->n_hash_helps}; - const uint16_t n_uniq{dict_index_get_n_unique_in_tree(index)}; + const uint16_t n_uniq= + uint16_t(index->n_uniq ? index->n_uniq : index->n_fields); dict_index_t::ahi &info= index->search_info; uint32_t left_bytes_fields{info.left_bytes_fields}; uint8_t n_hash_potential= info.n_hash_potential; @@ -1478,10 +1479,8 @@ static void btr_search_build_page_hash_index(dict_index_t *index, /* Check that the values for hash index build are sensible */ ut_ad(n_bytes_fields); - - if (dict_index_get_n_unique_in_tree(index) < - btr_search_get_n_fields(n_bytes_fields)) - return; + ut_ad(btr_search_get_n_fields(n_bytes_fields) <= + (index->n_uniq ? index->n_uniq : index->n_fields)); const page_t *const page= block->page.frame; size_t n_cached= 0; From 513ee480c06e71c0d902984e2d8051b3658aad9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:17:51 +0200 Subject: [PATCH 09/10] MDEV-35049: Always enable page_cur_search_with_match_bytes() For some reason, page_cur_search_with_match_bytes(), which can speed up append operations (PAGE_CUR_LE used by INSERT), was only enabled if innodb_adaptive_hash_index=ON even though it has nothing to do with the adaptive hash index. Furthermore, mysql/mysql-server@c9bbc83d11b56831658b03475b23306e3d48acca a.k.a. commit c9bbc83d11b56831658b03475b23306e3d48acca reduced a limit from 3 to 2 but forgot to adjust the PAGE_N_DIRECTION limit accordingly. We are adjusting that as well. Reviewed by: Vladislav Lesin --- storage/innobase/btr/btr0cur.cc | 12 ++++-------- storage/innobase/page/page0cur.cc | 10 ++++------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index a6b06d882736a..54d52f372a1cb 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1511,19 +1511,15 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, } reached_latched_leaf: -#ifdef BTR_CUR_HASH_ADAPT - if (!(tuple->info_bits & REC_INFO_MIN_REC_FLAG) && !index()->is_ibuf() && - btr_search.enabled) + if (!(tuple->info_bits & REC_INFO_MIN_REC_FLAG) && !index()->is_ibuf()) { if (page_cur_search_with_match_bytes(*tuple, mode, &up_match, &low_match, &page_cur, &up_bytes, &low_bytes)) goto corrupted; } - else -#endif /* BTR_CUR_HASH_ADAPT */ - if (page_cur_search_with_match(tuple, mode, &up_match, &low_match, - &page_cur, nullptr)) - goto corrupted; + else if (page_cur_search_with_match(tuple, mode, &up_match, &low_match, + &page_cur, nullptr)) + goto corrupted; ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_GE); ut_ad(up_match != uint16_t(~0U) || mode != PAGE_CUR_LE); diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index b5585fa02f279..a005c2d8d4fd1 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -36,7 +36,6 @@ Created 10/4/1994 Heikki Tuuri # include "trx0roll.h" #endif -#ifdef BTR_CUR_HASH_ADAPT /** Get the pad character code point for a type. @param type @return pad character code point @@ -424,7 +423,7 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, else if (uint16_t last= page_header_get_offs(page, PAGE_LAST_INSERT)) { const rec_t *rec= page + last; - if (page_header_get_field(page, PAGE_N_DIRECTION) > 3 && + if (page_header_get_field(page, PAGE_N_DIRECTION) > 2 && page_cur_try_search_shortcut_bytes(page, rec, index, tuple, iup_fields, ilow_fields, iup_bytes, ilow_bytes)) @@ -660,6 +659,7 @@ static int cmp_dtuple_rec_leaf(const dtuple_t &dtuple, const rec_t *rec, return ret; } +#ifdef BTR_CUR_HASH_ADAPT bool btr_cur_t::check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) noexcept { @@ -733,6 +733,7 @@ bool btr_cur_t::check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) return cmp_dtuple_rec_leaf(tuple, rec, *index(), &match, comp) <= 0; } } +#endif /* BTR_CUR_HASH_ADAPT */ /** Try a search shortcut based on the last insert. @param page index page @@ -784,7 +785,6 @@ static bool page_cur_try_search_shortcut(const page_t *page, const rec_t *rec, *ilow= low; return true; } -#endif /* BTR_CUR_HASH_ADAPT */ /****************************************************************//** Searches the right position for a page cursor. */ @@ -849,12 +849,11 @@ page_cur_search_with_match( block, (dict_index_t*)index, tuple, mode, cursor, rtr_info); } -#ifdef BTR_CUR_HASH_ADAPT } else if (!n_core || mode != PAGE_CUR_LE || !page_is_leaf(page) || page_get_direction(page) != PAGE_RIGHT) { } else if (uint16_t last = page_header_get_offs(page, PAGE_LAST_INSERT)) { - if (page_header_get_field(page, PAGE_N_DIRECTION) > 3 + if (page_header_get_field(page, PAGE_N_DIRECTION) > 2 && page_cur_try_search_shortcut(page, page + last, *index, *tuple, iup_matched_fields, @@ -862,7 +861,6 @@ page_cur_search_with_match( page_cur_position(page + last, block, cursor); return false; } -#endif /* BTR_CUR_HASH_ADAPT */ } /* If mode PAGE_CUR_G is specified, we are trying to position the From aab33d0cb06b1f929c6385268ee717a53048f64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2024 11:18:00 +0200 Subject: [PATCH 10/10] MDEV-35312 page_cur_search_with_match() could avoid rec_get_offsets() page_cur_search_with_match(): Remove rec_get_offsets(), and instead determine the start and end of each field while comparing. page_dir_slot_get_rec(), page_dir_slot_get_rec_validate(): Add a parameter to avoid invoking page_align(). page_cur_dtuple_cmp(): Replaces cmp_dtuple_rec_leaf() for both leaf and non-leaf pages. In SPATIAL INDEX, non-leaf records are special, because the child page number may be part of the comparison. Reviewed by: Vladislav Lesin --- storage/innobase/include/page0cur.h | 31 +- storage/innobase/include/page0page.h | 38 ++- storage/innobase/page/page0cur.cc | 470 +++++++++++++-------------- 3 files changed, 262 insertions(+), 277 deletions(-) diff --git a/storage/innobase/include/page0cur.h b/storage/innobase/include/page0cur.h index 6c3ac21973af0..3454be99f1610 100644 --- a/storage/innobase/include/page0cur.h +++ b/storage/innobase/include/page0cur.h @@ -222,23 +222,18 @@ bool page_apply_delete_dynamic(const buf_block_t &block, ulint prev, size_t hdr_size, size_t data_size); MY_ATTRIBUTE((warn_unused_result)) -/****************************************************************//** -Searches the right position for a page cursor. */ -bool -page_cur_search_with_match( -/*=======================*/ - const dtuple_t* tuple, /*!< in: data tuple */ - page_cur_mode_t mode, /*!< in: PAGE_CUR_L, - PAGE_CUR_LE, PAGE_CUR_G, or - PAGE_CUR_GE */ - uint16_t* iup_matched_fields, - /*!< in/out: already matched - fields in upper limit record */ - uint16_t* ilow_matched_fields, - /*!< in/out: already matched - fields in lower limit record */ - page_cur_t* cursor, /*!< in/out: page cursor */ - rtr_info_t* rtr_info);/*!< in/out: rtree search stack */ +/** Search the right position for a page cursor. +@param tuple search key +@param mode search mode +@param iup_fields matched fields in the upper limit record +@param ilow_fields matched fields in the low limit record +@param cursor page cursor +@param rtr_info R-tree search stack, or nullptr +@return whether the page is corrupted */ +bool page_cur_search_with_match(const dtuple_t *tuple, page_cur_mode_t mode, + uint16_t *iup_fields, uint16_t *ilow_fields, + page_cur_t *cursor, rtr_info_t *rtr_info) + noexcept; /** Search the right position for a page cursor. @param tuple search key @@ -248,7 +243,7 @@ page_cur_search_with_match( @param cursor page cursor @param iup_bytes matched bytes after iup_fields @param ilow_bytes matched bytes after ilow_fields -@return whether the first partially matched field in the lower limit record, +@return whether the first partially matched field is in the lower limit record, or the page is corrupted */ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, page_cur_mode_t mode, diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index 155f6e7fc450a..89d47a3bdb657 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -636,31 +636,55 @@ page_rec_check( /** Get the record pointed to by a directory slot. @param[in] slot directory slot @return pointer to record */ -inline rec_t *page_dir_slot_get_rec(page_dir_slot_t *slot) +inline rec_t *page_dir_slot_get_rec(page_t *page, page_dir_slot_t *slot) + noexcept { - return page_align(slot) + mach_read_from_2(my_assume_aligned<2>(slot)); + return page + mach_read_from_2(my_assume_aligned<2>(slot)); } -inline const rec_t *page_dir_slot_get_rec(const page_dir_slot_t *slot) +inline const rec_t *page_dir_slot_get_rec(const page_t *page, + const page_dir_slot_t *slot) noexcept +{ + return page_dir_slot_get_rec(const_cast(page), + const_cast(slot)); +} + +inline rec_t *page_dir_slot_get_rec(page_dir_slot_t *slot) noexcept +{ + return page_dir_slot_get_rec(page_align(slot), slot); +} +inline const rec_t *page_dir_slot_get_rec(const page_dir_slot_t *slot) noexcept { return page_dir_slot_get_rec(const_cast(slot)); } -inline rec_t *page_dir_slot_get_rec_validate(page_dir_slot_t *slot) +inline rec_t *page_dir_slot_get_rec_validate(page_t *page, + page_dir_slot_t *slot) noexcept { const size_t s= mach_read_from_2(my_assume_aligned<2>(slot)); - page_t *page= page_align(slot); - return UNIV_LIKELY(s >= PAGE_NEW_INFIMUM && s <= page_header_get_field(page, PAGE_HEAP_TOP)) ? page + s : nullptr; } + +inline const rec_t *page_dir_slot_get_rec_validate(const page_t *page, + const page_dir_slot_t *slot) + noexcept +{ + return page_dir_slot_get_rec_validate(const_cast(page), + const_cast(slot)); +} + +inline rec_t *page_dir_slot_get_rec_validate(page_dir_slot_t *slot) noexcept +{ + return page_dir_slot_get_rec_validate(page_align(slot), slot); +} inline const rec_t *page_dir_slot_get_rec_validate(const page_dir_slot_t *slot) + noexcept { return page_dir_slot_get_rec_validate(const_cast(slot)); } - /***************************************************************//** Gets the number of records owned by a directory slot. @return number of records */ diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index a005c2d8d4fd1..cc0a6224f7ef8 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -69,8 +69,7 @@ static ulint cmp_get_pad_char(const dtype_t &type) noexcept } /** Compare a data tuple to a physical record. -@param page B-tree index leaf page -@param rec PAGE_LAST_INSERT record +@param rec B-tree index record @param index index B-tree @param tuple search key @param match matched fields << 16 | bytes @@ -79,23 +78,24 @@ static ulint cmp_get_pad_char(const dtype_t &type) noexcept @retval 0 if dtuple is equal to rec @retval negative if dtuple is less than rec @retval positive if dtuple is greater than rec */ -static int cmp_dtuple_rec_bytes(const page_t *page, - const rec_t *rec, +static int cmp_dtuple_rec_bytes(const rec_t *rec, const dict_index_t &index, const dtuple_t &tuple, int *match, ulint comp) noexcept { ut_ad(dtuple_check_typed(&tuple)); - ut_ad(page_is_leaf(page)); + ut_ad(page_rec_is_leaf(rec)); ut_ad(!(REC_INFO_MIN_REC_FLAG & dtuple_get_info_bits(&tuple))); ut_ad(!!comp == index.table->not_redundant()); ut_ad(!index.is_ibuf()); if (UNIV_UNLIKELY(REC_INFO_MIN_REC_FLAG & rec_get_info_bits(rec, comp))) { + ut_d(const page_t *page= page_align(rec)); ut_ad(page_rec_is_first(rec, page)); ut_ad(!page_has_prev(page)); ut_ad(rec_is_metadata(rec, index)); + *match= 0; return 1; } @@ -112,9 +112,18 @@ static int cmp_dtuple_rec_bytes(const page_t *page, if (UNIV_LIKELY(comp != 0)) { - const unsigned n_core_null_bytes= index.n_core_null_bytes; const byte *nulls= rec - REC_N_NEW_EXTRA_BYTES; - const byte *lens= --nulls - n_core_null_bytes; + const byte *lens; + if (rec_get_status(rec) == REC_STATUS_INSTANT) + { + ulint n_fields= index.n_core_fields + rec_get_n_add_field(nulls) + 1; + ut_ad(n_fields <= index.n_fields); + const ulint n_nullable= index.get_n_nullable(n_fields); + ut_ad(n_nullable <= index.n_nullable); + lens= --nulls - UT_BITS_IN_BYTES(n_nullable); + } + else + lens= --nulls - index.n_core_null_bytes; byte null_mask= 1; size_t i= 0; @@ -356,7 +365,7 @@ static bool page_cur_try_search_shortcut_bytes(const page_t *page, int up= int(*iup_fields << 16 | *iup_bytes); up= low= std::min(low, up); const auto comp= page_is_comp(page); - if (cmp_dtuple_rec_bytes(page, rec, index, tuple, &low, comp) < 0) + if (cmp_dtuple_rec_bytes(rec, index, tuple, &low, comp) < 0) return false; const rec_t *next; if (UNIV_LIKELY(comp != 0)) @@ -366,7 +375,7 @@ static bool page_cur_try_search_shortcut_bytes(const page_t *page, if (next != page + PAGE_NEW_SUPREMUM) { cmp_up: - if (cmp_dtuple_rec_bytes(page, rec, index, tuple, &up, comp) >= 0) + if (cmp_dtuple_rec_bytes(rec, index, tuple, &up, comp) >= 0) return false; *iup_fields= uint16_t(up >> 16); *iup_bytes= uint16_t(up); @@ -385,24 +394,13 @@ static bool page_cur_try_search_shortcut_bytes(const page_t *page, return true; } -/** Search the right position for a page cursor. -@param tuple search key -@param mode search mode -@param iup_fields matched fields in the upper limit record -@param ilow_fields matched fields in the low limit record -@param cursor page cursor -@param iup_bytes matched bytes after iup_fields -@param ilow_bytes matched bytes after ilow_fields -@return whether the first partially matched field in the lower limit record, -or the page is corrupted */ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, page_cur_mode_t mode, uint16_t *iup_fields, uint16_t *ilow_fields, page_cur_t *cursor, uint16_t *iup_bytes, - uint16_t *ilow_bytes) - noexcept + uint16_t *ilow_bytes) noexcept { ut_ad(dtuple_validate(&tuple)); ut_ad(!(tuple.info_bits & REC_INFO_MIN_REC_FLAG)); @@ -452,11 +450,12 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, while (up - low > 1) { const size_t mid= (low + up) / 2; - mid_rec= page_dir_slot_get_rec_validate(page_dir_get_nth_slot(page, mid)); + mid_rec= page_dir_slot_get_rec_validate(page, + page_dir_get_nth_slot(page, mid)); if (UNIV_UNLIKELY(!mid_rec)) return true; int cur= std::min(low_cmp, up_cmp); - int cmp= cmp_dtuple_rec_bytes(page, mid_rec, index, tuple, &cur, comp); + int cmp= cmp_dtuple_rec_bytes(mid_rec, index, tuple, &cur, comp); if (cmp > 0) low_slot_match: low= mid, low_cmp= cur; @@ -470,9 +469,9 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, } const rec_t *up_rec= - page_dir_slot_get_rec_validate(page_dir_get_nth_slot(page, up));; + page_dir_slot_get_rec_validate(page, page_dir_get_nth_slot(page, up)); const rec_t *low_rec= - page_dir_slot_get_rec_validate(page_dir_get_nth_slot(page, low)); + page_dir_slot_get_rec_validate(page, page_dir_get_nth_slot(page, low)); if (UNIV_UNLIKELY(!low_rec || !up_rec)) return true; @@ -495,12 +494,12 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, if (UNIV_UNLIKELY(rec_get_info_bits(mid_rec, comp) & REC_INFO_MIN_REC_FLAG)) { - ut_ad(!page_has_prev(page_align(mid_rec))); + ut_ad(!page_has_prev(page)); ut_ad(rec_is_metadata(mid_rec, index)); goto low_rec_match; } - cmp= cmp_dtuple_rec_bytes(page, mid_rec, index, tuple, &cur, comp); + cmp= cmp_dtuple_rec_bytes(mid_rec, index, tuple, &cur, comp); if (cmp > 0) low_rec_match: @@ -521,6 +520,7 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, } /** Compare a data tuple to a physical record. +@tparam leaf whether this must be a leaf page @param page B-tree index page @param rec B-tree index record @param index index B-tree @@ -531,7 +531,8 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, @retval 0 if dtuple is equal to rec @retval negative if dtuple is less than rec @retval positive if dtuple is greater than rec */ -static int cmp_dtuple_rec_leaf(const dtuple_t &dtuple, const rec_t *rec, +template +static int page_cur_dtuple_cmp(const dtuple_t &dtuple, const rec_t *rec, const dict_index_t &index, uint16_t *matched_fields, ulint comp) noexcept { @@ -541,15 +542,17 @@ static int cmp_dtuple_rec_leaf(const dtuple_t &dtuple, const rec_t *rec, ut_ad(dtuple.n_fields_cmp > 0); ut_ad(dtuple.n_fields_cmp <= index.n_core_fields || index.is_ibuf()); ut_ad(cur_field <= dtuple.n_fields_cmp); - ut_ad(page_rec_is_leaf(rec)); - ut_ad(!(rec_get_info_bits(rec, comp) & REC_INFO_MIN_REC_FLAG) || + ut_ad(leaf == page_rec_is_leaf(rec)); + ut_ad(!leaf || !(rec_get_info_bits(rec, comp) & REC_INFO_MIN_REC_FLAG) || index.is_instant() || (index.is_primary() && trx_roll_crash_recv_trx && !trx_rollback_is_active)); - ut_ad(!(dtuple.info_bits & REC_INFO_MIN_REC_FLAG) || + ut_ad(!leaf || !(dtuple.info_bits & REC_INFO_MIN_REC_FLAG) || index.is_instant() || (index.is_primary() && trx_roll_crash_recv_trx && !trx_rollback_is_active)); + ut_ad(leaf || !index.is_spatial() || + dtuple.n_fields_cmp == DICT_INDEX_SPATIAL_NODEPTR_SIZE + 1); int ret= 0; if (dtuple.info_bits & REC_INFO_MIN_REC_FLAG) @@ -628,6 +631,17 @@ static int cmp_dtuple_rec_leaf(const dtuple_t &dtuple, const rec_t *rec, { const dfield_t *df= dtuple_get_nth_field(&dtuple, i); ut_ad(!dfield_is_ext(df)); + if (!leaf && i == DICT_INDEX_SPATIAL_NODEPTR_SIZE && + index.is_spatial()) + { + /* SPATIAL INDEX non-leaf records comprise + MBR (minimum bounding rectangle) and the child page number. + The function rtr_cur_restore_position() includes the + child page number in the search key, because the MBR alone + would not be unique. */ + ut_ad(dtuple.fields[DICT_INDEX_SPATIAL_NODEPTR_SIZE].len == 4); + len= 4; + } ret= cmp_data_data(df->type.mtype, df->type.prtype, static_cast(df->data), df->len, f, len); @@ -659,6 +673,16 @@ static int cmp_dtuple_rec_leaf(const dtuple_t &dtuple, const rec_t *rec, return ret; } +static int page_cur_dtuple_cmp(const dtuple_t &dtuple, const rec_t *rec, + const dict_index_t &index, + uint16_t *matched_fields, ulint comp, bool leaf) + noexcept +{ + return leaf + ? page_cur_dtuple_cmp(dtuple, rec, index, matched_fields, comp) + : page_cur_dtuple_cmp(dtuple, rec, index, matched_fields, comp); +} + #ifdef BTR_CUR_HASH_ADAPT bool btr_cur_t::check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) noexcept @@ -668,7 +692,7 @@ bool btr_cur_t::check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) const rec_t *rec= page_cur.rec; uint16_t match= 0; - int cmp= cmp_dtuple_rec_leaf(tuple, rec, *index(), &match, comp); + int cmp= page_cur_dtuple_cmp(tuple, rec, *index(), &match, comp); const auto uniq= dict_index_get_n_unique_in_tree(index()); ut_ad(match <= uniq); ut_ad(match <= tuple.n_fields_cmp); @@ -708,7 +732,7 @@ bool btr_cur_t::check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) if (uintptr_t(rec - page) == PAGE_OLD_SUPREMUM) goto le_supremum; } - return cmp_dtuple_rec_leaf(tuple, rec, *index(), &up_match, comp) >= 0; + return page_cur_dtuple_cmp(tuple, rec, *index(), &up_match, comp) >= 0; } else { @@ -730,34 +754,33 @@ bool btr_cur_t::check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) default: return true; } - return cmp_dtuple_rec_leaf(tuple, rec, *index(), &match, comp) <= 0; + return page_cur_dtuple_cmp(tuple, rec, *index(), &match, comp) <= 0; } } #endif /* BTR_CUR_HASH_ADAPT */ /** Try a search shortcut based on the last insert. -@param page index page -@param rec PAGE_LAST_INSERT record -@param index index tree -@param tuple search key -@param iup matched fields in the upper limit record -@param ilow matched fields in the lower limit record +@param page index page +@param rec PAGE_LAST_INSERT record +@param index index tree +@param tuple search key +@param iup matched fields in the upper limit record +@param ilow matched fields in the lower limit record +@param comp nonzero if ROW_FORMAT=REDUNDANT is not being used @return record @return nullptr if the tuple was not found */ static bool page_cur_try_search_shortcut(const page_t *page, const rec_t *rec, const dict_index_t &index, const dtuple_t &tuple, - uint16_t *iup, uint16_t *ilow) - noexcept + uint16_t *iup, uint16_t *ilow, + ulint comp) noexcept { ut_ad(dtuple_check_typed(&tuple)); - ut_ad(page_is_leaf(page)); - auto comp= page_is_comp(page); ut_ad(page_rec_is_user_rec(rec)); uint16_t low= std::min(*ilow, *iup), up= low; - if (cmp_dtuple_rec_leaf(tuple, rec, index, &low, comp) < 0) + if (page_cur_dtuple_cmp(tuple, rec, index, &low, comp) < 0) return false; if (comp) @@ -768,7 +791,7 @@ static bool page_cur_try_search_shortcut(const page_t *page, const rec_t *rec, if (rec != page + PAGE_NEW_SUPREMUM) { compare_next: - if (cmp_dtuple_rec_leaf(tuple, rec, index, &up, comp) >= 0) + if (page_cur_dtuple_cmp(tuple, rec, index, &up, comp) >= 0) return false; *iup= up; } @@ -786,210 +809,140 @@ static bool page_cur_try_search_shortcut(const page_t *page, const rec_t *rec, return true; } -/****************************************************************//** -Searches the right position for a page cursor. */ -bool -page_cur_search_with_match( -/*=======================*/ - const dtuple_t* tuple, /*!< in: data tuple */ - page_cur_mode_t mode, /*!< in: PAGE_CUR_L, - PAGE_CUR_LE, PAGE_CUR_G, or - PAGE_CUR_GE */ - uint16_t* iup_matched_fields, - /*!< in/out: already matched - fields in upper limit record */ - uint16_t* ilow_matched_fields, - /*!< in/out: already matched - fields in lower limit record */ - page_cur_t* cursor, /*!< out: page cursor */ - rtr_info_t* rtr_info)/*!< in/out: rtree search stack */ +bool page_cur_search_with_match(const dtuple_t *tuple, page_cur_mode_t mode, + uint16_t *iup_fields, uint16_t *ilow_fields, + page_cur_t *cursor, rtr_info_t *rtr_info) + noexcept { - ulint up; - ulint low; - ulint mid; - const page_t* page; - const rec_t* up_rec; - const rec_t* low_rec; - const rec_t* mid_rec; - uint16_t up_matched_fields; - uint16_t low_matched_fields; - uint16_t cur_matched_fields; - int cmp; - const dict_index_t* const index = cursor->index; - const buf_block_t* const block = cursor->block; -#ifdef UNIV_ZIP_DEBUG - const page_zip_des_t* page_zip = buf_block_get_page_zip(block); -#endif /* UNIV_ZIP_DEBUG */ - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - rec_offs_init(offsets_); - - ut_ad(dtuple_validate(tuple)); - ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE - || mode == PAGE_CUR_G || mode == PAGE_CUR_GE - || index->is_spatial()); - page = buf_block_get_frame(block); + ut_ad(dtuple_validate(tuple)); + ut_ad(mode == PAGE_CUR_L || mode == PAGE_CUR_LE || + mode == PAGE_CUR_G || mode == PAGE_CUR_GE || + cursor->index->is_spatial()); + const dict_index_t &index= *cursor->index; + const buf_block_t *const block= cursor->block; + const page_t *const page= block->page.frame; + ut_d(page_check_dir(page)); #ifdef UNIV_ZIP_DEBUG - ut_a(!page_zip || page_zip_validate(page_zip, page, index)); + if (const page_zip_des_t *page_zip= buf_block_get_page_zip(block)) + ut_a(page_zip_validate(page_zip, page, &index)); #endif /* UNIV_ZIP_DEBUG */ + const auto comp= page_is_comp(page); + const bool leaf{page_is_leaf(page)}; + + /* If the mode is for R-tree indexes, use the special MBR + related compare functions */ + if (mode == PAGE_CUR_RTREE_INSERT && leaf) + { + /* Leaf level insert uses the traditional compare function */ + mode= PAGE_CUR_LE; + goto check_last_insert; + } + else if (mode > PAGE_CUR_LE) + return rtr_cur_search_with_match(block, + const_cast(&index), + tuple, mode, cursor, rtr_info); + else if (mode == PAGE_CUR_LE && leaf) + { + check_last_insert: + if (page_get_direction(page) != PAGE_RIGHT || + (tuple->info_bits & REC_INFO_MIN_REC_FLAG)); + else if (uint16_t last= page_header_get_offs(page, PAGE_LAST_INSERT)) + { + const rec_t *rec= page + last; + if (page_header_get_field(page, PAGE_N_DIRECTION) > 2 && + page_cur_try_search_shortcut(page, rec, index, *tuple, + iup_fields, ilow_fields, comp)) + { + page_cur_position(rec, block, cursor); + return false; + } + } + } - ut_d(page_check_dir(page)); - const ulint n_core = page_is_leaf(page) ? index->n_core_fields : 0; - - /* If the mode is for R-tree indexes, use the special MBR - related compare functions */ - if (index->is_spatial()) { - /* For leaf level insert, we still use the traditional - compare function for now */ - if (n_core && mode == PAGE_CUR_RTREE_INSERT) { - mode = PAGE_CUR_LE; - } else if (mode > PAGE_CUR_LE) { - return rtr_cur_search_with_match( - block, (dict_index_t*)index, tuple, mode, - cursor, rtr_info); - } - } else if (!n_core || mode != PAGE_CUR_LE || !page_is_leaf(page) - || page_get_direction(page) != PAGE_RIGHT) { - } else if (uint16_t last = - page_header_get_offs(page, PAGE_LAST_INSERT)) { - if (page_header_get_field(page, PAGE_N_DIRECTION) > 2 - && page_cur_try_search_shortcut(page, page + last, *index, - *tuple, - iup_matched_fields, - ilow_matched_fields)) { - page_cur_position(page + last, block, cursor); - return false; - } - } - - /* If mode PAGE_CUR_G is specified, we are trying to position the - cursor to answer a query of the form "tuple < X", where tuple is - the input parameter, and X denotes an arbitrary physical record on - the page. We want to position the cursor on the first X which - satisfies the condition. */ - - up_matched_fields = *iup_matched_fields; - low_matched_fields = *ilow_matched_fields; - - /* Perform binary search. First the search is done through the page - directory, after that as a linear search in the list of records - owned by the upper limit directory slot. */ - - low = 0; - up = ulint(page_dir_get_n_slots(page)) - 1; - - /* Perform binary search until the lower and upper limit directory - slots come to the distance 1 of each other */ - - while (up - low > 1) { - mid = (low + up) / 2; - const page_dir_slot_t* slot = page_dir_get_nth_slot(page, mid); - if (UNIV_UNLIKELY(!(mid_rec - = page_dir_slot_get_rec_validate(slot)))) { - goto corrupted; - } - cur_matched_fields = std::min(low_matched_fields, - up_matched_fields); - - offsets = offsets_; - offsets = rec_get_offsets( - mid_rec, index, offsets, n_core, - dtuple_get_n_fields_cmp(tuple), &heap); - - cmp = cmp_dtuple_rec_with_match( - tuple, mid_rec, offsets, &cur_matched_fields); - - if (cmp > 0) { -low_slot_match: - low = mid; - low_matched_fields = cur_matched_fields; - - } else if (cmp) { -up_slot_match: - up = mid; - up_matched_fields = cur_matched_fields; + /* If mode PAGE_CUR_G is specified, we are trying to position the + cursor to answer a query of the form "tuple < X", where tuple is the + input parameter, and X denotes an arbitrary physical record on the + page. We want to position the cursor on the first X which satisfies + the condition. */ + uint16_t up_fields= *iup_fields, low_fields= *ilow_fields; - } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE) { - goto low_slot_match; - } else { + /* Perform binary search. First the search is done through the page + directory, after that as a linear search in the list of records + owned by the upper limit directory slot. */ + size_t low= 0, up= ulint{page_dir_get_n_slots(page)} - 1; + const rec_t *mid_rec; - goto up_slot_match; - } - } + /* Perform binary search until the lower and upper limit directory + slots come to the distance 1 of each other */ + while (up - low > 1) + { + const size_t mid= (low + up) / 2; + mid_rec= + page_dir_slot_get_rec_validate(page, page_dir_get_nth_slot(page, mid)); + if (UNIV_UNLIKELY(!mid_rec)) + return true; + uint16_t cur= std::min(low_fields, up_fields); + int cmp= page_cur_dtuple_cmp(*tuple, mid_rec, index, &cur, comp, leaf); + if (cmp > 0) + low_slot_match: + low= mid, low_fields= cur; + else if (cmp) + up_slot_match: + up= mid, up_fields= cur; + else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE) + goto low_slot_match; + else + goto up_slot_match; + } - low_rec = page_dir_slot_get_rec_validate( - page_dir_get_nth_slot(page, low)); - up_rec = page_dir_slot_get_rec_validate( - page_dir_get_nth_slot(page, up)); - if (UNIV_UNLIKELY(!low_rec || !up_rec)) { -corrupted: - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - return true; - } + const rec_t *up_rec= + page_dir_slot_get_rec_validate(page, page_dir_get_nth_slot(page, up));; + const rec_t *low_rec= + page_dir_slot_get_rec_validate(page, page_dir_get_nth_slot(page, low)); + if (UNIV_UNLIKELY(!low_rec || !up_rec)) + return true; - /* Perform linear search until the upper and lower records come to - distance 1 of each other. */ - - for (;;) { - if (const rec_t* next = page_rec_get_next_const(low_rec)) { - if (next == up_rec) { - break; - } - mid_rec = next; - } else { - goto corrupted; - } - cur_matched_fields = std::min(low_matched_fields, - up_matched_fields); - - offsets = offsets_; - offsets = rec_get_offsets( - mid_rec, index, offsets, n_core, - dtuple_get_n_fields_cmp(tuple), &heap); - - cmp = cmp_dtuple_rec_with_match( - tuple, mid_rec, offsets, &cur_matched_fields); - - if (cmp > 0) { -low_rec_match: - low_rec = mid_rec; - low_matched_fields = cur_matched_fields; - - } else if (cmp) { -up_rec_match: - up_rec = mid_rec; - up_matched_fields = cur_matched_fields; - } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE) { - if (!cmp && !cur_matched_fields) { - /* We got a match, but cur_matched_fields is - 0, it must have REC_INFO_MIN_REC_FLAG */ - ut_ad(rec_get_info_bits(mid_rec, - rec_offs_comp(offsets)) - & REC_INFO_MIN_REC_FLAG); - ut_ad(!page_has_prev(page)); - cur_matched_fields = tuple->n_fields_cmp; - } - - goto low_rec_match; - } else { - - goto up_rec_match; - } - } + /* Perform linear search until the upper and lower records come to + distance 1 of each other. */ - page_cur_position(mode <= PAGE_CUR_GE ? up_rec : low_rec, block, - cursor); + for (;;) + { + mid_rec= comp + ? page_rec_next_get(page, low_rec) + : page_rec_next_get(page, low_rec); + if (!mid_rec) + return true; + if (mid_rec == up_rec) + break; - *iup_matched_fields = up_matched_fields; - *ilow_matched_fields = low_matched_fields; - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } + uint16_t cur= std::min(low_fields, up_fields); + int cmp= page_cur_dtuple_cmp(*tuple, mid_rec, index, &cur, comp, leaf); + if (cmp > 0) + low_rec_match: + low_rec= mid_rec, low_fields= cur; + else if (cmp) + up_rec_match: + up_rec= mid_rec, up_fields= cur; + else if (mode == PAGE_CUR_G || mode == PAGE_CUR_LE) + { + if (cur == 0) + { + /* A match on 0 fields must be due to REC_INFO_MIN_REC_FLAG */ + ut_ad(rec_get_info_bits(mid_rec, comp) & REC_INFO_MIN_REC_FLAG); + ut_ad(!page_has_prev(page)); + ut_ad(!leaf || rec_is_metadata(mid_rec, index)); + cur= tuple->n_fields_cmp; + } + goto low_rec_match; + } + else + goto up_rec_match; + } - return false; + page_cur_position(mode <= PAGE_CUR_GE ? up_rec : low_rec, block, cursor); + *iup_fields= up_fields; + *ilow_fields= low_fields; + return false; } /***********************************************************//** @@ -1033,7 +986,8 @@ static bool page_dir_split_slot(const buf_block_t &block, PAGE_DIR_SLOT_MIN_N_OWNED, "compatibility"); /* Find a record approximately in the middle. */ - const rec_t *rec= page_dir_slot_get_rec_validate(slot + PAGE_DIR_SLOT_SIZE); + const rec_t *rec= page_dir_slot_get_rec_validate(block.page.frame, + slot + PAGE_DIR_SLOT_SIZE); for (ulint i= n_owned / 2; i--; ) { @@ -1066,8 +1020,10 @@ static bool page_dir_split_slot(const buf_block_t &block, mach_write_to_2(slot, rec - block.page.frame); const bool comp= page_is_comp(block.page.frame) != 0; - page_rec_set_n_owned(page_dir_slot_get_rec(slot), half_owned, comp); - page_rec_set_n_owned(page_dir_slot_get_rec(slot - PAGE_DIR_SLOT_SIZE), + page_rec_set_n_owned(page_dir_slot_get_rec(block.page.frame, slot), + half_owned, comp); + page_rec_set_n_owned(page_dir_slot_get_rec(block.page.frame, + slot - PAGE_DIR_SLOT_SIZE), n_owned - half_owned, comp); return false; } @@ -1093,7 +1049,8 @@ static void page_zip_dir_split_slot(buf_block_t *block, ulint s, mtr_t* mtr) /* 1. We loop to find a record approximately in the middle of the records owned by the slot. */ - const rec_t *rec= page_dir_slot_get_rec(slot + PAGE_DIR_SLOT_SIZE); + const rec_t *rec= page_dir_slot_get_rec(block->page.frame, + slot + PAGE_DIR_SLOT_SIZE); /* We do not try to prevent crash on corruption here. For ROW_FORMAT=COMPRESSED pages, the next-record links should @@ -1120,10 +1077,13 @@ static void page_zip_dir_split_slot(buf_block_t *block, ulint s, mtr_t* mtr) /* Log changes to the compressed page header and the dense page directory. */ memcpy_aligned<2>(&block->page.zip.data[n_slots_f], n_slots_p, 2); mach_write_to_2(slot, rec - block->page.frame); - page_rec_set_n_owned(block, page_dir_slot_get_rec(slot), half_owned, + page_rec_set_n_owned(block, + page_dir_slot_get_rec(block->page.frame, slot), + half_owned, true, mtr); page_rec_set_n_owned(block, - page_dir_slot_get_rec(slot - PAGE_DIR_SLOT_SIZE), + page_dir_slot_get_rec(block->page.frame, + slot - PAGE_DIR_SLOT_SIZE), n_owned - half_owned, true, mtr); } @@ -1151,12 +1111,15 @@ static void page_zip_dir_balance_slot(buf_block_t *block, ulint s, mtr_t *mtr) page_dir_slot_t* slot = page_dir_get_nth_slot(block->page.frame, s); rec_t* const up_rec = const_cast - (page_dir_slot_get_rec(slot - PAGE_DIR_SLOT_SIZE)); + (page_dir_slot_get_rec(block->page.frame, + slot - PAGE_DIR_SLOT_SIZE)); rec_t* const slot_rec = const_cast - (page_dir_slot_get_rec(slot)); + (page_dir_slot_get_rec(block->page.frame, + slot)); const ulint up_n_owned = rec_get_n_owned_new(up_rec); - ut_ad(rec_get_n_owned_new(page_dir_slot_get_rec(slot)) + ut_ad(rec_get_n_owned_new(page_dir_slot_get_rec(block->page.frame, + slot)) == PAGE_DIR_SLOT_MIN_N_OWNED - 1); if (up_n_owned <= PAGE_DIR_SLOT_MIN_N_OWNED) { @@ -1220,9 +1183,10 @@ static void page_dir_balance_slot(const buf_block_t &block, ulint s) page_dir_slot_t* slot = page_dir_get_nth_slot(block.page.frame, s); rec_t* const up_rec = const_cast - (page_dir_slot_get_rec(slot - PAGE_DIR_SLOT_SIZE)); + (page_dir_slot_get_rec(block.page.frame, + slot - PAGE_DIR_SLOT_SIZE)); rec_t* const slot_rec = const_cast - (page_dir_slot_get_rec(slot)); + (page_dir_slot_get_rec(block.page.frame, slot)); const ulint up_n_owned = comp ? rec_get_n_owned_new(up_rec) : rec_get_n_owned_old(up_rec); @@ -2508,7 +2472,8 @@ page_cur_delete_rec( left at the next record. */ rec = const_cast - (page_dir_slot_get_rec(cur_dir_slot + PAGE_DIR_SLOT_SIZE)); + (page_dir_slot_get_rec(block->page.frame, + cur_dir_slot + PAGE_DIR_SLOT_SIZE)); /* rec now points to the record of the previous directory slot. Look for the immediate predecessor of current_rec in a loop. */ @@ -2537,7 +2502,8 @@ page_cur_delete_rec( ut_ad(cur_n_owned > 1); rec_t* slot_rec = const_cast - (page_dir_slot_get_rec(cur_dir_slot)); + (page_dir_slot_get_rec(block->page.frame, + cur_dir_slot)); if (UNIV_LIKELY_NULL(block->page.zip.data)) { ut_ad(page_is_comp(block->page.frame));