From 0bcc03a2e6619f7c30b72bb4a148a4f332ab371e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 10 Apr 2025 12:58:10 +0300 Subject: [PATCH 1/2] Recovery cleanup recv_sys_t::report_progress(): Display the largest currently known LSN. recv_scan_log(): Display an error with fewer function calls. Reviewed by: Debarun Banerjee --- storage/innobase/log/log0recv.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 901e3ca3520d6..9a4bac91035cd 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3559,13 +3559,14 @@ void recv_sys_t::report_progress() const } else { + const lsn_t end{std::max(recv_sys.scanned_lsn, recv_sys.file_checkpoint)}; sql_print_information("InnoDB: To recover: LSN " LSN_PF "/" LSN_PF "; %zu pages", - recv_sys.lsn, recv_sys.scanned_lsn, n); + recv_sys.lsn, end, n); service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, "To recover: LSN " LSN_PF "/" LSN_PF "; %zu pages", - recv_sys.lsn, recv_sys.scanned_lsn, n); + recv_sys.lsn, end, n); } } @@ -4106,8 +4107,8 @@ static bool recv_scan_log(bool last_phase) {log_sys.buf + recv_sys.len, size})) { mysql_mutex_unlock(&recv_sys.mutex); - ib::error() << "Failed to read log at " << source_offset - << ": " << err; + sql_print_error("InnoDB: Failed to read log at %" PRIu64 ": %s", + source_offset, ut_strerr(err)); recv_sys.set_corrupt_log(); mysql_mutex_lock(&recv_sys.mutex); } From acd071f599f416ddb4821dec485c4d912844213f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 10 Apr 2025 13:02:17 +0300 Subject: [PATCH 2/2] MDEV-21923: LSN allocation is a bottleneck The parameter innodb_log_spin_wait_delay will be deprecated and ignored, because there is no spin loop anymore. Thanks to commit 685d958e38b825ad9829be311f26729cccf37c46 and commit a635c40648519fd6c3729c9657872a16a0a20821 multiple mtr_t::commit() can concurrently copy their slice of mtr_t::m_log to the shared log_sys.buf. Each writer would allocate their own log sequence number by invoking log_t::append_prepare() while holding a shared log_sys.latch. This function was too heavy, because it would invoke a minimum of 4 atomic read-modify-write operations as well as system calls in the supposedly fast code path. It turns out that with a simpler data structure, instead of having several data fields that needed to be kept consistent with each other, we only need one Atomic_relaxed write_lsn_offset, on which we can operate using fetch_add(), fetch_sub() as well as a single-bit fetch_or(), which reasonably modern compilers (GCC 7, Clang 15 or later) can translate into loop-free code on AMD64. Before anything can be written to the log, log_sys.clear_mmap() must be invoked. log_t::base_lsn: The LSN of the last write_buf() or persist(). This is a rough approximation of log_sys.lsn, which will be removed. log_t::write_lsn_offset: An Atomic_relaxed that buffers updates of write_to_buf and base_lsn. log_t::buf_free, log_t::max_buf_free, log_t::lsn. Remove. Replaced by base_lsn and write_lsn_offset. log_t::buf_size: Always reflects the usable size in append_prepare(). log_t::lsn_lock: Remove. For the memory-mapped log in resize_write(), there will be a resize_wrap_mutex. log_t::get_lsn_approx(): Return a lower bound of get_lsn(). This should be exact unless append_prepare_wait() is pending. log_get_lsn(): A wrapper for log_sys.get_lsn(), which must be invoked while holding an exclusive log_sys.latch. recv_recovery_from_checkpoint_start(): Do not invoke fil_names_clear(); it would seem to be unnecessary. In many places, references to log_sys.get_lsn() are replaced with log_sys.get_flushed_lsn(), which remains a simple std::atomic::load(). Reviewed by: Debarun Banerjee --- extra/mariabackup/xtrabackup.cc | 2 +- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- storage/innobase/buf/buf0buf.cc | 14 +- storage/innobase/buf/buf0dblwr.cc | 5 +- storage/innobase/buf/buf0flu.cc | 43 +-- storage/innobase/handler/ha_innodb.cc | 58 ++-- storage/innobase/include/fil0fil.h | 2 +- storage/innobase/include/log0log.h | 174 ++++++------ storage/innobase/include/mtr0mtr.h | 9 +- storage/innobase/log/log0crypt.cc | 2 +- storage/innobase/log/log0log.cc | 204 +++++++------- storage/innobase/log/log0recv.cc | 30 +- storage/innobase/mtr/mtr0mtr.cc | 265 ++++++------------ storage/innobase/row/row0mysql.cc | 4 +- storage/innobase/srv/srv0mon.cc | 8 +- storage/innobase/srv/srv0srv.cc | 6 +- storage/innobase/srv/srv0start.cc | 23 +- 17 files changed, 382 insertions(+), 469 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 837e82d82ca43..52c5b5341aa87 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2684,7 +2684,7 @@ static bool innodb_init() } recv_sys.lsn= log_sys.next_checkpoint_lsn= - log_sys.get_lsn() - SIZE_OF_FILE_CHECKPOINT; + log_get_lsn() - SIZE_OF_FILE_CHECKPOINT; log_sys.set_latest_format(false); // not encrypted log_hdr_init(); byte *b= &log_hdr_buf[log_t::START_OFFSET]; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 565a4fbf22cbd..5810f23f14e62 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1092,7 +1092,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL VARIABLE_TYPE INT UNSIGNED -VARIABLE_COMMENT Delay between log buffer spin lock polls (0 to use a blocking latch) +VARIABLE_COMMENT Deprecated parameter with no effect NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 6000 NUMERIC_BLOCK_SIZE 0 diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index f7a9110a7ba03..c3da36ede3a46 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -558,16 +558,18 @@ buf_page_is_checksum_valid_crc32( } #ifndef UNIV_INNOCHECKSUM -/** Checks whether the lsn present in the page is lesser than the -peek current lsn. -@param check_lsn lsn to check +/** Check whether a page is newer than the durable LSN. +@param check_lsn whether to check the LSN @param read_buf page frame -@return whether the FIL_PAGE_LSN is invalid */ -static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) +@return whether the FIL_PAGE_LSN is invalid (ahead of the durable LSN) */ +static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) noexcept { if (!check_lsn) return false; - lsn_t current_lsn= log_sys.get_lsn(); + /* A page may not be read before it is written, and it may not be + written before the corresponding log has been durably written. + Hence, we refer to the current durable LSN here */ + lsn_t current_lsn= log_sys.get_flushed_lsn(std::memory_order_relaxed); if (UNIV_UNLIKELY(current_lsn == log_sys.FIRST_LSN) && srv_force_recovery == SRV_FORCE_NO_LOG_REDO) return false; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index faea6d113df88..7a4b9b6172f74 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -365,7 +365,7 @@ void buf_dblwr_t::recover() noexcept ut_ad(log_sys.last_checkpoint_lsn); if (!is_created()) return; - const lsn_t max_lsn{log_sys.get_lsn()}; + const lsn_t max_lsn{log_sys.get_flushed_lsn(std::memory_order_relaxed)}; ut_ad(recv_sys.scanned_lsn == max_lsn); ut_ad(recv_sys.scanned_lsn >= recv_sys.lsn); @@ -780,6 +780,9 @@ void buf_dblwr_t::flush_buffered_writes_completed(const IORequest &request) ut_ad(lsn); ut_ad(lsn >= bpage->oldest_modification()); log_write_up_to(lsn, true); + ut_ad(!e.request.node->space->full_crc32() || + !buf_page_is_corrupted(true, static_cast(frame), + e.request.node->space->flags)); e.request.node->space->io(e.request, bpage->physical_offset(), e_size, frame, bpage); } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 061af1290cf46..13eabd92f3b07 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -694,7 +694,6 @@ static byte *buf_page_encrypt(fil_space_t *space, buf_page_t *bpage, byte *s, { static_assert(FIL_PAGE_FCRC32_CHECKSUM == 4, "alignment"); mach_write_to_4(tmp + len - 4, my_crc32c(0, tmp, len - 4)); - ut_ad(!buf_page_is_corrupted(true, tmp, space->flags)); } d= tmp; @@ -854,6 +853,8 @@ bool buf_page_t::flush(fil_space_t *space) noexcept if (!space->is_temporary() && !space->is_being_imported() && lsn > log_sys.get_flushed_lsn()) log_write_up_to(lsn, true); + ut_ad(space->is_temporary() || !space->full_crc32() || + !buf_page_is_corrupted(true, write_frame, space->flags)); space->io(IORequest{type, this, slot}, physical_offset(), size, write_frame, this); } @@ -1773,8 +1774,9 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept { ut_ad(!srv_read_only_mode); ut_ad(end_lsn >= next_checkpoint_lsn); - ut_ad(end_lsn <= get_lsn()); - ut_ad(end_lsn + SIZE_OF_FILE_CHECKPOINT <= get_lsn() || + ut_d(const lsn_t current_lsn{get_lsn()}); + ut_ad(end_lsn <= current_lsn); + ut_ad(end_lsn + SIZE_OF_FILE_CHECKPOINT <= current_lsn || srv_shutdown_state > SRV_SHUTDOWN_INITIATED); DBUG_PRINT("ib_log", @@ -1903,7 +1905,8 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept ut_ad(!is_opened()); my_munmap(buf, file_size); buf= resize_buf; - set_buf_free(START_OFFSET + (get_lsn() - resizing)); + buf_size= unsigned(std::min(resize_target - START_OFFSET, + buf_size_max)); } else #endif @@ -2038,9 +2041,9 @@ static bool log_checkpoint() noexcept } /** Make a checkpoint. */ -ATTRIBUTE_COLD void log_make_checkpoint() +ATTRIBUTE_COLD void log_make_checkpoint() noexcept { - buf_flush_wait_flushed(log_sys.get_lsn(std::memory_order_acquire)); + buf_flush_wait_flushed(log_get_lsn()); while (!log_checkpoint()); } @@ -2048,8 +2051,6 @@ ATTRIBUTE_COLD void log_make_checkpoint() NOTE: The calling thread is not allowed to hold any buffer page latches! */ static void buf_flush_wait(lsn_t lsn) noexcept { - ut_ad(lsn <= log_sys.get_lsn()); - lsn_t oldest_lsn; while ((oldest_lsn= buf_pool.get_oldest_modification(lsn)) < lsn) @@ -2258,13 +2259,13 @@ static void buf_flush_sync_for_checkpoint(lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); } -/** Check if the adpative flushing threshold is recommended based on +/** Check if the adaptive flushing threshold is recommended based on redo log capacity filled threshold. @param oldest_lsn buf_pool.get_oldest_modification() @return true if adaptive flushing is recommended. */ static bool af_needed_for_redo(lsn_t oldest_lsn) noexcept { - lsn_t age= (log_sys.get_lsn() - oldest_lsn); + lsn_t age= log_sys.get_lsn_approx() - oldest_lsn; lsn_t af_lwm= static_cast(srv_adaptive_flushing_lwm * static_cast(log_sys.log_capacity) / 100); @@ -2324,7 +2325,7 @@ static ulint page_cleaner_flush_pages_recommendation(ulint last_pages_in, lsn_t lsn_rate; ulint n_pages = 0; - const lsn_t cur_lsn = log_sys.get_lsn(); + const lsn_t cur_lsn = log_sys.get_lsn_approx(); ut_ad(oldest_lsn <= cur_lsn); ulint pct_for_lsn = af_get_pct_for_lsn(cur_lsn - oldest_lsn); time_t curr_time = time(nullptr); @@ -2796,7 +2797,7 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() noexcept NOTE: The calling thread is not allowed to hold any buffer page latches! */ void buf_flush_sync_batch(lsn_t lsn) noexcept { - lsn= std::max(lsn, log_sys.get_lsn()); + lsn= std::max(lsn, log_get_lsn()); mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_wait(lsn); mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -2815,20 +2816,26 @@ void buf_flush_sync() noexcept thd_wait_begin(nullptr, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - for (;;) + log_sys.latch.wr_lock(SRW_LOCK_CALL); + + for (lsn_t lsn= log_sys.get_lsn();;) { - const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_wait(lsn); /* Wait for the page cleaner to be idle (for log resizing at startup) */ while (buf_flush_sync_lsn) my_cond_wait(&buf_pool.done_flush_list, &buf_pool.flush_list_mutex.m_mutex); - if (lsn == log_sys.get_lsn()) + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t new_lsn= log_sys.get_lsn(); + if (lsn == new_lsn) break; + lsn= new_lsn; } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); + log_sys.latch.wr_unlock(); tpool::tpool_wait_end(); thd_wait_end(nullptr); } @@ -2848,7 +2855,7 @@ ATTRIBUTE_COLD void buf_pool_t::print_flush_info() const noexcept "-------------------", lru_size, free_size, dirty_size, dirty_pct); - lsn_t lsn= log_sys.get_lsn(); + lsn_t lsn= log_get_lsn(); lsn_t clsn= log_sys.last_checkpoint_lsn; sql_print_information("InnoDB: LSN flush parameters\n" "-------------------\n" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index fc44c404d56e4..a2b75f43291c7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -243,11 +243,11 @@ static void innodb_max_purge_lag_wait_update(THD *thd, st_mysql_sys_var *, if (thd_kill_level(thd)) break; /* Adjust for purge_coordinator_state::refresh() */ - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); const lsn_t last= log_sys.last_checkpoint_lsn, max_age= log_sys.max_checkpoint_age; - log_sys.latch.rd_unlock(); const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); if ((lsn - last) / 4 >= max_age / 5) buf_flush_ahead(last + max_age / 5, false); purge_sys.wake_if_not_active(); @@ -1155,7 +1155,7 @@ innobase_rollback_to_savepoint_can_release_mdl( be rolled back to savepoint */ /** Request notification of log writes */ -static void innodb_log_flush_request(void *cookie); +static void innodb_log_flush_request(void *cookie) noexcept; /** Requests for log flushes */ struct log_flush_request @@ -4749,11 +4749,13 @@ void log_flush_notify(lsn_t flush_lsn) We put the request in a queue, so that we can notify upper layer about checkpoint complete when we have flushed the redo log. If we have already flushed all relevant redo log, we notify immediately.*/ -static void innodb_log_flush_request(void *cookie) +static void innodb_log_flush_request(void *cookie) noexcept { + log_sys.latch.wr_lock(SRW_LOCK_CALL); lsn_t flush_lsn= log_sys.get_flushed_lsn(); /* Load lsn relaxed after flush_lsn was loaded from the same cache line */ const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); if (flush_lsn >= lsn) /* All log is already persistent. */; @@ -18440,11 +18442,16 @@ checkpoint_now_set(THD* thd, st_mysql_sys_var*, void*, const void *save) const auto size= log_sys.is_encrypted() ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT; mysql_mutex_unlock(&LOCK_global_system_variables); - lsn_t lsn; - while (!thd_kill_level(thd) && - log_sys.last_checkpoint_lsn.load(std::memory_order_acquire) + size < - (lsn= log_sys.get_lsn(std::memory_order_acquire))) + while (!thd_kill_level(thd)) + { + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t cp= log_sys.last_checkpoint_lsn.load(std::memory_order_relaxed), + lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + if (cp + size >= lsn) + break; log_make_checkpoint(); + } mysql_mutex_lock(&LOCK_global_system_variables); } @@ -18664,35 +18671,23 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, mysql_mutex_unlock(&buf_pool.flush_list_mutex); if (!resizing || !log_sys.resize_running(thd)) break; - if (resizing > log_sys.get_lsn()) + log_sys.latch.wr_lock(SRW_LOCK_CALL); + while (resizing > log_sys.get_lsn()) { ut_ad(!log_sys.is_mmap()); /* The server is almost idle. Write dummy FILE_CHECKPOINT records to ensure that the log resizing will complete. */ - log_sys.latch.wr_lock(SRW_LOCK_CALL); - while (resizing > log_sys.get_lsn()) - { - mtr_t mtr; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); + mtr_t mtr; + mtr.start(); + mtr.commit_files(log_sys.last_checkpoint_lsn); } + log_sys.latch.wr_unlock(); } } } mysql_mutex_lock(&LOCK_global_system_variables); } -static void innodb_log_spin_wait_delay_update(THD *, st_mysql_sys_var*, - void *, const void *save) -{ - log_sys.latch.wr_lock(SRW_LOCK_CALL); - mtr_t::spin_wait_delay= *static_cast(save); - mtr_t::finisher_update(); - log_sys.latch.wr_unlock(); -} - /** Update innodb_status_output or innodb_status_output_locks, which control InnoDB "status monitor" output to the error log. @param[out] var current value @@ -19551,11 +19546,12 @@ static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size, nullptr, innodb_log_file_size_update, 96 << 20, 4 << 20, std::numeric_limits::max(), 4096); -static MYSQL_SYSVAR_UINT(log_spin_wait_delay, mtr_t::spin_wait_delay, - PLUGIN_VAR_OPCMDARG, - "Delay between log buffer spin lock polls (0 to use a blocking latch)", - nullptr, innodb_log_spin_wait_delay_update, - 0, 0, 6000, 0); +static uint innodb_log_spin_wait_delay; + +static MYSQL_SYSVAR_UINT(log_spin_wait_delay, innodb_log_spin_wait_delay, + PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_DEPRECATED, + "Deprecated parameter with no effect", + nullptr, nullptr, 0, 0, 6000, 0); static MYSQL_SYSVAR_UINT(old_blocks_pct, innobase_old_blocks_pct, PLUGIN_VAR_RQCMDARG, diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index c3514c96cf7fa..689fbdbdbc7a3 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -351,7 +351,7 @@ struct fil_space_t final /** fil_system.spaces chain node */ fil_space_t *hash= nullptr; /** log_sys.get_lsn() of the most recent fil_names_write_if_was_clean(). - Reset to 0 by fil_names_clear(). Protected by log_sys.mutex. + Reset to 0 by fil_names_clear(). Protected by log_sys.latch_have_wr(). If and only if this is nonzero, the tablespace will be in named_spaces. */ lsn_t max_lsn= 0; /** base node for the chain of data files; multiple entries are diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 33df9d39ba6bb..55ffbc0df996a 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -64,20 +64,19 @@ void log_write_up_to(lsn_t lsn, bool durable, /** Write to the log file up to the last log entry. @param durable whether to wait for a durable write to complete */ -void log_buffer_flush_to_disk(bool durable= true); - +void log_buffer_flush_to_disk(bool durable= true) noexcept; /** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */ -ATTRIBUTE_COLD void log_write_and_flush_prepare(); +ATTRIBUTE_COLD void log_write_and_flush_prepare() noexcept; /** Durably write the log up to log_sys.get_lsn(). */ -ATTRIBUTE_COLD void log_write_and_flush(); +ATTRIBUTE_COLD void log_write_and_flush() noexcept; /** Make a checkpoint */ -ATTRIBUTE_COLD void log_make_checkpoint(); +ATTRIBUTE_COLD void log_make_checkpoint() noexcept; /** Make a checkpoint at the latest lsn on shutdown. */ -ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown(); +ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() noexcept; /******************************************************//** Prints info of the log. */ @@ -167,40 +166,35 @@ struct log_t static constexpr lsn_t FIRST_LSN= START_OFFSET; private: - /** the lock bit in buf_free */ - static constexpr size_t buf_free_LOCK= ~(~size_t{0} >> 1); + /** the least significant bit of the write_to_buf buffer */ + static constexpr size_t WRITE_TO_BUF_SHIFT{34}; + /** write_lsn_offset component for incrementing write_to_buf */ + static constexpr uint64_t WRITE_TO_BUF{1ULL << WRITE_TO_BUF_SHIFT}; + /** write_lsn_offset flag to indicate that append_prepare_wait() is active */ + static constexpr uint64_t WRITE_BACKOFF{1ULL << 33}; + + /** The current log sequence number, relative to base_lsn, and flags; + may be modified while latch_have_any() */ alignas(CPU_LEVEL1_DCACHE_LINESIZE) - /** first free offset within buf used; - the most significant bit is set by lock_lsn() to protect this field - as well as write_to_buf, waits */ - std::atomic buf_free; -public: - /** number of write requests (to buf); protected by lock_lsn() or lsn_lock */ - size_t write_to_buf; - /** log record buffer, written to by mtr_t::commit() */ - byte *buf; -private: - /** The log sequence number of the last change of durable InnoDB files; - protected by lock_lsn() or lsn_lock or latch.wr_lock() */ - std::atomic lsn; + Atomic_relaxed write_lsn_offset; + /** the LSN of the last write_buf() or persist(); protected by latch */ + std::atomic base_lsn; /** the first guaranteed-durable log sequence number */ std::atomic flushed_to_disk_lsn; public: - /** number of append_prepare_wait(); protected by lock_lsn() or lsn_lock */ - size_t waits; - /** innodb_log_buffer_size (size of buf,flush_buf if !is_mmap(), in bytes) */ + /** innodb_log_buffer_size (usable append_prepare() size in bytes) */ unsigned buf_size; /** log file size in bytes, including the header */ lsn_t file_size; #ifdef LOG_LATCH_DEBUG typedef srw_lock_debug log_rwlock; - typedef srw_mutex log_lsn_lock; bool latch_have_wr() const { return latch.have_wr(); } bool latch_have_rd() const { return latch.have_rd(); } bool latch_have_any() const { return latch.have_any(); } #else + typedef srw_lock log_rwlock; # ifndef UNIV_DEBUG # elif defined SUX_LOCK_GENERIC bool latch_have_wr() const { return true; } @@ -211,23 +205,23 @@ struct log_t bool latch_have_rd() const { return latch.is_locked(); } bool latch_have_any() const { return latch.is_locked(); } # endif -# ifdef __aarch64__ - /* On ARM, we spin more */ - typedef srw_spin_lock log_rwlock; - typedef pthread_mutex_wrapper log_lsn_lock; -# else - typedef srw_lock log_rwlock; - typedef srw_mutex log_lsn_lock; -# endif #endif - /** exclusive latch for checkpoint, shared for mtr_t::commit() to buf */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) log_rwlock latch; + /** latch_have_wr() for checkpoint, latch_have_any() for append_prepare() */ + log_rwlock latch; + + /** log record buffer, written to by mtr_t::commit() */ + alignas(CPU_LEVEL1_DCACHE_LINESIZE) byte *buf; + + /** number of write requests to buf, + excluding (write_lsn_offset & WRITE_TO_BUF); + protected by latch.wr_lock() */ + size_t write_to_buf; /** number of writes from buf or flush_buf to log; protected by latch.wr_lock() */ - ulint write_to_log; + size_t write_to_log; - /** Last written LSN */ + /** Last written LSN; protected by latch */ lsn_t write_lsn; /** Buffer for writing data to ib_logfile0, or nullptr if is_mmap(). @@ -241,8 +235,6 @@ struct log_t Atomic_relaxed checkpoint_pending; /** next checkpoint number (protected by latch.wr_lock()) */ byte next_checkpoint_no; - /** recommended maximum buf_free size, after which the buffer is flushed */ - unsigned max_buf_free; /** Log sequence number when a log file overwrite (broken crash recovery) was noticed. Protected by latch.wr_lock(). */ lsn_t overwrite_warned; @@ -266,12 +258,6 @@ struct log_t /** Buffer for writing to resize_log; @see flush_buf */ byte *resize_flush_buf; - /** Special implementation of lock_lsn() for IA-32 and AMD64 */ - void lsn_lock_bts() noexcept; - /** Acquire a lock for updating buf_free and related fields. - @return the value of buf_free */ - size_t lock_lsn() noexcept; - /** log sequence number when log resizing was initiated; 0 if the log is not being resized, 1 if resize_start() is in progress */ std::atomic resize_lsn; @@ -327,34 +313,24 @@ struct log_t private: /** the thread that initiated resize_lsn() */ Atomic_relaxed resize_initiator; - /** A lock when the spin-only lock_lsn() is not being used */ - log_lsn_lock lsn_lock; +#ifdef HAVE_PMEM + /** mutex protecting wrap-around in resize_write() */ + srw_mutex resize_wrap_mutex; +#endif public: + /** number of long append_prepare_wait(); protected by latch_have_wr() */ + size_t waits; - bool is_initialised() const noexcept { return max_buf_free != 0; } - - /** whether there is capacity in the log buffer */ - bool buf_free_ok() const noexcept - { - ut_ad(!is_mmap()); - return (buf_free.load(std::memory_order_relaxed) & ~buf_free_LOCK) < - max_buf_free; - } - + bool is_initialised() const noexcept + { return base_lsn.load(std::memory_order_relaxed) != 0; } inline void set_recovered() noexcept; - void set_buf_free(size_t f) noexcept - { ut_ad(f < buf_free_LOCK); buf_free.store(f, std::memory_order_relaxed); } - bool is_mmap() const noexcept { return !flush_buf; } /** @return whether a handle to the log is open; is_mmap() && !is_opened() holds for PMEM */ bool is_opened() const noexcept { return log.is_opened(); } - /** @return target write LSN to react on !buf_free_ok() */ - inline lsn_t get_write_target() const; - /** @return LSN at which log resizing was started and is still in progress @retval 0 if no log resizing is in progress @retval 1 if resize_start() is in progress */ @@ -407,53 +383,64 @@ struct log_t { return resize_buf + resize_target; } /** Initialise the redo log subsystem. */ - void create(); + void create() noexcept; /** Attach a log file. @return whether the memory allocation succeeded */ - bool attach(log_file_t file, os_offset_t size); + bool attach(log_file_t file, os_offset_t size) noexcept; /** Disable memory-mapped access (update log_mmap) */ - void clear_mmap(); - void close_file(bool really_close= true); + void clear_mmap() noexcept; + void close_file(bool really_close= true) noexcept; #if defined __linux__ || defined _WIN32 /** Try to enable or disable file system caching (update log_buffered) */ - void set_buffered(bool buffered); + void set_buffered(bool buffered) noexcept; #endif /** Calculate the checkpoint safety margins. */ - static void set_capacity(); + static void set_capacity() noexcept; /** Write a log file header. @param buf log header buffer @param lsn log sequence number corresponding to log_sys.START_OFFSET @param encrypted whether the log is encrypted */ - static void header_write(byte *buf, lsn_t lsn, bool encrypted); + static void header_write(byte *buf, lsn_t lsn, bool encrypted) noexcept; + + /** @return a lower bound estimate of get_lsn(), + using acquire-release ordering with write_buf() or persist(); + this is exact unless append_prepare_wait() is pending */ + lsn_t get_lsn_approx() const noexcept + { + /* acquire-release ordering with write_buf() and persist() */ + lsn_t lsn= base_lsn.load(std::memory_order_acquire); + lsn += write_lsn_offset.load(std::memory_order_relaxed) & + (WRITE_BACKOFF - 1); + return lsn; + } - lsn_t get_lsn(std::memory_order order= std::memory_order_relaxed) const - { return lsn.load(order); } + /** @return the current log sequence number (logical time stamp) */ + lsn_t get_lsn() const noexcept + { + ut_ad(latch_have_wr()); + return base_lsn.load(std::memory_order_relaxed) + + (write_lsn_offset & (WRITE_BACKOFF - 1)); + } lsn_t get_flushed_lsn(std::memory_order order= std::memory_order_acquire) const noexcept { return flushed_to_disk_lsn.load(order); } /** Initialize the LSN on initial log file creation. */ - lsn_t init_lsn() noexcept - { - latch.wr_lock(SRW_LOCK_CALL); - const lsn_t lsn{get_lsn()}; - flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); - write_lsn= lsn; - latch.wr_unlock(); - return lsn; - } + inline lsn_t init_lsn() noexcept; void set_recovered_lsn(lsn_t lsn) noexcept { ut_ad(latch_have_wr()); - write_lsn= lsn; - this->lsn.store(lsn, std::memory_order_relaxed); + uint64_t lsn_offset= ((write_size - 1) & (lsn - first_lsn)); + write_lsn_offset= lsn_offset; + base_lsn.store(lsn - lsn_offset, std::memory_order_relaxed); flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn= lsn; } #ifdef HAVE_PMEM @@ -491,22 +478,16 @@ struct log_t void writer_update(bool resizing) noexcept; /** Wait in append_prepare() for buffer to become available - @tparam spin whether to use the spin-only lock_lsn() - @param b the value of buf_free - @param ex whether log_sys.latch is exclusively locked - @param lsn log sequence number to write up to - @return the new value of buf_free */ - template - ATTRIBUTE_COLD size_t append_prepare_wait(size_t b, bool ex, lsn_t lsn) - noexcept; + @param late whether the WRITE_BACKOFF flag had already been set + @param ex whether log_sys.latch is exclusively locked */ + ATTRIBUTE_COLD void append_prepare_wait(bool late, bool ex) noexcept; public: /** Reserve space in the log buffer for appending data. - @tparam spin whether to use the spin-only lock_lsn() @tparam mmap log_sys.is_mmap() @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ - template + template std::pair append_prepare(size_t size, bool ex) noexcept; /** Append a string of bytes to the redo log. @@ -577,7 +558,10 @@ extern log_t log_sys; /** Wait for a log checkpoint if needed. NOTE that this function may only be called while not holding any synchronization objects except dict_sys.latch. */ -void log_free_check(); +void log_free_check() noexcept; + +/** @return the current log sequence number (may be stale) */ +lsn_t log_get_lsn() noexcept; /** Release the latches that protect log resizing. */ -void log_resize_release(); +void log_resize_release() noexcept; diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index d6dd203cb1065..1ba52a00c7e61 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -700,19 +700,19 @@ struct mtr_t { @param mtr mini-transaction @param lsns {start_lsn,flush_ahead} */ template - static void commit_log(mtr_t *mtr, std::pair lsns); + static void commit_log(mtr_t *mtr, std::pair lsns) + noexcept; /** Append the redo log records to the redo log buffer. @return {start_lsn,flush_ahead} */ std::pair do_write(); /** Append the redo log records to the redo log buffer. - @tparam spin whether to use the spin-only log_sys.lock_lsn() @tparam mmap log_sys.is_mmap() @param mtr mini-transaction @param len number of bytes to write @return {start_lsn,flush_ahead} */ - template static + template static std::pair finish_writer(mtr_t *mtr, size_t len); /** The applicable variant of commit_log() */ @@ -723,9 +723,6 @@ struct mtr_t { std::pair finish_write(size_t len) { return finisher(this, len); } public: - /** Poll interval in log_sys.lock_lsn(); 0 to use log_sys.lsn_lock. - Protected by LOCK_global_system_variables and log_sys.latch. */ - static unsigned spin_wait_delay; /** Update finisher when spin_wait_delay is changing to or from 0. */ static void finisher_update(); private: diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index 8a7714101ba14..92eaf4ce0341f 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -566,7 +566,7 @@ ATTRIBUTE_NOINLINE void mtr_t::encrypt() alignas(8) byte iv[MY_AES_BLOCK_SIZE]; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); ut_ad(m_commit_lsn); byte *tmp= static_cast(alloca(srv_page_size)), *t= tmp; byte *dst= static_cast(alloca(srv_page_size)); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 268871cc5afad..22994e389c9c4 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -68,7 +68,7 @@ log_t log_sys; #define LOG_BUF_FLUSH_MARGIN ((4 * 4096) /* cf. log_t::append_prepare() */ \ + (4U << srv_page_size_shift)) -void log_t::set_capacity() +void log_t::set_capacity() noexcept { ut_ad(log_sys.latch_have_wr()); /* Margin for the free space in the smallest log, before a new query @@ -87,13 +87,15 @@ void log_t::set_capacity() log_sys.max_checkpoint_age = margin; } -void log_t::create() +void log_t::create() noexcept { ut_ad(this == &log_sys); ut_ad(!is_initialised()); + latch.SRW_LOCK_INIT(log_latch_key); + write_lsn_offset= 0; /* LSN 0 and 1 are reserved; @see buf_page_t::oldest_modification_ */ - lsn.store(FIRST_LSN, std::memory_order_relaxed); + base_lsn.store(FIRST_LSN, std::memory_order_relaxed); flushed_to_disk_lsn.store(FIRST_LSN, std::memory_order_relaxed); need_checkpoint.store(true, std::memory_order_relaxed); write_lsn= FIRST_LSN; @@ -102,10 +104,10 @@ void log_t::create() ut_ad(!buf); ut_ad(!flush_buf); ut_ad(!writer); - max_buf_free= 1; - latch.SRW_LOCK_INIT(log_latch_key); - lsn_lock.init(); +#ifdef HAVE_PMEM + resize_wrap_mutex.init(); +#endif last_checkpoint_lsn= FIRST_LSN; log_capacity= 0; @@ -114,8 +116,6 @@ void log_t::create() next_checkpoint_lsn= 0; checkpoint_pending= false; - set_buf_free(0); - ut_ad(is_initialised()); } @@ -306,7 +306,7 @@ static void *log_mmap(os_file_t file, #if defined __linux__ || defined _WIN32 /** Display a message about opening the log */ -ATTRIBUTE_COLD static void log_file_message() +ATTRIBUTE_COLD static void log_file_message() noexcept { sql_print_information("InnoDB: %s (block size=%u bytes)", log_sys.log_mmap @@ -320,10 +320,10 @@ ATTRIBUTE_COLD static void log_file_message() log_sys.write_size); } #else -static inline void log_file_message() {} +static inline void log_file_message() noexcept {} #endif -bool log_t::attach(log_file_t file, os_offset_t size) +bool log_t::attach(log_file_t file, os_offset_t size) noexcept { log= file; ut_ad(!size || size >= START_OFFSET + SIZE_OF_FILE_CHECKPOINT); @@ -352,7 +352,6 @@ bool log_t::attach(log_file_t file, os_offset_t size) } # endif buf= static_cast(ptr); - max_buf_free= 1; writer_update(false); # ifdef HAVE_PMEM if (is_pmem) @@ -366,7 +365,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) if (!buf) { alloc_fail: - max_buf_free= 0; + base_lsn.store(0, std::memory_order_relaxed); sql_print_error("InnoDB: Cannot allocate memory;" " too large innodb_log_buffer_size?"); return false; @@ -394,7 +393,6 @@ bool log_t::attach(log_file_t file, os_offset_t size) TRASH_ALLOC(buf, buf_size); TRASH_ALLOC(flush_buf, buf_size); - max_buf_free= buf_size / LOG_BUF_FLUSH_RATIO - LOG_BUF_FLUSH_MARGIN; writer_update(false); memset_aligned<512>(checkpoint_buf, 0, write_size); @@ -407,7 +405,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) @param buf log header buffer @param lsn log sequence number corresponding to log_sys.START_OFFSET @param encrypted whether the log is encrypted */ -void log_t::header_write(byte *buf, lsn_t lsn, bool encrypted) +void log_t::header_write(byte *buf, lsn_t lsn, bool encrypted) noexcept { mach_write_to_4(my_assume_aligned<4>(buf) + LOG_HEADER_FORMAT, log_sys.FORMAT_10_8); @@ -436,8 +434,9 @@ void log_t::create(lsn_t lsn) noexcept ut_ad(is_latest()); ut_ad(this == &log_sys); - this->lsn.store(lsn, std::memory_order_relaxed); - this->flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn_offset= 0; + base_lsn.store(lsn, std::memory_order_relaxed); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); first_lsn= lsn; write_lsn= lsn; @@ -452,14 +451,13 @@ void log_t::create(lsn_t lsn) noexcept mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); memset_aligned<4096>(buf, 0, 4096); log_sys.header_write(buf, lsn, is_encrypted()); - set_buf_free(START_OFFSET); pmem_persist(buf, 512); + buf_size= unsigned(std::min(capacity(), buf_size_max)); } else #endif { ut_ad(!is_mmap()); - set_buf_free(0); memset_aligned<4096>(flush_buf, 0, buf_size); memset_aligned<4096>(buf, 0, buf_size); log_sys.header_write(buf, lsn, is_encrypted()); @@ -468,12 +466,12 @@ void log_t::create(lsn_t lsn) noexcept } } -ATTRIBUTE_COLD static void log_close_failed(dberr_t err) +ATTRIBUTE_COLD static void log_close_failed(dberr_t err) noexcept { ib::fatal() << "closing ib_logfile0 failed: " << err; } -void log_t::close_file(bool really_close) +void log_t::close_file(bool really_close) noexcept { if (is_mmap()) { @@ -508,16 +506,25 @@ void log_t::close_file(bool really_close) log_close_failed(err); } +/** @return the current log sequence number (may be stale) */ +lsn_t log_get_lsn() noexcept +{ + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + return lsn; +} + /** Acquire all latches that protect the log. */ -static void log_resize_acquire() +static void log_resize_acquire() noexcept { #ifdef HAVE_PMEM if (!log_sys.is_mmap()) #endif { - while (flush_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); - while (write_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (write_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); } @@ -525,7 +532,7 @@ static void log_resize_acquire() } /** Release the latches that protect the log. */ -void log_resize_release() +void log_resize_release() noexcept { log_sys.latch.wr_unlock(); @@ -542,7 +549,7 @@ void log_resize_release() #if defined __linux__ || defined _WIN32 /** Try to enable or disable file system caching (update log_buffered) */ -void log_t::set_buffered(bool buffered) +void log_t::set_buffered(bool buffered) noexcept { if (!log_maybe_unbuffered || #ifdef HAVE_PMEM @@ -887,9 +894,7 @@ void log_t::persist(lsn_t lsn) noexcept ut_ad(!is_opened()); ut_ad(!write_lock.is_owner()); ut_ad(!flush_lock.is_owner()); -#ifdef LOG_LATCH_DEBUG - ut_ad(latch_have_any()); -#endif + ut_ad(latch_have_wr()); lsn_t old= flushed_to_disk_lsn.load(std::memory_order_relaxed); @@ -907,26 +912,26 @@ void log_t::persist(lsn_t lsn) noexcept else pmem_persist(buf + start, end - start); - old= flushed_to_disk_lsn.load(std::memory_order_relaxed); - - if (old < lsn) - { - while (!flushed_to_disk_lsn.compare_exchange_weak - (old, lsn, std::memory_order_release, std::memory_order_relaxed)) - if (old >= lsn) - break; - - log_flush_notify(lsn); - DBUG_EXECUTE_IF("crash_after_log_write_upto", DBUG_SUICIDE();); - } + uint64_t offset{write_lsn_offset}; + const lsn_t new_base_lsn= base_lsn.load(std::memory_order_relaxed) + + (offset & (WRITE_BACKOFF - 1)); + ut_ad(new_base_lsn >= lsn); + write_to_buf+= size_t(offset >> WRITE_TO_BUF_SHIFT); + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(0, std::memory_order_relaxed); + base_lsn.store(new_base_lsn, std::memory_order_release); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + log_flush_notify(lsn); + DBUG_EXECUTE_IF("crash_after_log_write_upto", DBUG_SUICIDE();); } ATTRIBUTE_NOINLINE static void log_write_persist(lsn_t lsn) noexcept { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); log_sys.persist(lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); } #endif @@ -977,7 +982,7 @@ lsn_t log_t::write_buf() noexcept ut_ad(resizing == RETAIN_LATCH || (resizing == RESIZING) == (resize_in_progress() > 1)); - const lsn_t lsn{get_lsn(std::memory_order_relaxed)}; + const lsn_t lsn{get_lsn()}; if (write_lsn >= lsn) { @@ -993,7 +998,8 @@ lsn_t log_t::write_buf() noexcept ut_ad(write_lsn >= get_flushed_lsn()); const size_t write_size_1{write_size - 1}; ut_ad(ut_is_2pow(write_size)); - size_t length{buf_free.load(std::memory_order_relaxed)}; + lsn_t base= base_lsn.load(std::memory_order_relaxed); + size_t length{size_t(lsn - base)}; lsn_t offset{calc_lsn_offset(write_lsn)}; ut_ad(length >= (offset & write_size_1)); ut_ad(write_size_1 >= 511); @@ -1015,14 +1021,8 @@ lsn_t log_t::write_buf() noexcept { ut_ad(!((length ^ (size_t(lsn) - size_t(first_lsn))) & write_size_1)); /* Keep filling the same buffer until we have more than one block. */ -#if 0 /* TODO: Pad the last log block with dummy records. */ - buf_free= log_pad(lsn, (write_size_1 + 1) - length, - buf + length, flush_buf); - ... /* TODO: Update the LSN and adjust other code. */ -#else MEM_MAKE_DEFINED(buf + length, (write_size_1 + 1) - length); buf[length]= 0; /* ensure that recovery catches EOF */ -#endif if (UNIV_LIKELY_NULL(re_write_buf)) { MEM_MAKE_DEFINED(re_write_buf + length, (write_size_1 + 1) - length); @@ -1033,8 +1033,13 @@ lsn_t log_t::write_buf() noexcept else { const size_t new_buf_free{length & write_size_1}; + base+= length & ~write_size_1; ut_ad(new_buf_free == ((lsn - first_lsn) & write_size_1)); - buf_free.store(new_buf_free, std::memory_order_relaxed); + write_to_buf+= size_t(write_lsn_offset >> WRITE_TO_BUF_SHIFT); + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(new_buf_free, std::memory_order_relaxed); + base_lsn.store(base, std::memory_order_release); if (new_buf_free) { @@ -1063,7 +1068,9 @@ lsn_t log_t::write_buf() noexcept std::swap(resize_buf, resize_flush_buf); } + ut_ad(base + (write_lsn_offset & (WRITE_TO_BUF - 1)) == lsn); write_to_log++; + if (resizing != RETAIN_LATCH) latch.wr_unlock(); @@ -1107,7 +1114,7 @@ bool log_t::flush(lsn_t lsn) noexcept @retval 0 if there are no pending callbacks on flush_lock or there is another group commit lead. */ -static lsn_t log_flush(lsn_t lsn) +static lsn_t log_flush(lsn_t lsn) noexcept { ut_ad(!log_sys.is_mmap()); ut_a(log_sys.flush(lsn)); @@ -1126,7 +1133,7 @@ wait and check if an already running write is covering the request. void log_write_up_to(lsn_t lsn, bool durable, const completion_callback *callback) noexcept { - ut_ad(!srv_read_only_mode || log_sys.buf_free_ok()); + ut_ad(!srv_read_only_mode); ut_ad(lsn != LSN_MAX); ut_ad(lsn != 0); ut_ad(!log_sys.is_mmap() || !callback || durable); @@ -1139,8 +1146,6 @@ void log_write_up_to(lsn_t lsn, bool durable, return; } - ut_ad(lsn <= log_sys.get_lsn()); - #ifdef HAVE_PMEM if (log_sys.is_mmap()) { @@ -1157,10 +1162,10 @@ void log_write_up_to(lsn_t lsn, bool durable, if (flush_lock.acquire(lsn, callback) != group_commit_lock::ACQUIRED) return; /* Promise to other concurrent flush_lock.acquire() that we - will durable at least up to the current LSN. The LSN may still - advance until we acquire log_sys.latch below. */ - lsn= log_sys.get_lsn(); - flush_lock.set_pending(lsn); + will be durable at least up to the current LSN. The LSN may still + advance when we acquire log_sys.latch below. */ + if (lsn > log_sys.get_flushed_lsn()) + flush_lock.set_pending(lsn); } lsn_t pending_write_lsn= 0, pending_flush_lsn= 0; @@ -1206,33 +1211,41 @@ void log_t::writer_update(bool resizing) noexcept /** Write to the log file up to the last log entry. @param durable whether to wait for a durable write to complete */ -void log_buffer_flush_to_disk(bool durable) +void log_buffer_flush_to_disk(bool durable) noexcept { - log_write_up_to(log_sys.get_lsn(std::memory_order_acquire), durable); + log_write_up_to(log_get_lsn(), durable); } /** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */ -ATTRIBUTE_COLD void log_write_and_flush_prepare() +ATTRIBUTE_COLD void log_write_and_flush_prepare() noexcept { #ifdef HAVE_PMEM if (log_sys.is_mmap()) return; #endif - while (flush_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); - while (write_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (write_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); } -void log_t::clear_mmap() +void log_t::clear_mmap() noexcept { - if (!is_mmap() || + if (!is_mmap() || high_level_read_only) + return; #ifdef HAVE_PMEM - !is_opened() || -#endif - high_level_read_only) + if (!is_opened()) + { + latch.wr_lock(SRW_LOCK_CALL); + ut_ad(!resize_in_progress()); + ut_ad(get_lsn() == get_flushed_lsn(std::memory_order_relaxed)); + buf_size= unsigned(std::min(capacity(), buf_size_max)); + latch.wr_unlock(); return; + } +#endif + log_resize_acquire(); ut_ad(!resize_in_progress()); ut_ad(write_lsn == get_lsn()); @@ -1242,10 +1255,10 @@ void log_t::clear_mmap() { alignas(16) byte log_block[4096]; const size_t bs{write_size}; - const size_t bf{buf_free.load(std::memory_order_relaxed)}; { - byte *const b= buf; - memcpy_aligned<16>(log_block, b + (bf & ~(bs - 1)), bs); + const size_t bf= + size_t(write_lsn - base_lsn.load(std::memory_order_relaxed)); + memcpy_aligned<16>(log_block, buf + (bf & ~(bs - 1)), bs); } close_file(false); @@ -1253,14 +1266,13 @@ void log_t::clear_mmap() ut_a(attach(log, file_size)); ut_ad(!is_mmap()); - set_buf_free(bf & (bs - 1)); - memcpy_aligned<16>(log_sys.buf, log_block, bs); + memcpy_aligned<16>(buf, log_block, bs); } log_resize_release(); } /** Durably write the log up to log_sys.get_lsn(). */ -ATTRIBUTE_COLD void log_write_and_flush() +ATTRIBUTE_COLD void log_write_and_flush() noexcept { ut_ad(!srv_read_only_mode); #ifdef HAVE_PMEM @@ -1280,17 +1292,17 @@ Tries to establish a big enough margin of free space in the log, such that a new log entry can be catenated without an immediate need for a checkpoint. NOTE: this function may only be called if the calling thread owns no synchronization objects! */ -ATTRIBUTE_COLD static void log_checkpoint_margin() +ATTRIBUTE_COLD static void log_checkpoint_margin() noexcept { while (log_sys.check_for_checkpoint()) { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); ut_ad(!recv_no_log_write); if (!log_sys.check_for_checkpoint()) { func_exit: - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); return; } @@ -1308,7 +1320,7 @@ ATTRIBUTE_COLD static void log_checkpoint_margin() } DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", goto skip_checkpoint;); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); /* We must wait to prevent the tail of the log overwriting the head. */ buf_flush_wait_flushed(std::min(sync_lsn, checkpoint + (1U << 20))); @@ -1320,7 +1332,7 @@ ATTRIBUTE_COLD static void log_checkpoint_margin() /** Wait for a log checkpoint if needed. NOTE that this function may only be called while not holding any synchronization objects except dict_sys.latch. */ -void log_free_check() +void log_free_check() noexcept { ut_ad(!lock_sys.is_holder()); if (log_sys.check_for_checkpoint()) @@ -1337,7 +1349,7 @@ inline void buf_mem_pressure_shutdown() noexcept {} #endif /** Make a checkpoint at the latest lsn on shutdown. */ -ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() +ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() noexcept { lsn_t lsn; ulint count = 0; @@ -1474,7 +1486,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT; - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); lsn = log_sys.get_lsn(); @@ -1482,7 +1494,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() && lsn != log_sys.last_checkpoint_lsn + sizeof_cp; ut_ad(lsn >= log_sys.last_checkpoint_lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); if (lsn_changed) { goto loop; @@ -1500,7 +1512,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() "Free innodb buffer pool"); ut_d(buf_pool.assert_all_freed()); - ut_a(lsn == log_sys.get_lsn() + ut_a(lsn == log_get_lsn() || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); if (UNIV_UNLIKELY(lsn < recv_sys.lsn)) { @@ -1514,7 +1526,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() /* Make some checks that the server really is quiet */ ut_ad(!srv_any_background_activity()); - ut_a(lsn == log_sys.get_lsn() + ut_a(lsn == log_get_lsn() || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); } @@ -1525,44 +1537,42 @@ log_print( /*======*/ FILE* file) /*!< in: file where to print */ { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); const lsn_t lsn= log_sys.get_lsn(); mysql_mutex_lock(&buf_pool.flush_list_mutex); const lsn_t pages_flushed = buf_pool.get_oldest_modification(lsn); mysql_mutex_unlock(&buf_pool.flush_list_mutex); + const lsn_t flushed_lsn{log_sys.get_flushed_lsn()}; + const lsn_t checkpoint_lsn{log_sys.last_checkpoint_lsn}; + log_sys.latch.wr_unlock(); fprintf(file, "Log sequence number " LSN_PF "\n" "Log flushed up to " LSN_PF "\n" "Pages flushed up to " LSN_PF "\n" "Last checkpoint at " LSN_PF "\n", - lsn, - log_sys.get_flushed_lsn(), - pages_flushed, - lsn_t{log_sys.last_checkpoint_lsn}); - - log_sys.latch.rd_unlock(); + lsn, flushed_lsn, pages_flushed, checkpoint_lsn); } /** Shut down the redo log subsystem. */ void log_t::close() { ut_ad(this == &log_sys); - ut_ad(!(buf_free & buf_free_LOCK)); if (!is_initialised()) return; close_file(); ut_ad(!checkpoint_buf); ut_ad(!buf); ut_ad(!flush_buf); + base_lsn.store(0, std::memory_order_relaxed); latch.destroy(); - lsn_lock.destroy(); +#ifdef HAVE_PMEM + resize_wrap_mutex.destroy(); +#endif recv_sys.close(); - - max_buf_free= 0; } std::string get_log_file_path(const char *filename) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 9a4bac91035cd..30b99ea0bc7fb 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1466,6 +1466,7 @@ void recv_sys_t::debug_free() mysql_mutex_lock(&mutex); recovery_on= false; + recv_needed_recovery= false; pages.clear(); pages_it= pages.end(); @@ -1473,7 +1474,6 @@ void recv_sys_t::debug_free() log_sys.clear_mmap(); } - /** Free a redo log snippet. @param data buffer allocated in add() */ inline void recv_sys_t::free(const void *data) @@ -2972,7 +2972,7 @@ recv_sys_t::parse_mtr_result recv_sys_t::parse(source &l, bool if_exists) l - recs + rlen))) { lsn= start_lsn; - if (lsn > log_sys.get_lsn()) + if (lsn > log_sys.get_flushed_lsn(std::memory_order_relaxed)) log_sys.set_recovered_lsn(start_lsn); l+= rlen; offset= begin.ptr - log_sys.buf; @@ -4582,19 +4582,16 @@ dberr_t recv_recovery_read_checkpoint() inline void log_t::set_recovered() noexcept { ut_ad(get_flushed_lsn() == get_lsn()); - ut_ad(recv_sys.lsn == get_lsn()); - size_t offset{recv_sys.offset}; + ut_ad(recv_sys.lsn == get_flushed_lsn()); if (!is_mmap()) { const size_t bs{log_sys.write_size}, bs_1{bs - 1}; - memmove_aligned<512>(buf, buf + (offset & ~bs_1), bs); - offset&= bs_1; + memmove_aligned<512>(buf, buf + (recv_sys.offset & ~bs_1), bs); } -#ifndef _WIN32 +#ifdef HAVE_PMEM else mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); #endif - set_buf_free(offset); } inline bool recv_sys_t::validate_checkpoint() const noexcept @@ -4668,7 +4665,7 @@ dberr_t recv_recovery_from_checkpoint_start() goto err_exit; } ut_ad(recv_sys.file_checkpoint); - ut_ad(log_sys.get_lsn() >= recv_sys.scanned_lsn); + ut_ad(log_sys.get_flushed_lsn() >= recv_sys.scanned_lsn); if (rewind) { recv_sys.lsn = log_sys.next_checkpoint_lsn; recv_sys.offset = 0; @@ -4730,7 +4727,7 @@ dberr_t recv_recovery_from_checkpoint_start() tablespaces (not individual pages), while retaining the initial recv_sys.pages. */ mysql_mutex_lock(&recv_sys.mutex); - ut_ad(log_sys.get_lsn() >= recv_sys.lsn); + ut_ad(log_sys.get_flushed_lsn() >= recv_sys.lsn); recv_sys.clear(); recv_sys.lsn = log_sys.next_checkpoint_lsn; mysql_mutex_unlock(&recv_sys.mutex); @@ -4738,7 +4735,8 @@ dberr_t recv_recovery_from_checkpoint_start() if (srv_operation <= SRV_OPERATION_EXPORT_RESTORED) { mysql_mutex_lock(&recv_sys.mutex); - deferred_spaces.deferred_dblwr(log_sys.get_lsn()); + deferred_spaces.deferred_dblwr( + log_sys.get_flushed_lsn()); buf_dblwr.recover(); mysql_mutex_unlock(&recv_sys.mutex); } @@ -4771,16 +4769,6 @@ dberr_t recv_recovery_from_checkpoint_start() if (!srv_read_only_mode && log_sys.is_latest()) { log_sys.set_recovered(); - if (recv_needed_recovery - && srv_operation <= SRV_OPERATION_EXPORT_RESTORED - && recv_sys.lsn - log_sys.next_checkpoint_lsn - < log_sys.log_capacity) { - /* Write a FILE_CHECKPOINT marker as the first thing, - before generating any other redo log. This ensures - that subsequent crash recovery will be possible even - if the server were killed soon after this. */ - fil_names_clear(log_sys.next_checkpoint_lsn); - } } DBUG_EXECUTE_IF("before_final_redo_apply", goto err_exit;); diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index bb632bd66b7f5..716dac624d5a8 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -44,7 +44,6 @@ void (*mtr_t::commit_logger)(mtr_t *, std::pair); #endif std::pair (*mtr_t::finisher)(mtr_t *, size_t); -unsigned mtr_t::spin_wait_delay; void mtr_t::finisher_update() { @@ -53,15 +52,12 @@ void mtr_t::finisher_update() if (log_sys.is_mmap()) { commit_logger= mtr_t::commit_log; - finisher= spin_wait_delay - ? mtr_t::finish_writer : mtr_t::finish_writer; + finisher= mtr_t::finish_writer; return; } commit_logger= mtr_t::commit_log; #endif - finisher= - (spin_wait_delay - ? mtr_t::finish_writer : mtr_t::finish_writer); + finisher= mtr_t::finish_writer; } void mtr_memo_slot_t::release() const @@ -257,7 +253,7 @@ static void insert_imported(buf_block_t *block) { if (block->page.oldest_modification() <= 1) { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); /* For unlogged mtrs (MTR_LOG_NO_REDO), we use the current system LSN. The mtr that generated the LSN is either already committed or in mtr_t::commit. Shared latch and relaxed atomics should be fine here as it is guaranteed @@ -269,7 +265,7 @@ static void insert_imported(buf_block_t *block) mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_pool.insert_into_flush_list (buf_pool.prepare_insert_into_flush_list(lsn), block, lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); mysql_mutex_unlock(&buf_pool.flush_list_mutex); } } @@ -339,24 +335,11 @@ void mtr_t::release() m_memo.clear(); } -inline lsn_t log_t::get_write_target() const -{ - ut_ad(latch_have_any()); - if (UNIV_LIKELY(buf_free_ok())) - return 0; - /* The LSN corresponding to the end of buf is - write_lsn - (first_lsn & 4095) + buf_free, - but we use simpler arithmetics to return a smaller write target in - order to minimize waiting in log_write_up_to(). */ - ut_ad(max_buf_free >= 4096 * 4); - return write_lsn + max_buf_free / 2; -} - template void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) + noexcept { size_t modified= 0; - const lsn_t write_lsn= mmap ? 0 : log_sys.get_write_target(); if (mtr->m_made_dirty) { @@ -475,9 +458,6 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) if (UNIV_UNLIKELY(lsns.second != PAGE_FLUSH_NO)) buf_flush_ahead(mtr->m_commit_lsn, lsns.second == PAGE_FLUSH_SYNC); - - if (!mmap && UNIV_UNLIKELY(write_lsn != 0)) - log_write_up_to(write_lsn, false); } /** Commit a mini-transaction. */ @@ -690,7 +670,7 @@ bool mtr_t::commit_file(fil_space_t &space, const char *name) /* We will not encrypt any FILE_ records, but we will reserve a nonce at the end. */ size+= 8; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); } else m_commit_lsn= 0; @@ -775,7 +755,7 @@ ATTRIBUTE_COLD lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn) /* We will not encrypt any FILE_ records, but we will reserve a nonce at the end. */ size+= 8; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); } else m_commit_lsn= 0; @@ -897,181 +877,110 @@ ATTRIBUTE_COLD static void log_overwrite_warning(lsn_t lsn) ? ". Shutdown is in progress" : ""); } -static ATTRIBUTE_NOINLINE void lsn_delay(size_t delay, size_t mult) noexcept -{ - delay*= mult * 2; // GCC 13.2.0 -O2 targeting AMD64 wants to unroll twice - HMT_low(); - do - MY_RELAX_CPU(); - while (--delay); - HMT_medium(); -} - -#if defined __clang_major__ && __clang_major__ < 10 -/* Only clang-10 introduced support for asm goto */ -#elif defined __APPLE__ -/* At least some versions of Apple Xcode do not support asm goto */ -#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) -# if SIZEOF_SIZE_T == 8 -# define LOCK_TSET \ - __asm__ goto("lock btsq $63, %0\n\t" "jnc %l1" \ - : : "m"(buf_free) : "cc", "memory" : got) -# else -# define LOCK_TSET \ - __asm__ goto("lock btsl $31, %0\n\t" "jnc %l1" \ - : : "m"(buf_free) : "cc", "memory" : got) -# endif -#elif defined _MSC_VER && (defined _M_IX86 || defined _M_X64) -# if SIZEOF_SIZE_T == 8 -# define LOCK_TSET \ - if (!_interlockedbittestandset64 \ - (reinterpret_cast(&buf_free), 63)) return -# else -# define LOCK_TSET \ - if (!_interlockedbittestandset \ - (reinterpret_cast(&buf_free), 31)) return -# endif -#endif - -#ifdef LOCK_TSET -ATTRIBUTE_NOINLINE -void log_t::lsn_lock_bts() noexcept +ATTRIBUTE_COLD void log_t::append_prepare_wait(bool late, bool ex) noexcept { - LOCK_TSET; + if (UNIV_LIKELY(!ex)) { - const size_t m= mtr_t::spin_wait_delay; - constexpr size_t DELAY= 10, MAX_ITERATIONS= 10; - for (size_t delay_count= DELAY, delay_iterations= 1;; - lsn_delay(delay_iterations, m)) + latch.rd_unlock(); + if (!late) { - if (!(buf_free.load(std::memory_order_relaxed) & buf_free_LOCK)) - LOCK_TSET; - if (!delay_count); - else if (delay_iterations < MAX_ITERATIONS) - delay_count= DELAY, delay_iterations++; - else - delay_count--; + /* Wait for all threads to back off. */ + latch.wr_lock(SRW_LOCK_CALL); + goto got_ex; } - } -# ifdef __GNUC__ - got: - return; -# endif -} + const auto delay= my_cpu_relax_multiplier / 4 * srv_spin_wait_delay; + const auto rounds= srv_n_spin_wait_rounds; -inline -#else -ATTRIBUTE_NOINLINE -#endif -size_t log_t::lock_lsn() noexcept -{ -#ifdef LOCK_TSET - lsn_lock_bts(); - return ~buf_free_LOCK & buf_free.load(std::memory_order_relaxed); -# undef LOCK_TSET -#else - size_t b= buf_free.fetch_or(buf_free_LOCK, std::memory_order_acquire); - if (b & buf_free_LOCK) - { - const size_t m= mtr_t::spin_wait_delay; - constexpr size_t DELAY= 10, MAX_ITERATIONS= 10; - for (size_t delay_count= DELAY, delay_iterations= 1; - ((b= buf_free.load(std::memory_order_relaxed)) & buf_free_LOCK) || - (buf_free_LOCK & (b= buf_free.fetch_or(buf_free_LOCK, - std::memory_order_acquire))); - lsn_delay(delay_iterations, m)) - if (!delay_count); - else if (delay_iterations < MAX_ITERATIONS) - delay_count= DELAY, delay_iterations++; - else - delay_count--; + for (;;) + { + HMT_low(); + for (auto r= rounds + 1; r--; ) + { + if (write_lsn_offset.load(std::memory_order_relaxed) & WRITE_BACKOFF) + { + for (auto d= delay; d--; ) + MY_RELAX_CPU(); + } + else + { + HMT_medium(); + goto done; + } + } + HMT_medium(); + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } } - return b; -#endif -} - -template -ATTRIBUTE_COLD size_t log_t::append_prepare_wait(size_t b, bool ex, lsn_t lsn) - noexcept -{ - waits++; - ut_ad(buf_free.load(std::memory_order_relaxed) == - (spin ? (b | buf_free_LOCK) : b)); - if (spin) - buf_free.store(b, std::memory_order_release); else - lsn_lock.wr_unlock(); - - if (ex) + { + got_ex: + const uint64_t l= write_lsn_offset.load(std::memory_order_relaxed); + const lsn_t lsn{base_lsn.load(std::memory_order_relaxed)}; + ut_d(lsn_t ll= lsn + (l & (WRITE_BACKOFF - 1))); + ut_ad(is_mmap() + ? ll - get_flushed_lsn(std::memory_order_relaxed) < capacity() + : ll - write_lsn - ((write_size - 1) & (write_lsn - first_lsn)) < + buf_size); + waits++; +#ifdef HAVE_PMEM + const bool is_pmem{is_mmap()}; + if (is_pmem) + persist(lsn + (l & (WRITE_BACKOFF - 1))); +#endif latch.wr_unlock(); - else - latch.rd_unlock(); - - log_write_up_to(lsn, is_mmap()); - - if (ex) - latch.wr_lock(SRW_LOCK_CALL); - else - latch.rd_lock(SRW_LOCK_CALL); - - if (spin) - return lock_lsn(); + /* write_buf() or persist() will clear the WRITE_BACKOFF flag, + which our caller will recheck. */ +#ifdef HAVE_PMEM + if (!is_pmem) +#endif + log_write_up_to(lsn + (l & (WRITE_BACKOFF - 1)), false); + if (ex) + { + latch.wr_lock(SRW_LOCK_CALL); + return; + } + } - lsn_lock.wr_lock(); - return buf_free.load(std::memory_order_relaxed); +done: + latch.rd_lock(SRW_LOCK_CALL); } /** Reserve space in the log buffer for appending data. -@tparam spin whether to use the spin-only lock_lsn() @tparam mmap log_sys.is_mmap() @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ -template +template inline std::pair log_t::append_prepare(size_t size, bool ex) noexcept { ut_ad(ex ? latch_have_wr() : latch_have_rd()); ut_ad(mmap == is_mmap()); - if (!spin) - lsn_lock.wr_lock(); - size_t b{spin ? lock_lsn() : buf_free.load(std::memory_order_relaxed)}; - write_to_buf++; - - lsn_t l{lsn.load(std::memory_order_relaxed)}, end_lsn{l + size}; - - if (UNIV_UNLIKELY(mmap - ? (end_lsn - - get_flushed_lsn(std::memory_order_relaxed)) > capacity() - : b + size >= buf_size)) + ut_ad(!mmap || buf_size == std::min(capacity(), buf_size_max)); + const size_t buf_size{this->buf_size - size}; + uint64_t l; + static_assert(WRITE_TO_BUF == WRITE_BACKOFF << 1, ""); + while (UNIV_UNLIKELY((l= write_lsn_offset.fetch_add(size + WRITE_TO_BUF) & + (WRITE_TO_BUF - 1)) >= buf_size)) { - b= append_prepare_wait(b, ex, l); - /* While flushing log, we had released the lsn lock and LSN could have - progressed in the meantime. */ - l= lsn.load(std::memory_order_relaxed); - end_lsn= l + size; + /* The following is inlined here instead of being part of + append_prepare_wait(), in order to increase the locality of reference + and to set the WRITE_BACKOFF flag as soon as possible. */ + bool late(write_lsn_offset.fetch_or(WRITE_BACKOFF) & WRITE_BACKOFF); + /* Subtract our LSN overshoot. */ + write_lsn_offset.fetch_sub(size); + append_prepare_wait(late, ex); } - size_t new_buf_free= b + size; - if (mmap && new_buf_free >= file_size) - new_buf_free-= size_t(capacity()); - - lsn.store(end_lsn, std::memory_order_relaxed); + const lsn_t lsn{l + base_lsn.load(std::memory_order_relaxed)}, + end_lsn{lsn + size}; if (UNIV_UNLIKELY(end_lsn >= last_checkpoint_lsn + log_capacity)) set_check_for_checkpoint(true); - byte *our_buf= buf; - if (spin) - buf_free.store(new_buf_free, std::memory_order_release); - else - { - buf_free.store(new_buf_free, std::memory_order_relaxed); - lsn_lock.wr_unlock(); - } - - return {l, our_buf + b}; + return {lsn, + buf + size_t(mmap ? FIRST_LSN + (lsn - first_lsn) % capacity() : l)}; } /** Finish appending data to the log. @@ -1216,7 +1125,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, if (!resize_flush_buf) { ut_ad(is_mmap()); - lsn_lock.wr_lock(); + resize_wrap_mutex.wr_lock(); const size_t resize_capacity{resize_target - START_OFFSET}; { const lsn_t resizing{resize_in_progress()}; @@ -1227,7 +1136,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, if (UNIV_UNLIKELY(lsn < resizing)) { /* This function may execute in multiple concurrent threads - that hold a shared log_sys.latch. Before we got lsn_lock, + that hold a shared log_sys.latch. Before we got resize_wrap_mutex, another thread could have executed resize_lsn.store(lsn) below with a larger lsn than ours. @@ -1277,7 +1186,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, ut_ad(resize_buf[s] <= 1); resize_buf[s]= 1; mmap_done: - lsn_lock.wr_unlock(); + resize_wrap_mutex.wr_unlock(); } else #endif @@ -1304,7 +1213,7 @@ inline void log_t::append(byte *&d, const void *s, size_t size) noexcept d+= size; } -template +template std::pair mtr_t::finish_writer(mtr_t *mtr, size_t len) { @@ -1315,7 +1224,7 @@ mtr_t::finish_writer(mtr_t *mtr, size_t len) const size_t size{mtr->m_commit_lsn ? 5U + 8U : 5U}; std::pair start= - log_sys.append_prepare(len, mtr->m_latch_ex); + log_sys.append_prepare(len, mtr->m_latch_ex); if (!mmap) { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 30b4f5975a144..cb06ec9f52a4f 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -69,7 +69,7 @@ Created 9/17/2000 Heikki Tuuri /** Delay an INSERT, DELETE or UPDATE operation if the purge is lagging. */ -static void row_mysql_delay_if_needed() +static void row_mysql_delay_if_needed() noexcept { const auto delay= srv_dml_needed_delay; if (UNIV_UNLIKELY(delay != 0)) @@ -78,8 +78,8 @@ static void row_mysql_delay_if_needed() log_sys.latch.rd_lock(SRW_LOCK_CALL); const lsn_t last= log_sys.last_checkpoint_lsn, max_age= log_sys.max_checkpoint_age; + const lsn_t lsn= log_sys.get_flushed_lsn(); log_sys.latch.rd_unlock(); - const lsn_t lsn= log_sys.get_lsn(); if ((lsn - last) / 4 >= max_age / 5) buf_flush_ahead(last + max_age / 5, false); purge_sys.wake_if_not_active(); diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index a9bc310753bc1..ff646ba1b79a8 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -1454,7 +1454,7 @@ srv_mon_process_existing_counter( /* innodb_os_log_written */ case MONITOR_OVLD_OS_LOG_WRITTEN: - value = log_sys.get_lsn() - recv_sys.lsn; + value = log_get_lsn() - recv_sys.lsn; break; /* innodb_log_waits */ @@ -1587,7 +1587,7 @@ srv_mon_process_existing_counter( break; case MONITOR_OVLD_LSN_CURRENT: - value = log_sys.get_lsn(); + value = log_get_lsn(); break; case MONITOR_OVLD_CHECKPOINTS: @@ -1595,10 +1595,10 @@ srv_mon_process_existing_counter( break; case MONITOR_LSN_CHECKPOINT_AGE: - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); value = static_cast(log_sys.get_lsn() - log_sys.last_checkpoint_lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); break; case MONITOR_OVLD_BUF_OLDEST_LSN: diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 10f148a50da2f..0a21c69fb3899 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -979,13 +979,13 @@ srv_export_innodb_status(void) mysql_mutex_unlock(&srv_innodb_monitor_mutex); - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); export_vars.innodb_lsn_current = log_sys.get_lsn(); export_vars.innodb_lsn_flushed = log_sys.get_flushed_lsn(); export_vars.innodb_lsn_last_checkpoint = log_sys.last_checkpoint_lsn; export_vars.innodb_checkpoint_max_age = static_cast( log_sys.max_checkpoint_age); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); export_vars.innodb_os_log_written = export_vars.innodb_lsn_current - recv_sys.lsn; @@ -1072,7 +1072,7 @@ void srv_monitor_task(void*) /* Try to track a strange bug reported by Harald Fuchs and others, where the lsn seems to decrease at times */ - lsn_t new_lsn = log_sys.get_lsn(); + lsn_t new_lsn = log_get_lsn(); ut_a(new_lsn >= old_lsn); old_lsn = new_lsn; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index dc2ad75848f73..3577cd7b1184f 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1057,7 +1057,7 @@ srv_init_abort_low( /** Prepare to delete the redo log file. Flush the dirty pages from all the buffer pools. Flush the redo log buffer to the redo log file. @return lsn upto which data pages have been flushed. */ -static lsn_t srv_prepare_to_delete_redo_log_file() +static lsn_t srv_prepare_to_delete_redo_log_file() noexcept { DBUG_ENTER("srv_prepare_to_delete_redo_log_file"); @@ -1071,7 +1071,7 @@ static lsn_t srv_prepare_to_delete_redo_log_file() log_sys.latch.wr_lock(SRW_LOCK_CALL); const bool latest_format{log_sys.is_latest()}; - lsn_t flushed_lsn{log_sys.get_lsn()}; + lsn_t flushed_lsn{log_sys.get_flushed_lsn(std::memory_order_relaxed)}; if (latest_format && !(log_sys.file_size & 4095) && flushed_lsn != log_sys.next_checkpoint_lsn + @@ -1079,6 +1079,11 @@ static lsn_t srv_prepare_to_delete_redo_log_file() ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT)) { +#ifdef HAVE_PMEM + if (!log_sys.is_opened()) + log_sys.buf_size= unsigned(std::min(log_sys.capacity(), + log_sys.buf_size_max)); +#endif fil_names_clear(flushed_lsn); flushed_lsn= log_sys.get_lsn(); } @@ -1119,7 +1124,7 @@ static lsn_t srv_prepare_to_delete_redo_log_file() if (latest_format) log_write_up_to(flushed_lsn, false); - ut_ad(flushed_lsn == log_sys.get_lsn()); + ut_ad(flushed_lsn == log_get_lsn()); ut_ad(!os_aio_pending_reads()); ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); ut_ad(!buf_pool.get_oldest_modification(0)); @@ -1134,6 +1139,18 @@ static tpool::task rollback_all_recovered_task(trx_rollback_all_recovered, nullptr, &rollback_all_recovered_group); +inline lsn_t log_t::init_lsn() noexcept +{ + latch.wr_lock(SRW_LOCK_CALL); + ut_ad(!write_lsn_offset); + write_lsn_offset= 0; + const lsn_t lsn{base_lsn.load(std::memory_order_relaxed)}; + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn= lsn; + latch.wr_unlock(); + return lsn; +} + /** Start InnoDB. @param[in] create_new_db whether to create a new database @return DB_SUCCESS or error code */